Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 */
5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6#include <linux/libnvdimm.h>
7#include <linux/sched/mm.h>
8#include <linux/vmalloc.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/blkdev.h>
12#include <linux/fcntl.h>
13#include <linux/async.h>
14#include <linux/ndctl.h>
15#include <linux/sched.h>
16#include <linux/slab.h>
17#include <linux/cpu.h>
18#include <linux/fs.h>
19#include <linux/io.h>
20#include <linux/mm.h>
21#include <linux/nd.h>
22#include "nd-core.h"
23#include "nd.h"
24#include "pfn.h"
25
26int nvdimm_major;
27static int nvdimm_bus_major;
28static struct class *nd_class;
29static DEFINE_IDA(nd_ida);
30
31static int to_nd_device_type(const struct device *dev)
32{
33 if (is_nvdimm(dev))
34 return ND_DEVICE_DIMM;
35 else if (is_memory(dev))
36 return ND_DEVICE_REGION_PMEM;
37 else if (is_nd_dax(dev))
38 return ND_DEVICE_DAX_PMEM;
39 else if (is_nd_region(dev->parent))
40 return nd_region_to_nstype(to_nd_region(dev->parent));
41
42 return 0;
43}
44
45static int nvdimm_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
46{
47 return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
48 to_nd_device_type(dev));
49}
50
51static struct module *to_bus_provider(struct device *dev)
52{
53 /* pin bus providers while regions are enabled */
54 if (is_nd_region(dev)) {
55 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
56
57 return nvdimm_bus->nd_desc->module;
58 }
59 return NULL;
60}
61
62static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
63{
64 nvdimm_bus_lock(&nvdimm_bus->dev);
65 nvdimm_bus->probe_active++;
66 nvdimm_bus_unlock(&nvdimm_bus->dev);
67}
68
69static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
70{
71 nvdimm_bus_lock(&nvdimm_bus->dev);
72 if (--nvdimm_bus->probe_active == 0)
73 wake_up(&nvdimm_bus->wait);
74 nvdimm_bus_unlock(&nvdimm_bus->dev);
75}
76
77static int nvdimm_bus_probe(struct device *dev)
78{
79 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
80 struct module *provider = to_bus_provider(dev);
81 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
82 int rc;
83
84 if (!try_module_get(provider))
85 return -ENXIO;
86
87 dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
88 dev->driver->name, dev_name(dev));
89
90 nvdimm_bus_probe_start(nvdimm_bus);
91 rc = nd_drv->probe(dev);
92 if ((rc == 0 || rc == -EOPNOTSUPP) &&
93 dev->parent && is_nd_region(dev->parent))
94 nd_region_advance_seeds(to_nd_region(dev->parent), dev);
95 nvdimm_bus_probe_end(nvdimm_bus);
96
97 dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
98 dev_name(dev), rc);
99
100 if (rc != 0)
101 module_put(provider);
102 return rc;
103}
104
105static void nvdimm_bus_remove(struct device *dev)
106{
107 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
108 struct module *provider = to_bus_provider(dev);
109 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
110
111 if (nd_drv->remove)
112 nd_drv->remove(dev);
113
114 dev_dbg(&nvdimm_bus->dev, "%s.remove(%s)\n", dev->driver->name,
115 dev_name(dev));
116 module_put(provider);
117}
118
119static void nvdimm_bus_shutdown(struct device *dev)
120{
121 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
122 struct nd_device_driver *nd_drv = NULL;
123
124 if (dev->driver)
125 nd_drv = to_nd_device_driver(dev->driver);
126
127 if (nd_drv && nd_drv->shutdown) {
128 nd_drv->shutdown(dev);
129 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
130 dev->driver->name, dev_name(dev));
131 }
132}
133
134void nd_device_notify(struct device *dev, enum nvdimm_event event)
135{
136 device_lock(dev);
137 if (dev->driver) {
138 struct nd_device_driver *nd_drv;
139
140 nd_drv = to_nd_device_driver(dev->driver);
141 if (nd_drv->notify)
142 nd_drv->notify(dev, event);
143 }
144 device_unlock(dev);
145}
146EXPORT_SYMBOL(nd_device_notify);
147
148void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
149{
150 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
151
152 if (!nvdimm_bus)
153 return;
154
155 /* caller is responsible for holding a reference on the device */
156 nd_device_notify(&nd_region->dev, event);
157}
158EXPORT_SYMBOL_GPL(nvdimm_region_notify);
159
160struct clear_badblocks_context {
161 resource_size_t phys, cleared;
162};
163
164static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
165{
166 struct clear_badblocks_context *ctx = data;
167 struct nd_region *nd_region;
168 resource_size_t ndr_end;
169 sector_t sector;
170
171 /* make sure device is a region */
172 if (!is_memory(dev))
173 return 0;
174
175 nd_region = to_nd_region(dev);
176 ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
177
178 /* make sure we are in the region */
179 if (ctx->phys < nd_region->ndr_start ||
180 (ctx->phys + ctx->cleared - 1) > ndr_end)
181 return 0;
182
183 sector = (ctx->phys - nd_region->ndr_start) / 512;
184 badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
185
186 if (nd_region->bb_state)
187 sysfs_notify_dirent(nd_region->bb_state);
188
189 return 0;
190}
191
192static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
193 phys_addr_t phys, u64 cleared)
194{
195 struct clear_badblocks_context ctx = {
196 .phys = phys,
197 .cleared = cleared,
198 };
199
200 device_for_each_child(&nvdimm_bus->dev, &ctx,
201 nvdimm_clear_badblocks_region);
202}
203
204static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
205 phys_addr_t phys, u64 cleared)
206{
207 if (cleared > 0)
208 badrange_forget(&nvdimm_bus->badrange, phys, cleared);
209
210 if (cleared > 0 && cleared / 512)
211 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
212}
213
214long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
215 unsigned int len)
216{
217 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
218 struct nvdimm_bus_descriptor *nd_desc;
219 struct nd_cmd_clear_error clear_err;
220 struct nd_cmd_ars_cap ars_cap;
221 u32 clear_err_unit, mask;
222 unsigned int noio_flag;
223 int cmd_rc, rc;
224
225 if (!nvdimm_bus)
226 return -ENXIO;
227
228 nd_desc = nvdimm_bus->nd_desc;
229 /*
230 * if ndctl does not exist, it's PMEM_LEGACY and
231 * we want to just pretend everything is handled.
232 */
233 if (!nd_desc->ndctl)
234 return len;
235
236 memset(&ars_cap, 0, sizeof(ars_cap));
237 ars_cap.address = phys;
238 ars_cap.length = len;
239 noio_flag = memalloc_noio_save();
240 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
241 sizeof(ars_cap), &cmd_rc);
242 memalloc_noio_restore(noio_flag);
243 if (rc < 0)
244 return rc;
245 if (cmd_rc < 0)
246 return cmd_rc;
247 clear_err_unit = ars_cap.clear_err_unit;
248 if (!clear_err_unit || !is_power_of_2(clear_err_unit))
249 return -ENXIO;
250
251 mask = clear_err_unit - 1;
252 if ((phys | len) & mask)
253 return -ENXIO;
254 memset(&clear_err, 0, sizeof(clear_err));
255 clear_err.address = phys;
256 clear_err.length = len;
257 noio_flag = memalloc_noio_save();
258 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
259 sizeof(clear_err), &cmd_rc);
260 memalloc_noio_restore(noio_flag);
261 if (rc < 0)
262 return rc;
263 if (cmd_rc < 0)
264 return cmd_rc;
265
266 nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
267
268 return clear_err.cleared;
269}
270EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
271
272static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
273
274static struct bus_type nvdimm_bus_type = {
275 .name = "nd",
276 .uevent = nvdimm_bus_uevent,
277 .match = nvdimm_bus_match,
278 .probe = nvdimm_bus_probe,
279 .remove = nvdimm_bus_remove,
280 .shutdown = nvdimm_bus_shutdown,
281};
282
283static void nvdimm_bus_release(struct device *dev)
284{
285 struct nvdimm_bus *nvdimm_bus;
286
287 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
288 ida_free(&nd_ida, nvdimm_bus->id);
289 kfree(nvdimm_bus);
290}
291
292static const struct device_type nvdimm_bus_dev_type = {
293 .release = nvdimm_bus_release,
294 .groups = nvdimm_bus_attribute_groups,
295};
296
297bool is_nvdimm_bus(struct device *dev)
298{
299 return dev->type == &nvdimm_bus_dev_type;
300}
301
302struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
303{
304 struct device *dev;
305
306 for (dev = nd_dev; dev; dev = dev->parent)
307 if (is_nvdimm_bus(dev))
308 break;
309 dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
310 if (dev)
311 return to_nvdimm_bus(dev);
312 return NULL;
313}
314
315struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
316{
317 struct nvdimm_bus *nvdimm_bus;
318
319 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
320 WARN_ON(!is_nvdimm_bus(dev));
321 return nvdimm_bus;
322}
323EXPORT_SYMBOL_GPL(to_nvdimm_bus);
324
325struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
326{
327 return to_nvdimm_bus(nvdimm->dev.parent);
328}
329EXPORT_SYMBOL_GPL(nvdimm_to_bus);
330
331static struct lock_class_key nvdimm_bus_key;
332
333struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
334 struct nvdimm_bus_descriptor *nd_desc)
335{
336 struct nvdimm_bus *nvdimm_bus;
337 int rc;
338
339 nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
340 if (!nvdimm_bus)
341 return NULL;
342 INIT_LIST_HEAD(&nvdimm_bus->list);
343 INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
344 init_waitqueue_head(&nvdimm_bus->wait);
345 nvdimm_bus->id = ida_alloc(&nd_ida, GFP_KERNEL);
346 if (nvdimm_bus->id < 0) {
347 kfree(nvdimm_bus);
348 return NULL;
349 }
350 mutex_init(&nvdimm_bus->reconfig_mutex);
351 badrange_init(&nvdimm_bus->badrange);
352 nvdimm_bus->nd_desc = nd_desc;
353 nvdimm_bus->dev.parent = parent;
354 nvdimm_bus->dev.type = &nvdimm_bus_dev_type;
355 nvdimm_bus->dev.groups = nd_desc->attr_groups;
356 nvdimm_bus->dev.bus = &nvdimm_bus_type;
357 nvdimm_bus->dev.of_node = nd_desc->of_node;
358 device_initialize(&nvdimm_bus->dev);
359 lockdep_set_class(&nvdimm_bus->dev.mutex, &nvdimm_bus_key);
360 device_set_pm_not_required(&nvdimm_bus->dev);
361 rc = dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
362 if (rc)
363 goto err;
364
365 rc = device_add(&nvdimm_bus->dev);
366 if (rc) {
367 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
368 goto err;
369 }
370
371 return nvdimm_bus;
372 err:
373 put_device(&nvdimm_bus->dev);
374 return NULL;
375}
376EXPORT_SYMBOL_GPL(nvdimm_bus_register);
377
378void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
379{
380 if (!nvdimm_bus)
381 return;
382 device_unregister(&nvdimm_bus->dev);
383}
384EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
385
386static int child_unregister(struct device *dev, void *data)
387{
388 /*
389 * the singular ndctl class device per bus needs to be
390 * "device_destroy"ed, so skip it here
391 *
392 * i.e. remove classless children
393 */
394 if (dev->class)
395 return 0;
396
397 if (is_nvdimm(dev))
398 nvdimm_delete(to_nvdimm(dev));
399 else
400 nd_device_unregister(dev, ND_SYNC);
401
402 return 0;
403}
404
405static void free_badrange_list(struct list_head *badrange_list)
406{
407 struct badrange_entry *bre, *next;
408
409 list_for_each_entry_safe(bre, next, badrange_list, list) {
410 list_del(&bre->list);
411 kfree(bre);
412 }
413 list_del_init(badrange_list);
414}
415
416static void nd_bus_remove(struct device *dev)
417{
418 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
419
420 mutex_lock(&nvdimm_bus_list_mutex);
421 list_del_init(&nvdimm_bus->list);
422 mutex_unlock(&nvdimm_bus_list_mutex);
423
424 wait_event(nvdimm_bus->wait,
425 atomic_read(&nvdimm_bus->ioctl_active) == 0);
426
427 nd_synchronize();
428 device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
429
430 spin_lock(&nvdimm_bus->badrange.lock);
431 free_badrange_list(&nvdimm_bus->badrange.list);
432 spin_unlock(&nvdimm_bus->badrange.lock);
433
434 nvdimm_bus_destroy_ndctl(nvdimm_bus);
435}
436
437static int nd_bus_probe(struct device *dev)
438{
439 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
440 int rc;
441
442 rc = nvdimm_bus_create_ndctl(nvdimm_bus);
443 if (rc)
444 return rc;
445
446 mutex_lock(&nvdimm_bus_list_mutex);
447 list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
448 mutex_unlock(&nvdimm_bus_list_mutex);
449
450 /* enable bus provider attributes to look up their local context */
451 dev_set_drvdata(dev, nvdimm_bus->nd_desc);
452
453 return 0;
454}
455
456static struct nd_device_driver nd_bus_driver = {
457 .probe = nd_bus_probe,
458 .remove = nd_bus_remove,
459 .drv = {
460 .name = "nd_bus",
461 .suppress_bind_attrs = true,
462 .bus = &nvdimm_bus_type,
463 .owner = THIS_MODULE,
464 .mod_name = KBUILD_MODNAME,
465 },
466};
467
468static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
469{
470 struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
471
472 if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
473 return true;
474
475 return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
476}
477
478static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
479
480void nd_synchronize(void)
481{
482 async_synchronize_full_domain(&nd_async_domain);
483}
484EXPORT_SYMBOL_GPL(nd_synchronize);
485
486static void nd_async_device_register(void *d, async_cookie_t cookie)
487{
488 struct device *dev = d;
489
490 if (device_add(dev) != 0) {
491 dev_err(dev, "%s: failed\n", __func__);
492 put_device(dev);
493 }
494 put_device(dev);
495 if (dev->parent)
496 put_device(dev->parent);
497}
498
499static void nd_async_device_unregister(void *d, async_cookie_t cookie)
500{
501 struct device *dev = d;
502
503 /* flush bus operations before delete */
504 nvdimm_bus_lock(dev);
505 nvdimm_bus_unlock(dev);
506
507 device_unregister(dev);
508 put_device(dev);
509}
510
511static void __nd_device_register(struct device *dev, bool sync)
512{
513 if (!dev)
514 return;
515
516 /*
517 * Ensure that region devices always have their NUMA node set as
518 * early as possible. This way we are able to make certain that
519 * any memory associated with the creation and the creation
520 * itself of the region is associated with the correct node.
521 */
522 if (is_nd_region(dev))
523 set_dev_node(dev, to_nd_region(dev)->numa_node);
524
525 dev->bus = &nvdimm_bus_type;
526 device_set_pm_not_required(dev);
527 if (dev->parent) {
528 get_device(dev->parent);
529 if (dev_to_node(dev) == NUMA_NO_NODE)
530 set_dev_node(dev, dev_to_node(dev->parent));
531 }
532 get_device(dev);
533
534 if (sync)
535 nd_async_device_register(dev, 0);
536 else
537 async_schedule_dev_domain(nd_async_device_register, dev,
538 &nd_async_domain);
539}
540
541void nd_device_register(struct device *dev)
542{
543 __nd_device_register(dev, false);
544}
545EXPORT_SYMBOL(nd_device_register);
546
547void nd_device_register_sync(struct device *dev)
548{
549 __nd_device_register(dev, true);
550}
551
552void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
553{
554 bool killed;
555
556 switch (mode) {
557 case ND_ASYNC:
558 /*
559 * In the async case this is being triggered with the
560 * device lock held and the unregistration work needs to
561 * be moved out of line iff this is thread has won the
562 * race to schedule the deletion.
563 */
564 if (!kill_device(dev))
565 return;
566
567 get_device(dev);
568 async_schedule_domain(nd_async_device_unregister, dev,
569 &nd_async_domain);
570 break;
571 case ND_SYNC:
572 /*
573 * In the sync case the device is being unregistered due
574 * to a state change of the parent. Claim the kill state
575 * to synchronize against other unregistration requests,
576 * or otherwise let the async path handle it if the
577 * unregistration was already queued.
578 */
579 device_lock(dev);
580 killed = kill_device(dev);
581 device_unlock(dev);
582
583 if (!killed)
584 return;
585
586 nd_synchronize();
587 device_unregister(dev);
588 break;
589 }
590}
591EXPORT_SYMBOL(nd_device_unregister);
592
593/**
594 * __nd_driver_register() - register a region or a namespace driver
595 * @nd_drv: driver to register
596 * @owner: automatically set by nd_driver_register() macro
597 * @mod_name: automatically set by nd_driver_register() macro
598 */
599int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
600 const char *mod_name)
601{
602 struct device_driver *drv = &nd_drv->drv;
603
604 if (!nd_drv->type) {
605 pr_debug("driver type bitmask not set (%ps)\n",
606 __builtin_return_address(0));
607 return -EINVAL;
608 }
609
610 if (!nd_drv->probe) {
611 pr_debug("%s ->probe() must be specified\n", mod_name);
612 return -EINVAL;
613 }
614
615 drv->bus = &nvdimm_bus_type;
616 drv->owner = owner;
617 drv->mod_name = mod_name;
618
619 return driver_register(drv);
620}
621EXPORT_SYMBOL(__nd_driver_register);
622
623void nvdimm_check_and_set_ro(struct gendisk *disk)
624{
625 struct device *dev = disk_to_dev(disk)->parent;
626 struct nd_region *nd_region = to_nd_region(dev->parent);
627 int disk_ro = get_disk_ro(disk);
628
629 /* catch the disk up with the region ro state */
630 if (disk_ro == nd_region->ro)
631 return;
632
633 dev_info(dev, "%s read-%s, marking %s read-%s\n",
634 dev_name(&nd_region->dev), nd_region->ro ? "only" : "write",
635 disk->disk_name, nd_region->ro ? "only" : "write");
636 set_disk_ro(disk, nd_region->ro);
637}
638EXPORT_SYMBOL(nvdimm_check_and_set_ro);
639
640static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
641 char *buf)
642{
643 return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
644 to_nd_device_type(dev));
645}
646static DEVICE_ATTR_RO(modalias);
647
648static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
649 char *buf)
650{
651 return sprintf(buf, "%s\n", dev->type->name);
652}
653static DEVICE_ATTR_RO(devtype);
654
655static struct attribute *nd_device_attributes[] = {
656 &dev_attr_modalias.attr,
657 &dev_attr_devtype.attr,
658 NULL,
659};
660
661/*
662 * nd_device_attribute_group - generic attributes for all devices on an nd bus
663 */
664const struct attribute_group nd_device_attribute_group = {
665 .attrs = nd_device_attributes,
666};
667
668static ssize_t numa_node_show(struct device *dev,
669 struct device_attribute *attr, char *buf)
670{
671 return sprintf(buf, "%d\n", dev_to_node(dev));
672}
673static DEVICE_ATTR_RO(numa_node);
674
675static int nvdimm_dev_to_target_node(struct device *dev)
676{
677 struct device *parent = dev->parent;
678 struct nd_region *nd_region = NULL;
679
680 if (is_nd_region(dev))
681 nd_region = to_nd_region(dev);
682 else if (parent && is_nd_region(parent))
683 nd_region = to_nd_region(parent);
684
685 if (!nd_region)
686 return NUMA_NO_NODE;
687 return nd_region->target_node;
688}
689
690static ssize_t target_node_show(struct device *dev,
691 struct device_attribute *attr, char *buf)
692{
693 return sprintf(buf, "%d\n", nvdimm_dev_to_target_node(dev));
694}
695static DEVICE_ATTR_RO(target_node);
696
697static struct attribute *nd_numa_attributes[] = {
698 &dev_attr_numa_node.attr,
699 &dev_attr_target_node.attr,
700 NULL,
701};
702
703static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
704 int n)
705{
706 struct device *dev = container_of(kobj, typeof(*dev), kobj);
707
708 if (!IS_ENABLED(CONFIG_NUMA))
709 return 0;
710
711 if (a == &dev_attr_target_node.attr &&
712 nvdimm_dev_to_target_node(dev) == NUMA_NO_NODE)
713 return 0;
714
715 return a->mode;
716}
717
718/*
719 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
720 */
721const struct attribute_group nd_numa_attribute_group = {
722 .attrs = nd_numa_attributes,
723 .is_visible = nd_numa_attr_visible,
724};
725
726static void ndctl_release(struct device *dev)
727{
728 kfree(dev);
729}
730
731static struct lock_class_key nvdimm_ndctl_key;
732
733int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
734{
735 dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
736 struct device *dev;
737 int rc;
738
739 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
740 if (!dev)
741 return -ENOMEM;
742 device_initialize(dev);
743 lockdep_set_class(&dev->mutex, &nvdimm_ndctl_key);
744 device_set_pm_not_required(dev);
745 dev->class = nd_class;
746 dev->parent = &nvdimm_bus->dev;
747 dev->devt = devt;
748 dev->release = ndctl_release;
749 rc = dev_set_name(dev, "ndctl%d", nvdimm_bus->id);
750 if (rc)
751 goto err;
752
753 rc = device_add(dev);
754 if (rc) {
755 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %d\n",
756 nvdimm_bus->id, rc);
757 goto err;
758 }
759 return 0;
760
761err:
762 put_device(dev);
763 return rc;
764}
765
766void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
767{
768 device_destroy(nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
769}
770
771static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
772 [ND_CMD_IMPLEMENTED] = { },
773 [ND_CMD_SMART] = {
774 .out_num = 2,
775 .out_sizes = { 4, 128, },
776 },
777 [ND_CMD_SMART_THRESHOLD] = {
778 .out_num = 2,
779 .out_sizes = { 4, 8, },
780 },
781 [ND_CMD_DIMM_FLAGS] = {
782 .out_num = 2,
783 .out_sizes = { 4, 4 },
784 },
785 [ND_CMD_GET_CONFIG_SIZE] = {
786 .out_num = 3,
787 .out_sizes = { 4, 4, 4, },
788 },
789 [ND_CMD_GET_CONFIG_DATA] = {
790 .in_num = 2,
791 .in_sizes = { 4, 4, },
792 .out_num = 2,
793 .out_sizes = { 4, UINT_MAX, },
794 },
795 [ND_CMD_SET_CONFIG_DATA] = {
796 .in_num = 3,
797 .in_sizes = { 4, 4, UINT_MAX, },
798 .out_num = 1,
799 .out_sizes = { 4, },
800 },
801 [ND_CMD_VENDOR] = {
802 .in_num = 3,
803 .in_sizes = { 4, 4, UINT_MAX, },
804 .out_num = 3,
805 .out_sizes = { 4, 4, UINT_MAX, },
806 },
807 [ND_CMD_CALL] = {
808 .in_num = 2,
809 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
810 .out_num = 1,
811 .out_sizes = { UINT_MAX, },
812 },
813};
814
815const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
816{
817 if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
818 return &__nd_cmd_dimm_descs[cmd];
819 return NULL;
820}
821EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
822
823static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
824 [ND_CMD_IMPLEMENTED] = { },
825 [ND_CMD_ARS_CAP] = {
826 .in_num = 2,
827 .in_sizes = { 8, 8, },
828 .out_num = 4,
829 .out_sizes = { 4, 4, 4, 4, },
830 },
831 [ND_CMD_ARS_START] = {
832 .in_num = 5,
833 .in_sizes = { 8, 8, 2, 1, 5, },
834 .out_num = 2,
835 .out_sizes = { 4, 4, },
836 },
837 [ND_CMD_ARS_STATUS] = {
838 .out_num = 3,
839 .out_sizes = { 4, 4, UINT_MAX, },
840 },
841 [ND_CMD_CLEAR_ERROR] = {
842 .in_num = 2,
843 .in_sizes = { 8, 8, },
844 .out_num = 3,
845 .out_sizes = { 4, 4, 8, },
846 },
847 [ND_CMD_CALL] = {
848 .in_num = 2,
849 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
850 .out_num = 1,
851 .out_sizes = { UINT_MAX, },
852 },
853};
854
855const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
856{
857 if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
858 return &__nd_cmd_bus_descs[cmd];
859 return NULL;
860}
861EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
862
863u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
864 const struct nd_cmd_desc *desc, int idx, void *buf)
865{
866 if (idx >= desc->in_num)
867 return UINT_MAX;
868
869 if (desc->in_sizes[idx] < UINT_MAX)
870 return desc->in_sizes[idx];
871
872 if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
873 struct nd_cmd_set_config_hdr *hdr = buf;
874
875 return hdr->in_length;
876 } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
877 struct nd_cmd_vendor_hdr *hdr = buf;
878
879 return hdr->in_length;
880 } else if (cmd == ND_CMD_CALL) {
881 struct nd_cmd_pkg *pkg = buf;
882
883 return pkg->nd_size_in;
884 }
885
886 return UINT_MAX;
887}
888EXPORT_SYMBOL_GPL(nd_cmd_in_size);
889
890u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
891 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
892 const u32 *out_field, unsigned long remainder)
893{
894 if (idx >= desc->out_num)
895 return UINT_MAX;
896
897 if (desc->out_sizes[idx] < UINT_MAX)
898 return desc->out_sizes[idx];
899
900 if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
901 return in_field[1];
902 else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
903 return out_field[1];
904 else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
905 /*
906 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
907 * "Size of Output Buffer in bytes, including this
908 * field."
909 */
910 if (out_field[1] < 4)
911 return 0;
912 /*
913 * ACPI 6.1 is ambiguous if 'status' is included in the
914 * output size. If we encounter an output size that
915 * overshoots the remainder by 4 bytes, assume it was
916 * including 'status'.
917 */
918 if (out_field[1] - 4 == remainder)
919 return remainder;
920 return out_field[1] - 8;
921 } else if (cmd == ND_CMD_CALL) {
922 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
923
924 return pkg->nd_size_out;
925 }
926
927
928 return UINT_MAX;
929}
930EXPORT_SYMBOL_GPL(nd_cmd_out_size);
931
932void wait_nvdimm_bus_probe_idle(struct device *dev)
933{
934 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
935
936 do {
937 if (nvdimm_bus->probe_active == 0)
938 break;
939 nvdimm_bus_unlock(dev);
940 device_unlock(dev);
941 wait_event(nvdimm_bus->wait,
942 nvdimm_bus->probe_active == 0);
943 device_lock(dev);
944 nvdimm_bus_lock(dev);
945 } while (true);
946}
947
948static int nd_pmem_forget_poison_check(struct device *dev, void *data)
949{
950 struct nd_cmd_clear_error *clear_err =
951 (struct nd_cmd_clear_error *)data;
952 struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
953 struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
954 struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
955 struct nd_namespace_common *ndns = NULL;
956 struct nd_namespace_io *nsio;
957 resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
958
959 if (nd_dax || !dev->driver)
960 return 0;
961
962 start = clear_err->address;
963 end = clear_err->address + clear_err->cleared - 1;
964
965 if (nd_btt || nd_pfn || nd_dax) {
966 if (nd_btt)
967 ndns = nd_btt->ndns;
968 else if (nd_pfn)
969 ndns = nd_pfn->ndns;
970 else if (nd_dax)
971 ndns = nd_dax->nd_pfn.ndns;
972
973 if (!ndns)
974 return 0;
975 } else
976 ndns = to_ndns(dev);
977
978 nsio = to_nd_namespace_io(&ndns->dev);
979 pstart = nsio->res.start + offset;
980 pend = nsio->res.end - end_trunc;
981
982 if ((pstart >= start) && (pend <= end))
983 return -EBUSY;
984
985 return 0;
986
987}
988
989static int nd_ns_forget_poison_check(struct device *dev, void *data)
990{
991 return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
992}
993
994/* set_config requires an idle interleave set */
995static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
996 struct nvdimm *nvdimm, unsigned int cmd, void *data)
997{
998 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
999
1000 /* ask the bus provider if it would like to block this request */
1001 if (nd_desc->clear_to_send) {
1002 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
1003
1004 if (rc)
1005 return rc;
1006 }
1007
1008 /* require clear error to go through the pmem driver */
1009 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
1010 return device_for_each_child(&nvdimm_bus->dev, data,
1011 nd_ns_forget_poison_check);
1012
1013 if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
1014 return 0;
1015
1016 /* prevent label manipulation while the kernel owns label updates */
1017 wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
1018 if (atomic_read(&nvdimm->busy))
1019 return -EBUSY;
1020 return 0;
1021}
1022
1023static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
1024 int read_only, unsigned int ioctl_cmd, unsigned long arg)
1025{
1026 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
1027 const struct nd_cmd_desc *desc = NULL;
1028 unsigned int cmd = _IOC_NR(ioctl_cmd);
1029 struct device *dev = &nvdimm_bus->dev;
1030 void __user *p = (void __user *) arg;
1031 char *out_env = NULL, *in_env = NULL;
1032 const char *cmd_name, *dimm_name;
1033 u32 in_len = 0, out_len = 0;
1034 unsigned int func = cmd;
1035 unsigned long cmd_mask;
1036 struct nd_cmd_pkg pkg;
1037 int rc, i, cmd_rc;
1038 void *buf = NULL;
1039 u64 buf_len = 0;
1040
1041 if (nvdimm) {
1042 desc = nd_cmd_dimm_desc(cmd);
1043 cmd_name = nvdimm_cmd_name(cmd);
1044 cmd_mask = nvdimm->cmd_mask;
1045 dimm_name = dev_name(&nvdimm->dev);
1046 } else {
1047 desc = nd_cmd_bus_desc(cmd);
1048 cmd_name = nvdimm_bus_cmd_name(cmd);
1049 cmd_mask = nd_desc->cmd_mask;
1050 dimm_name = "bus";
1051 }
1052
1053 /* Validate command family support against bus declared support */
1054 if (cmd == ND_CMD_CALL) {
1055 unsigned long *mask;
1056
1057 if (copy_from_user(&pkg, p, sizeof(pkg)))
1058 return -EFAULT;
1059
1060 if (nvdimm) {
1061 if (pkg.nd_family > NVDIMM_FAMILY_MAX)
1062 return -EINVAL;
1063 mask = &nd_desc->dimm_family_mask;
1064 } else {
1065 if (pkg.nd_family > NVDIMM_BUS_FAMILY_MAX)
1066 return -EINVAL;
1067 mask = &nd_desc->bus_family_mask;
1068 }
1069
1070 if (!test_bit(pkg.nd_family, mask))
1071 return -EINVAL;
1072 }
1073
1074 if (!desc ||
1075 (desc->out_num + desc->in_num == 0) ||
1076 cmd > ND_CMD_CALL ||
1077 !test_bit(cmd, &cmd_mask))
1078 return -ENOTTY;
1079
1080 /* fail write commands (when read-only) */
1081 if (read_only)
1082 switch (cmd) {
1083 case ND_CMD_VENDOR:
1084 case ND_CMD_SET_CONFIG_DATA:
1085 case ND_CMD_ARS_START:
1086 case ND_CMD_CLEAR_ERROR:
1087 case ND_CMD_CALL:
1088 dev_dbg(dev, "'%s' command while read-only.\n",
1089 nvdimm ? nvdimm_cmd_name(cmd)
1090 : nvdimm_bus_cmd_name(cmd));
1091 return -EPERM;
1092 default:
1093 break;
1094 }
1095
1096 /* process an input envelope */
1097 in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1098 if (!in_env)
1099 return -ENOMEM;
1100 for (i = 0; i < desc->in_num; i++) {
1101 u32 in_size, copy;
1102
1103 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
1104 if (in_size == UINT_MAX) {
1105 dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
1106 __func__, dimm_name, cmd_name, i);
1107 rc = -ENXIO;
1108 goto out;
1109 }
1110 if (in_len < ND_CMD_MAX_ENVELOPE)
1111 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
1112 else
1113 copy = 0;
1114 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
1115 rc = -EFAULT;
1116 goto out;
1117 }
1118 in_len += in_size;
1119 }
1120
1121 if (cmd == ND_CMD_CALL) {
1122 func = pkg.nd_command;
1123 dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
1124 dimm_name, pkg.nd_command,
1125 in_len, out_len, buf_len);
1126 }
1127
1128 /* process an output envelope */
1129 out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1130 if (!out_env) {
1131 rc = -ENOMEM;
1132 goto out;
1133 }
1134
1135 for (i = 0; i < desc->out_num; i++) {
1136 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
1137 (u32 *) in_env, (u32 *) out_env, 0);
1138 u32 copy;
1139
1140 if (out_size == UINT_MAX) {
1141 dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
1142 dimm_name, cmd_name, i);
1143 rc = -EFAULT;
1144 goto out;
1145 }
1146 if (out_len < ND_CMD_MAX_ENVELOPE)
1147 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
1148 else
1149 copy = 0;
1150 if (copy && copy_from_user(&out_env[out_len],
1151 p + in_len + out_len, copy)) {
1152 rc = -EFAULT;
1153 goto out;
1154 }
1155 out_len += out_size;
1156 }
1157
1158 buf_len = (u64) out_len + (u64) in_len;
1159 if (buf_len > ND_IOCTL_MAX_BUFLEN) {
1160 dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
1161 cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
1162 rc = -EINVAL;
1163 goto out;
1164 }
1165
1166 buf = vmalloc(buf_len);
1167 if (!buf) {
1168 rc = -ENOMEM;
1169 goto out;
1170 }
1171
1172 if (copy_from_user(buf, p, buf_len)) {
1173 rc = -EFAULT;
1174 goto out;
1175 }
1176
1177 device_lock(dev);
1178 nvdimm_bus_lock(dev);
1179 rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
1180 if (rc)
1181 goto out_unlock;
1182
1183 rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
1184 if (rc < 0)
1185 goto out_unlock;
1186
1187 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
1188 struct nd_cmd_clear_error *clear_err = buf;
1189
1190 nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
1191 clear_err->cleared);
1192 }
1193
1194 if (copy_to_user(p, buf, buf_len))
1195 rc = -EFAULT;
1196
1197out_unlock:
1198 nvdimm_bus_unlock(dev);
1199 device_unlock(dev);
1200out:
1201 kfree(in_env);
1202 kfree(out_env);
1203 vfree(buf);
1204 return rc;
1205}
1206
1207enum nd_ioctl_mode {
1208 BUS_IOCTL,
1209 DIMM_IOCTL,
1210};
1211
1212static int match_dimm(struct device *dev, void *data)
1213{
1214 long id = (long) data;
1215
1216 if (is_nvdimm(dev)) {
1217 struct nvdimm *nvdimm = to_nvdimm(dev);
1218
1219 return nvdimm->id == id;
1220 }
1221
1222 return 0;
1223}
1224
1225static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
1226 enum nd_ioctl_mode mode)
1227
1228{
1229 struct nvdimm_bus *nvdimm_bus, *found = NULL;
1230 long id = (long) file->private_data;
1231 struct nvdimm *nvdimm = NULL;
1232 int rc, ro;
1233
1234 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
1235 mutex_lock(&nvdimm_bus_list_mutex);
1236 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
1237 if (mode == DIMM_IOCTL) {
1238 struct device *dev;
1239
1240 dev = device_find_child(&nvdimm_bus->dev,
1241 file->private_data, match_dimm);
1242 if (!dev)
1243 continue;
1244 nvdimm = to_nvdimm(dev);
1245 found = nvdimm_bus;
1246 } else if (nvdimm_bus->id == id) {
1247 found = nvdimm_bus;
1248 }
1249
1250 if (found) {
1251 atomic_inc(&nvdimm_bus->ioctl_active);
1252 break;
1253 }
1254 }
1255 mutex_unlock(&nvdimm_bus_list_mutex);
1256
1257 if (!found)
1258 return -ENXIO;
1259
1260 nvdimm_bus = found;
1261 rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
1262
1263 if (nvdimm)
1264 put_device(&nvdimm->dev);
1265 if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
1266 wake_up(&nvdimm_bus->wait);
1267
1268 return rc;
1269}
1270
1271static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1272{
1273 return nd_ioctl(file, cmd, arg, BUS_IOCTL);
1274}
1275
1276static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1277{
1278 return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
1279}
1280
1281static int nd_open(struct inode *inode, struct file *file)
1282{
1283 long minor = iminor(inode);
1284
1285 file->private_data = (void *) minor;
1286 return 0;
1287}
1288
1289static const struct file_operations nvdimm_bus_fops = {
1290 .owner = THIS_MODULE,
1291 .open = nd_open,
1292 .unlocked_ioctl = bus_ioctl,
1293 .compat_ioctl = compat_ptr_ioctl,
1294 .llseek = noop_llseek,
1295};
1296
1297static const struct file_operations nvdimm_fops = {
1298 .owner = THIS_MODULE,
1299 .open = nd_open,
1300 .unlocked_ioctl = dimm_ioctl,
1301 .compat_ioctl = compat_ptr_ioctl,
1302 .llseek = noop_llseek,
1303};
1304
1305int __init nvdimm_bus_init(void)
1306{
1307 int rc;
1308
1309 rc = bus_register(&nvdimm_bus_type);
1310 if (rc)
1311 return rc;
1312
1313 rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
1314 if (rc < 0)
1315 goto err_bus_chrdev;
1316 nvdimm_bus_major = rc;
1317
1318 rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
1319 if (rc < 0)
1320 goto err_dimm_chrdev;
1321 nvdimm_major = rc;
1322
1323 nd_class = class_create("nd");
1324 if (IS_ERR(nd_class)) {
1325 rc = PTR_ERR(nd_class);
1326 goto err_class;
1327 }
1328
1329 rc = driver_register(&nd_bus_driver.drv);
1330 if (rc)
1331 goto err_nd_bus;
1332
1333 return 0;
1334
1335 err_nd_bus:
1336 class_destroy(nd_class);
1337 err_class:
1338 unregister_chrdev(nvdimm_major, "dimmctl");
1339 err_dimm_chrdev:
1340 unregister_chrdev(nvdimm_bus_major, "ndctl");
1341 err_bus_chrdev:
1342 bus_unregister(&nvdimm_bus_type);
1343
1344 return rc;
1345}
1346
1347void nvdimm_bus_exit(void)
1348{
1349 driver_unregister(&nd_bus_driver.drv);
1350 class_destroy(nd_class);
1351 unregister_chrdev(nvdimm_bus_major, "ndctl");
1352 unregister_chrdev(nvdimm_major, "dimmctl");
1353 bus_unregister(&nvdimm_bus_type);
1354 ida_destroy(&nd_ida);
1355}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 */
5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6#include <linux/libnvdimm.h>
7#include <linux/sched/mm.h>
8#include <linux/vmalloc.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/blkdev.h>
12#include <linux/fcntl.h>
13#include <linux/async.h>
14#include <linux/ndctl.h>
15#include <linux/sched.h>
16#include <linux/slab.h>
17#include <linux/cpu.h>
18#include <linux/fs.h>
19#include <linux/io.h>
20#include <linux/mm.h>
21#include <linux/nd.h>
22#include "nd-core.h"
23#include "nd.h"
24#include "pfn.h"
25
26int nvdimm_major;
27static int nvdimm_bus_major;
28static DEFINE_IDA(nd_ida);
29
30static const struct class nd_class = {
31 .name = "nd",
32};
33
34static int to_nd_device_type(const struct device *dev)
35{
36 if (is_nvdimm(dev))
37 return ND_DEVICE_DIMM;
38 else if (is_memory(dev))
39 return ND_DEVICE_REGION_PMEM;
40 else if (is_nd_dax(dev))
41 return ND_DEVICE_DAX_PMEM;
42 else if (is_nd_region(dev->parent))
43 return nd_region_to_nstype(to_nd_region(dev->parent));
44
45 return 0;
46}
47
48static int nvdimm_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
49{
50 return add_uevent_var(env, "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
51 to_nd_device_type(dev));
52}
53
54static struct module *to_bus_provider(struct device *dev)
55{
56 /* pin bus providers while regions are enabled */
57 if (is_nd_region(dev)) {
58 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
59
60 return nvdimm_bus->nd_desc->module;
61 }
62 return NULL;
63}
64
65static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
66{
67 nvdimm_bus_lock(&nvdimm_bus->dev);
68 nvdimm_bus->probe_active++;
69 nvdimm_bus_unlock(&nvdimm_bus->dev);
70}
71
72static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
73{
74 nvdimm_bus_lock(&nvdimm_bus->dev);
75 if (--nvdimm_bus->probe_active == 0)
76 wake_up(&nvdimm_bus->wait);
77 nvdimm_bus_unlock(&nvdimm_bus->dev);
78}
79
80static int nvdimm_bus_probe(struct device *dev)
81{
82 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
83 struct module *provider = to_bus_provider(dev);
84 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
85 int rc;
86
87 if (!try_module_get(provider))
88 return -ENXIO;
89
90 dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
91 dev->driver->name, dev_name(dev));
92
93 nvdimm_bus_probe_start(nvdimm_bus);
94 rc = nd_drv->probe(dev);
95 if ((rc == 0 || rc == -EOPNOTSUPP) &&
96 dev->parent && is_nd_region(dev->parent))
97 nd_region_advance_seeds(to_nd_region(dev->parent), dev);
98 nvdimm_bus_probe_end(nvdimm_bus);
99
100 dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
101 dev_name(dev), rc);
102
103 if (rc != 0)
104 module_put(provider);
105 return rc;
106}
107
108static void nvdimm_bus_remove(struct device *dev)
109{
110 struct nd_device_driver *nd_drv = to_nd_device_driver(dev->driver);
111 struct module *provider = to_bus_provider(dev);
112 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
113
114 if (nd_drv->remove)
115 nd_drv->remove(dev);
116
117 dev_dbg(&nvdimm_bus->dev, "%s.remove(%s)\n", dev->driver->name,
118 dev_name(dev));
119 module_put(provider);
120}
121
122static void nvdimm_bus_shutdown(struct device *dev)
123{
124 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
125 struct nd_device_driver *nd_drv = NULL;
126
127 if (dev->driver)
128 nd_drv = to_nd_device_driver(dev->driver);
129
130 if (nd_drv && nd_drv->shutdown) {
131 nd_drv->shutdown(dev);
132 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
133 dev->driver->name, dev_name(dev));
134 }
135}
136
137void nd_device_notify(struct device *dev, enum nvdimm_event event)
138{
139 device_lock(dev);
140 if (dev->driver) {
141 struct nd_device_driver *nd_drv;
142
143 nd_drv = to_nd_device_driver(dev->driver);
144 if (nd_drv->notify)
145 nd_drv->notify(dev, event);
146 }
147 device_unlock(dev);
148}
149EXPORT_SYMBOL(nd_device_notify);
150
151void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
152{
153 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev);
154
155 if (!nvdimm_bus)
156 return;
157
158 /* caller is responsible for holding a reference on the device */
159 nd_device_notify(&nd_region->dev, event);
160}
161EXPORT_SYMBOL_GPL(nvdimm_region_notify);
162
163struct clear_badblocks_context {
164 resource_size_t phys, cleared;
165};
166
167static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
168{
169 struct clear_badblocks_context *ctx = data;
170 struct nd_region *nd_region;
171 resource_size_t ndr_end;
172 sector_t sector;
173
174 /* make sure device is a region */
175 if (!is_memory(dev))
176 return 0;
177
178 nd_region = to_nd_region(dev);
179 ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
180
181 /* make sure we are in the region */
182 if (ctx->phys < nd_region->ndr_start ||
183 (ctx->phys + ctx->cleared - 1) > ndr_end)
184 return 0;
185
186 sector = (ctx->phys - nd_region->ndr_start) / 512;
187 badblocks_clear(&nd_region->bb, sector, ctx->cleared / 512);
188
189 if (nd_region->bb_state)
190 sysfs_notify_dirent(nd_region->bb_state);
191
192 return 0;
193}
194
195static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
196 phys_addr_t phys, u64 cleared)
197{
198 struct clear_badblocks_context ctx = {
199 .phys = phys,
200 .cleared = cleared,
201 };
202
203 device_for_each_child(&nvdimm_bus->dev, &ctx,
204 nvdimm_clear_badblocks_region);
205}
206
207static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
208 phys_addr_t phys, u64 cleared)
209{
210 if (cleared > 0)
211 badrange_forget(&nvdimm_bus->badrange, phys, cleared);
212
213 if (cleared > 0 && cleared / 512)
214 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
215}
216
217long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
218 unsigned int len)
219{
220 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
221 struct nvdimm_bus_descriptor *nd_desc;
222 struct nd_cmd_clear_error clear_err;
223 struct nd_cmd_ars_cap ars_cap;
224 u32 clear_err_unit, mask;
225 unsigned int noio_flag;
226 int cmd_rc, rc;
227
228 if (!nvdimm_bus)
229 return -ENXIO;
230
231 nd_desc = nvdimm_bus->nd_desc;
232 /*
233 * if ndctl does not exist, it's PMEM_LEGACY and
234 * we want to just pretend everything is handled.
235 */
236 if (!nd_desc->ndctl)
237 return len;
238
239 memset(&ars_cap, 0, sizeof(ars_cap));
240 ars_cap.address = phys;
241 ars_cap.length = len;
242 noio_flag = memalloc_noio_save();
243 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
244 sizeof(ars_cap), &cmd_rc);
245 memalloc_noio_restore(noio_flag);
246 if (rc < 0)
247 return rc;
248 if (cmd_rc < 0)
249 return cmd_rc;
250 clear_err_unit = ars_cap.clear_err_unit;
251 if (!clear_err_unit || !is_power_of_2(clear_err_unit))
252 return -ENXIO;
253
254 mask = clear_err_unit - 1;
255 if ((phys | len) & mask)
256 return -ENXIO;
257 memset(&clear_err, 0, sizeof(clear_err));
258 clear_err.address = phys;
259 clear_err.length = len;
260 noio_flag = memalloc_noio_save();
261 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
262 sizeof(clear_err), &cmd_rc);
263 memalloc_noio_restore(noio_flag);
264 if (rc < 0)
265 return rc;
266 if (cmd_rc < 0)
267 return cmd_rc;
268
269 nvdimm_account_cleared_poison(nvdimm_bus, phys, clear_err.cleared);
270
271 return clear_err.cleared;
272}
273EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
274
275static int nvdimm_bus_match(struct device *dev, const struct device_driver *drv);
276
277static const struct bus_type nvdimm_bus_type = {
278 .name = "nd",
279 .uevent = nvdimm_bus_uevent,
280 .match = nvdimm_bus_match,
281 .probe = nvdimm_bus_probe,
282 .remove = nvdimm_bus_remove,
283 .shutdown = nvdimm_bus_shutdown,
284};
285
286static void nvdimm_bus_release(struct device *dev)
287{
288 struct nvdimm_bus *nvdimm_bus;
289
290 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
291 ida_free(&nd_ida, nvdimm_bus->id);
292 kfree(nvdimm_bus);
293}
294
295static const struct device_type nvdimm_bus_dev_type = {
296 .release = nvdimm_bus_release,
297 .groups = nvdimm_bus_attribute_groups,
298};
299
300bool is_nvdimm_bus(struct device *dev)
301{
302 return dev->type == &nvdimm_bus_dev_type;
303}
304
305struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
306{
307 struct device *dev;
308
309 for (dev = nd_dev; dev; dev = dev->parent)
310 if (is_nvdimm_bus(dev))
311 break;
312 dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
313 if (dev)
314 return to_nvdimm_bus(dev);
315 return NULL;
316}
317
318struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
319{
320 struct nvdimm_bus *nvdimm_bus;
321
322 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
323 WARN_ON(!is_nvdimm_bus(dev));
324 return nvdimm_bus;
325}
326EXPORT_SYMBOL_GPL(to_nvdimm_bus);
327
328struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
329{
330 return to_nvdimm_bus(nvdimm->dev.parent);
331}
332EXPORT_SYMBOL_GPL(nvdimm_to_bus);
333
334static struct lock_class_key nvdimm_bus_key;
335
336struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
337 struct nvdimm_bus_descriptor *nd_desc)
338{
339 struct nvdimm_bus *nvdimm_bus;
340 int rc;
341
342 nvdimm_bus = kzalloc(sizeof(*nvdimm_bus), GFP_KERNEL);
343 if (!nvdimm_bus)
344 return NULL;
345 INIT_LIST_HEAD(&nvdimm_bus->list);
346 INIT_LIST_HEAD(&nvdimm_bus->mapping_list);
347 init_waitqueue_head(&nvdimm_bus->wait);
348 nvdimm_bus->id = ida_alloc(&nd_ida, GFP_KERNEL);
349 if (nvdimm_bus->id < 0) {
350 kfree(nvdimm_bus);
351 return NULL;
352 }
353 mutex_init(&nvdimm_bus->reconfig_mutex);
354 badrange_init(&nvdimm_bus->badrange);
355 nvdimm_bus->nd_desc = nd_desc;
356 nvdimm_bus->dev.parent = parent;
357 nvdimm_bus->dev.type = &nvdimm_bus_dev_type;
358 nvdimm_bus->dev.groups = nd_desc->attr_groups;
359 nvdimm_bus->dev.bus = &nvdimm_bus_type;
360 nvdimm_bus->dev.of_node = nd_desc->of_node;
361 device_initialize(&nvdimm_bus->dev);
362 lockdep_set_class(&nvdimm_bus->dev.mutex, &nvdimm_bus_key);
363 device_set_pm_not_required(&nvdimm_bus->dev);
364 rc = dev_set_name(&nvdimm_bus->dev, "ndbus%d", nvdimm_bus->id);
365 if (rc)
366 goto err;
367
368 rc = device_add(&nvdimm_bus->dev);
369 if (rc) {
370 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
371 goto err;
372 }
373
374 return nvdimm_bus;
375 err:
376 put_device(&nvdimm_bus->dev);
377 return NULL;
378}
379EXPORT_SYMBOL_GPL(nvdimm_bus_register);
380
381void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
382{
383 if (!nvdimm_bus)
384 return;
385 device_unregister(&nvdimm_bus->dev);
386}
387EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
388
389static int child_unregister(struct device *dev, void *data)
390{
391 /*
392 * the singular ndctl class device per bus needs to be
393 * "device_destroy"ed, so skip it here
394 *
395 * i.e. remove classless children
396 */
397 if (dev->class)
398 return 0;
399
400 if (is_nvdimm(dev))
401 nvdimm_delete(to_nvdimm(dev));
402 else
403 nd_device_unregister(dev, ND_SYNC);
404
405 return 0;
406}
407
408static void free_badrange_list(struct list_head *badrange_list)
409{
410 struct badrange_entry *bre, *next;
411
412 list_for_each_entry_safe(bre, next, badrange_list, list) {
413 list_del(&bre->list);
414 kfree(bre);
415 }
416 list_del_init(badrange_list);
417}
418
419static void nd_bus_remove(struct device *dev)
420{
421 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
422
423 mutex_lock(&nvdimm_bus_list_mutex);
424 list_del_init(&nvdimm_bus->list);
425 mutex_unlock(&nvdimm_bus_list_mutex);
426
427 wait_event(nvdimm_bus->wait,
428 atomic_read(&nvdimm_bus->ioctl_active) == 0);
429
430 nd_synchronize();
431 device_for_each_child(&nvdimm_bus->dev, NULL, child_unregister);
432
433 spin_lock(&nvdimm_bus->badrange.lock);
434 free_badrange_list(&nvdimm_bus->badrange.list);
435 spin_unlock(&nvdimm_bus->badrange.lock);
436
437 nvdimm_bus_destroy_ndctl(nvdimm_bus);
438}
439
440static int nd_bus_probe(struct device *dev)
441{
442 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
443 int rc;
444
445 rc = nvdimm_bus_create_ndctl(nvdimm_bus);
446 if (rc)
447 return rc;
448
449 mutex_lock(&nvdimm_bus_list_mutex);
450 list_add_tail(&nvdimm_bus->list, &nvdimm_bus_list);
451 mutex_unlock(&nvdimm_bus_list_mutex);
452
453 /* enable bus provider attributes to look up their local context */
454 dev_set_drvdata(dev, nvdimm_bus->nd_desc);
455
456 return 0;
457}
458
459static struct nd_device_driver nd_bus_driver = {
460 .probe = nd_bus_probe,
461 .remove = nd_bus_remove,
462 .drv = {
463 .name = "nd_bus",
464 .suppress_bind_attrs = true,
465 .bus = &nvdimm_bus_type,
466 .owner = THIS_MODULE,
467 .mod_name = KBUILD_MODNAME,
468 },
469};
470
471static int nvdimm_bus_match(struct device *dev, const struct device_driver *drv)
472{
473 const struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
474
475 if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
476 return true;
477
478 return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
479}
480
481static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
482
483void nd_synchronize(void)
484{
485 async_synchronize_full_domain(&nd_async_domain);
486}
487EXPORT_SYMBOL_GPL(nd_synchronize);
488
489static void nd_async_device_register(void *d, async_cookie_t cookie)
490{
491 struct device *dev = d;
492
493 if (device_add(dev) != 0) {
494 dev_err(dev, "%s: failed\n", __func__);
495 put_device(dev);
496 }
497 put_device(dev);
498 if (dev->parent)
499 put_device(dev->parent);
500}
501
502static void nd_async_device_unregister(void *d, async_cookie_t cookie)
503{
504 struct device *dev = d;
505
506 /* flush bus operations before delete */
507 nvdimm_bus_lock(dev);
508 nvdimm_bus_unlock(dev);
509
510 device_unregister(dev);
511 put_device(dev);
512}
513
514static void __nd_device_register(struct device *dev, bool sync)
515{
516 if (!dev)
517 return;
518
519 /*
520 * Ensure that region devices always have their NUMA node set as
521 * early as possible. This way we are able to make certain that
522 * any memory associated with the creation and the creation
523 * itself of the region is associated with the correct node.
524 */
525 if (is_nd_region(dev))
526 set_dev_node(dev, to_nd_region(dev)->numa_node);
527
528 dev->bus = &nvdimm_bus_type;
529 device_set_pm_not_required(dev);
530 if (dev->parent) {
531 get_device(dev->parent);
532 if (dev_to_node(dev) == NUMA_NO_NODE)
533 set_dev_node(dev, dev_to_node(dev->parent));
534 }
535 get_device(dev);
536
537 if (sync)
538 nd_async_device_register(dev, 0);
539 else
540 async_schedule_dev_domain(nd_async_device_register, dev,
541 &nd_async_domain);
542}
543
544void nd_device_register(struct device *dev)
545{
546 __nd_device_register(dev, false);
547}
548EXPORT_SYMBOL(nd_device_register);
549
550void nd_device_register_sync(struct device *dev)
551{
552 __nd_device_register(dev, true);
553}
554
555void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
556{
557 bool killed;
558
559 switch (mode) {
560 case ND_ASYNC:
561 /*
562 * In the async case this is being triggered with the
563 * device lock held and the unregistration work needs to
564 * be moved out of line iff this is thread has won the
565 * race to schedule the deletion.
566 */
567 if (!kill_device(dev))
568 return;
569
570 get_device(dev);
571 async_schedule_domain(nd_async_device_unregister, dev,
572 &nd_async_domain);
573 break;
574 case ND_SYNC:
575 /*
576 * In the sync case the device is being unregistered due
577 * to a state change of the parent. Claim the kill state
578 * to synchronize against other unregistration requests,
579 * or otherwise let the async path handle it if the
580 * unregistration was already queued.
581 */
582 device_lock(dev);
583 killed = kill_device(dev);
584 device_unlock(dev);
585
586 if (!killed)
587 return;
588
589 nd_synchronize();
590 device_unregister(dev);
591 break;
592 }
593}
594EXPORT_SYMBOL(nd_device_unregister);
595
596/**
597 * __nd_driver_register() - register a region or a namespace driver
598 * @nd_drv: driver to register
599 * @owner: automatically set by nd_driver_register() macro
600 * @mod_name: automatically set by nd_driver_register() macro
601 */
602int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
603 const char *mod_name)
604{
605 struct device_driver *drv = &nd_drv->drv;
606
607 if (!nd_drv->type) {
608 pr_debug("driver type bitmask not set (%ps)\n",
609 __builtin_return_address(0));
610 return -EINVAL;
611 }
612
613 if (!nd_drv->probe) {
614 pr_debug("%s ->probe() must be specified\n", mod_name);
615 return -EINVAL;
616 }
617
618 drv->bus = &nvdimm_bus_type;
619 drv->owner = owner;
620 drv->mod_name = mod_name;
621
622 return driver_register(drv);
623}
624EXPORT_SYMBOL(__nd_driver_register);
625
626void nvdimm_check_and_set_ro(struct gendisk *disk)
627{
628 struct device *dev = disk_to_dev(disk)->parent;
629 struct nd_region *nd_region = to_nd_region(dev->parent);
630 int disk_ro = get_disk_ro(disk);
631
632 /* catch the disk up with the region ro state */
633 if (disk_ro == nd_region->ro)
634 return;
635
636 dev_info(dev, "%s read-%s, marking %s read-%s\n",
637 dev_name(&nd_region->dev), nd_region->ro ? "only" : "write",
638 disk->disk_name, nd_region->ro ? "only" : "write");
639 set_disk_ro(disk, nd_region->ro);
640}
641EXPORT_SYMBOL(nvdimm_check_and_set_ro);
642
643static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
644 char *buf)
645{
646 return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
647 to_nd_device_type(dev));
648}
649static DEVICE_ATTR_RO(modalias);
650
651static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
652 char *buf)
653{
654 return sprintf(buf, "%s\n", dev->type->name);
655}
656static DEVICE_ATTR_RO(devtype);
657
658static struct attribute *nd_device_attributes[] = {
659 &dev_attr_modalias.attr,
660 &dev_attr_devtype.attr,
661 NULL,
662};
663
664/*
665 * nd_device_attribute_group - generic attributes for all devices on an nd bus
666 */
667const struct attribute_group nd_device_attribute_group = {
668 .attrs = nd_device_attributes,
669};
670
671static ssize_t numa_node_show(struct device *dev,
672 struct device_attribute *attr, char *buf)
673{
674 return sprintf(buf, "%d\n", dev_to_node(dev));
675}
676static DEVICE_ATTR_RO(numa_node);
677
678static int nvdimm_dev_to_target_node(struct device *dev)
679{
680 struct device *parent = dev->parent;
681 struct nd_region *nd_region = NULL;
682
683 if (is_nd_region(dev))
684 nd_region = to_nd_region(dev);
685 else if (parent && is_nd_region(parent))
686 nd_region = to_nd_region(parent);
687
688 if (!nd_region)
689 return NUMA_NO_NODE;
690 return nd_region->target_node;
691}
692
693static ssize_t target_node_show(struct device *dev,
694 struct device_attribute *attr, char *buf)
695{
696 return sprintf(buf, "%d\n", nvdimm_dev_to_target_node(dev));
697}
698static DEVICE_ATTR_RO(target_node);
699
700static struct attribute *nd_numa_attributes[] = {
701 &dev_attr_numa_node.attr,
702 &dev_attr_target_node.attr,
703 NULL,
704};
705
706static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
707 int n)
708{
709 struct device *dev = container_of(kobj, typeof(*dev), kobj);
710
711 if (!IS_ENABLED(CONFIG_NUMA))
712 return 0;
713
714 if (a == &dev_attr_target_node.attr &&
715 nvdimm_dev_to_target_node(dev) == NUMA_NO_NODE)
716 return 0;
717
718 return a->mode;
719}
720
721/*
722 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
723 */
724const struct attribute_group nd_numa_attribute_group = {
725 .attrs = nd_numa_attributes,
726 .is_visible = nd_numa_attr_visible,
727};
728
729static void ndctl_release(struct device *dev)
730{
731 kfree(dev);
732}
733
734static struct lock_class_key nvdimm_ndctl_key;
735
736int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
737{
738 dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
739 struct device *dev;
740 int rc;
741
742 dev = kzalloc(sizeof(*dev), GFP_KERNEL);
743 if (!dev)
744 return -ENOMEM;
745 device_initialize(dev);
746 lockdep_set_class(&dev->mutex, &nvdimm_ndctl_key);
747 device_set_pm_not_required(dev);
748 dev->class = &nd_class;
749 dev->parent = &nvdimm_bus->dev;
750 dev->devt = devt;
751 dev->release = ndctl_release;
752 rc = dev_set_name(dev, "ndctl%d", nvdimm_bus->id);
753 if (rc)
754 goto err;
755
756 rc = device_add(dev);
757 if (rc) {
758 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %d\n",
759 nvdimm_bus->id, rc);
760 goto err;
761 }
762 return 0;
763
764err:
765 put_device(dev);
766 return rc;
767}
768
769void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
770{
771 device_destroy(&nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
772}
773
774static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
775 [ND_CMD_IMPLEMENTED] = { },
776 [ND_CMD_SMART] = {
777 .out_num = 2,
778 .out_sizes = { 4, 128, },
779 },
780 [ND_CMD_SMART_THRESHOLD] = {
781 .out_num = 2,
782 .out_sizes = { 4, 8, },
783 },
784 [ND_CMD_DIMM_FLAGS] = {
785 .out_num = 2,
786 .out_sizes = { 4, 4 },
787 },
788 [ND_CMD_GET_CONFIG_SIZE] = {
789 .out_num = 3,
790 .out_sizes = { 4, 4, 4, },
791 },
792 [ND_CMD_GET_CONFIG_DATA] = {
793 .in_num = 2,
794 .in_sizes = { 4, 4, },
795 .out_num = 2,
796 .out_sizes = { 4, UINT_MAX, },
797 },
798 [ND_CMD_SET_CONFIG_DATA] = {
799 .in_num = 3,
800 .in_sizes = { 4, 4, UINT_MAX, },
801 .out_num = 1,
802 .out_sizes = { 4, },
803 },
804 [ND_CMD_VENDOR] = {
805 .in_num = 3,
806 .in_sizes = { 4, 4, UINT_MAX, },
807 .out_num = 3,
808 .out_sizes = { 4, 4, UINT_MAX, },
809 },
810 [ND_CMD_CALL] = {
811 .in_num = 2,
812 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
813 .out_num = 1,
814 .out_sizes = { UINT_MAX, },
815 },
816};
817
818const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
819{
820 if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
821 return &__nd_cmd_dimm_descs[cmd];
822 return NULL;
823}
824EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
825
826static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
827 [ND_CMD_IMPLEMENTED] = { },
828 [ND_CMD_ARS_CAP] = {
829 .in_num = 2,
830 .in_sizes = { 8, 8, },
831 .out_num = 4,
832 .out_sizes = { 4, 4, 4, 4, },
833 },
834 [ND_CMD_ARS_START] = {
835 .in_num = 5,
836 .in_sizes = { 8, 8, 2, 1, 5, },
837 .out_num = 2,
838 .out_sizes = { 4, 4, },
839 },
840 [ND_CMD_ARS_STATUS] = {
841 .out_num = 3,
842 .out_sizes = { 4, 4, UINT_MAX, },
843 },
844 [ND_CMD_CLEAR_ERROR] = {
845 .in_num = 2,
846 .in_sizes = { 8, 8, },
847 .out_num = 3,
848 .out_sizes = { 4, 4, 8, },
849 },
850 [ND_CMD_CALL] = {
851 .in_num = 2,
852 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
853 .out_num = 1,
854 .out_sizes = { UINT_MAX, },
855 },
856};
857
858const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
859{
860 if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
861 return &__nd_cmd_bus_descs[cmd];
862 return NULL;
863}
864EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
865
866u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
867 const struct nd_cmd_desc *desc, int idx, void *buf)
868{
869 if (idx >= desc->in_num)
870 return UINT_MAX;
871
872 if (desc->in_sizes[idx] < UINT_MAX)
873 return desc->in_sizes[idx];
874
875 if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
876 struct nd_cmd_set_config_hdr *hdr = buf;
877
878 return hdr->in_length;
879 } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
880 struct nd_cmd_vendor_hdr *hdr = buf;
881
882 return hdr->in_length;
883 } else if (cmd == ND_CMD_CALL) {
884 struct nd_cmd_pkg *pkg = buf;
885
886 return pkg->nd_size_in;
887 }
888
889 return UINT_MAX;
890}
891EXPORT_SYMBOL_GPL(nd_cmd_in_size);
892
893u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
894 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
895 const u32 *out_field, unsigned long remainder)
896{
897 if (idx >= desc->out_num)
898 return UINT_MAX;
899
900 if (desc->out_sizes[idx] < UINT_MAX)
901 return desc->out_sizes[idx];
902
903 if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
904 return in_field[1];
905 else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
906 return out_field[1];
907 else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
908 /*
909 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
910 * "Size of Output Buffer in bytes, including this
911 * field."
912 */
913 if (out_field[1] < 4)
914 return 0;
915 /*
916 * ACPI 6.1 is ambiguous if 'status' is included in the
917 * output size. If we encounter an output size that
918 * overshoots the remainder by 4 bytes, assume it was
919 * including 'status'.
920 */
921 if (out_field[1] - 4 == remainder)
922 return remainder;
923 return out_field[1] - 8;
924 } else if (cmd == ND_CMD_CALL) {
925 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
926
927 return pkg->nd_size_out;
928 }
929
930
931 return UINT_MAX;
932}
933EXPORT_SYMBOL_GPL(nd_cmd_out_size);
934
935void wait_nvdimm_bus_probe_idle(struct device *dev)
936{
937 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
938
939 do {
940 if (nvdimm_bus->probe_active == 0)
941 break;
942 nvdimm_bus_unlock(dev);
943 device_unlock(dev);
944 wait_event(nvdimm_bus->wait,
945 nvdimm_bus->probe_active == 0);
946 device_lock(dev);
947 nvdimm_bus_lock(dev);
948 } while (true);
949}
950
951static int nd_pmem_forget_poison_check(struct device *dev, void *data)
952{
953 struct nd_cmd_clear_error *clear_err =
954 (struct nd_cmd_clear_error *)data;
955 struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
956 struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
957 struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
958 struct nd_namespace_common *ndns = NULL;
959 struct nd_namespace_io *nsio;
960 resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
961
962 if (nd_dax || !dev->driver)
963 return 0;
964
965 start = clear_err->address;
966 end = clear_err->address + clear_err->cleared - 1;
967
968 if (nd_btt || nd_pfn || nd_dax) {
969 if (nd_btt)
970 ndns = nd_btt->ndns;
971 else if (nd_pfn)
972 ndns = nd_pfn->ndns;
973 else if (nd_dax)
974 ndns = nd_dax->nd_pfn.ndns;
975
976 if (!ndns)
977 return 0;
978 } else
979 ndns = to_ndns(dev);
980
981 nsio = to_nd_namespace_io(&ndns->dev);
982 pstart = nsio->res.start + offset;
983 pend = nsio->res.end - end_trunc;
984
985 if ((pstart >= start) && (pend <= end))
986 return -EBUSY;
987
988 return 0;
989
990}
991
992static int nd_ns_forget_poison_check(struct device *dev, void *data)
993{
994 return device_for_each_child(dev, data, nd_pmem_forget_poison_check);
995}
996
997/* set_config requires an idle interleave set */
998static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
999 struct nvdimm *nvdimm, unsigned int cmd, void *data)
1000{
1001 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
1002
1003 /* ask the bus provider if it would like to block this request */
1004 if (nd_desc->clear_to_send) {
1005 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
1006
1007 if (rc)
1008 return rc;
1009 }
1010
1011 /* require clear error to go through the pmem driver */
1012 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
1013 return device_for_each_child(&nvdimm_bus->dev, data,
1014 nd_ns_forget_poison_check);
1015
1016 if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
1017 return 0;
1018
1019 /* prevent label manipulation while the kernel owns label updates */
1020 wait_nvdimm_bus_probe_idle(&nvdimm_bus->dev);
1021 if (atomic_read(&nvdimm->busy))
1022 return -EBUSY;
1023 return 0;
1024}
1025
1026static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
1027 int read_only, unsigned int ioctl_cmd, unsigned long arg)
1028{
1029 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
1030 const struct nd_cmd_desc *desc = NULL;
1031 unsigned int cmd = _IOC_NR(ioctl_cmd);
1032 struct device *dev = &nvdimm_bus->dev;
1033 void __user *p = (void __user *) arg;
1034 char *out_env = NULL, *in_env = NULL;
1035 const char *cmd_name, *dimm_name;
1036 u32 in_len = 0, out_len = 0;
1037 unsigned int func = cmd;
1038 unsigned long cmd_mask;
1039 struct nd_cmd_pkg pkg;
1040 int rc, i, cmd_rc;
1041 void *buf = NULL;
1042 u64 buf_len = 0;
1043
1044 if (nvdimm) {
1045 desc = nd_cmd_dimm_desc(cmd);
1046 cmd_name = nvdimm_cmd_name(cmd);
1047 cmd_mask = nvdimm->cmd_mask;
1048 dimm_name = dev_name(&nvdimm->dev);
1049 } else {
1050 desc = nd_cmd_bus_desc(cmd);
1051 cmd_name = nvdimm_bus_cmd_name(cmd);
1052 cmd_mask = nd_desc->cmd_mask;
1053 dimm_name = "bus";
1054 }
1055
1056 /* Validate command family support against bus declared support */
1057 if (cmd == ND_CMD_CALL) {
1058 unsigned long *mask;
1059
1060 if (copy_from_user(&pkg, p, sizeof(pkg)))
1061 return -EFAULT;
1062
1063 if (nvdimm) {
1064 if (pkg.nd_family > NVDIMM_FAMILY_MAX)
1065 return -EINVAL;
1066 mask = &nd_desc->dimm_family_mask;
1067 } else {
1068 if (pkg.nd_family > NVDIMM_BUS_FAMILY_MAX)
1069 return -EINVAL;
1070 mask = &nd_desc->bus_family_mask;
1071 }
1072
1073 if (!test_bit(pkg.nd_family, mask))
1074 return -EINVAL;
1075 }
1076
1077 if (!desc ||
1078 (desc->out_num + desc->in_num == 0) ||
1079 cmd > ND_CMD_CALL ||
1080 !test_bit(cmd, &cmd_mask))
1081 return -ENOTTY;
1082
1083 /* fail write commands (when read-only) */
1084 if (read_only)
1085 switch (cmd) {
1086 case ND_CMD_VENDOR:
1087 case ND_CMD_SET_CONFIG_DATA:
1088 case ND_CMD_ARS_START:
1089 case ND_CMD_CLEAR_ERROR:
1090 case ND_CMD_CALL:
1091 dev_dbg(dev, "'%s' command while read-only.\n",
1092 nvdimm ? nvdimm_cmd_name(cmd)
1093 : nvdimm_bus_cmd_name(cmd));
1094 return -EPERM;
1095 default:
1096 break;
1097 }
1098
1099 /* process an input envelope */
1100 in_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1101 if (!in_env)
1102 return -ENOMEM;
1103 for (i = 0; i < desc->in_num; i++) {
1104 u32 in_size, copy;
1105
1106 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
1107 if (in_size == UINT_MAX) {
1108 dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
1109 __func__, dimm_name, cmd_name, i);
1110 rc = -ENXIO;
1111 goto out;
1112 }
1113 if (in_len < ND_CMD_MAX_ENVELOPE)
1114 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
1115 else
1116 copy = 0;
1117 if (copy && copy_from_user(&in_env[in_len], p + in_len, copy)) {
1118 rc = -EFAULT;
1119 goto out;
1120 }
1121 in_len += in_size;
1122 }
1123
1124 if (cmd == ND_CMD_CALL) {
1125 func = pkg.nd_command;
1126 dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
1127 dimm_name, pkg.nd_command,
1128 in_len, out_len, buf_len);
1129 }
1130
1131 /* process an output envelope */
1132 out_env = kzalloc(ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1133 if (!out_env) {
1134 rc = -ENOMEM;
1135 goto out;
1136 }
1137
1138 for (i = 0; i < desc->out_num; i++) {
1139 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
1140 (u32 *) in_env, (u32 *) out_env, 0);
1141 u32 copy;
1142
1143 if (out_size == UINT_MAX) {
1144 dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
1145 dimm_name, cmd_name, i);
1146 rc = -EFAULT;
1147 goto out;
1148 }
1149 if (out_len < ND_CMD_MAX_ENVELOPE)
1150 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
1151 else
1152 copy = 0;
1153 if (copy && copy_from_user(&out_env[out_len],
1154 p + in_len + out_len, copy)) {
1155 rc = -EFAULT;
1156 goto out;
1157 }
1158 out_len += out_size;
1159 }
1160
1161 buf_len = (u64) out_len + (u64) in_len;
1162 if (buf_len > ND_IOCTL_MAX_BUFLEN) {
1163 dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
1164 cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
1165 rc = -EINVAL;
1166 goto out;
1167 }
1168
1169 buf = vmalloc(buf_len);
1170 if (!buf) {
1171 rc = -ENOMEM;
1172 goto out;
1173 }
1174
1175 if (copy_from_user(buf, p, buf_len)) {
1176 rc = -EFAULT;
1177 goto out;
1178 }
1179
1180 device_lock(dev);
1181 nvdimm_bus_lock(dev);
1182 rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, func, buf);
1183 if (rc)
1184 goto out_unlock;
1185
1186 rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
1187 if (rc < 0)
1188 goto out_unlock;
1189
1190 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
1191 struct nd_cmd_clear_error *clear_err = buf;
1192
1193 nvdimm_account_cleared_poison(nvdimm_bus, clear_err->address,
1194 clear_err->cleared);
1195 }
1196
1197 if (copy_to_user(p, buf, buf_len))
1198 rc = -EFAULT;
1199
1200out_unlock:
1201 nvdimm_bus_unlock(dev);
1202 device_unlock(dev);
1203out:
1204 kfree(in_env);
1205 kfree(out_env);
1206 vfree(buf);
1207 return rc;
1208}
1209
1210enum nd_ioctl_mode {
1211 BUS_IOCTL,
1212 DIMM_IOCTL,
1213};
1214
1215static int match_dimm(struct device *dev, void *data)
1216{
1217 long id = (long) data;
1218
1219 if (is_nvdimm(dev)) {
1220 struct nvdimm *nvdimm = to_nvdimm(dev);
1221
1222 return nvdimm->id == id;
1223 }
1224
1225 return 0;
1226}
1227
1228static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
1229 enum nd_ioctl_mode mode)
1230
1231{
1232 struct nvdimm_bus *nvdimm_bus, *found = NULL;
1233 long id = (long) file->private_data;
1234 struct nvdimm *nvdimm = NULL;
1235 int rc, ro;
1236
1237 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
1238 mutex_lock(&nvdimm_bus_list_mutex);
1239 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
1240 if (mode == DIMM_IOCTL) {
1241 struct device *dev;
1242
1243 dev = device_find_child(&nvdimm_bus->dev,
1244 file->private_data, match_dimm);
1245 if (!dev)
1246 continue;
1247 nvdimm = to_nvdimm(dev);
1248 found = nvdimm_bus;
1249 } else if (nvdimm_bus->id == id) {
1250 found = nvdimm_bus;
1251 }
1252
1253 if (found) {
1254 atomic_inc(&nvdimm_bus->ioctl_active);
1255 break;
1256 }
1257 }
1258 mutex_unlock(&nvdimm_bus_list_mutex);
1259
1260 if (!found)
1261 return -ENXIO;
1262
1263 nvdimm_bus = found;
1264 rc = __nd_ioctl(nvdimm_bus, nvdimm, ro, cmd, arg);
1265
1266 if (nvdimm)
1267 put_device(&nvdimm->dev);
1268 if (atomic_dec_and_test(&nvdimm_bus->ioctl_active))
1269 wake_up(&nvdimm_bus->wait);
1270
1271 return rc;
1272}
1273
1274static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1275{
1276 return nd_ioctl(file, cmd, arg, BUS_IOCTL);
1277}
1278
1279static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1280{
1281 return nd_ioctl(file, cmd, arg, DIMM_IOCTL);
1282}
1283
1284static int nd_open(struct inode *inode, struct file *file)
1285{
1286 long minor = iminor(inode);
1287
1288 file->private_data = (void *) minor;
1289 return 0;
1290}
1291
1292static const struct file_operations nvdimm_bus_fops = {
1293 .owner = THIS_MODULE,
1294 .open = nd_open,
1295 .unlocked_ioctl = bus_ioctl,
1296 .compat_ioctl = compat_ptr_ioctl,
1297 .llseek = noop_llseek,
1298};
1299
1300static const struct file_operations nvdimm_fops = {
1301 .owner = THIS_MODULE,
1302 .open = nd_open,
1303 .unlocked_ioctl = dimm_ioctl,
1304 .compat_ioctl = compat_ptr_ioctl,
1305 .llseek = noop_llseek,
1306};
1307
1308int __init nvdimm_bus_init(void)
1309{
1310 int rc;
1311
1312 rc = bus_register(&nvdimm_bus_type);
1313 if (rc)
1314 return rc;
1315
1316 rc = register_chrdev(0, "ndctl", &nvdimm_bus_fops);
1317 if (rc < 0)
1318 goto err_bus_chrdev;
1319 nvdimm_bus_major = rc;
1320
1321 rc = register_chrdev(0, "dimmctl", &nvdimm_fops);
1322 if (rc < 0)
1323 goto err_dimm_chrdev;
1324 nvdimm_major = rc;
1325
1326 rc = class_register(&nd_class);
1327 if (rc)
1328 goto err_class;
1329
1330 rc = driver_register(&nd_bus_driver.drv);
1331 if (rc)
1332 goto err_nd_bus;
1333
1334 return 0;
1335
1336 err_nd_bus:
1337 class_unregister(&nd_class);
1338 err_class:
1339 unregister_chrdev(nvdimm_major, "dimmctl");
1340 err_dimm_chrdev:
1341 unregister_chrdev(nvdimm_bus_major, "ndctl");
1342 err_bus_chrdev:
1343 bus_unregister(&nvdimm_bus_type);
1344
1345 return rc;
1346}
1347
1348void nvdimm_bus_exit(void)
1349{
1350 driver_unregister(&nd_bus_driver.drv);
1351 class_unregister(&nd_class);
1352 unregister_chrdev(nvdimm_bus_major, "ndctl");
1353 unregister_chrdev(nvdimm_major, "dimmctl");
1354 bus_unregister(&nvdimm_bus_type);
1355 ida_destroy(&nd_ida);
1356}