Loading...
1//#define DEBUG
2#include <linux/spinlock.h>
3#include <linux/slab.h>
4#include <linux/blkdev.h>
5#include <linux/hdreg.h>
6#include <linux/module.h>
7#include <linux/mutex.h>
8#include <linux/virtio.h>
9#include <linux/virtio_blk.h>
10#include <linux/scatterlist.h>
11#include <linux/string_helpers.h>
12#include <scsi/scsi_cmnd.h>
13#include <linux/idr.h>
14
15#define PART_BITS 4
16
17static int major;
18static DEFINE_IDA(vd_index_ida);
19
20struct workqueue_struct *virtblk_wq;
21
22struct virtio_blk
23{
24 struct virtio_device *vdev;
25 struct virtqueue *vq;
26
27 /* The disk structure for the kernel. */
28 struct gendisk *disk;
29
30 mempool_t *pool;
31
32 /* Process context for config space updates */
33 struct work_struct config_work;
34
35 /* Lock for config space updates */
36 struct mutex config_lock;
37
38 /* enable config space updates */
39 bool config_enable;
40
41 /* What host tells us, plus 2 for header & tailer. */
42 unsigned int sg_elems;
43
44 /* Ida index - used to track minor number allocations. */
45 int index;
46
47 /* Scatterlist: can be too big for stack. */
48 struct scatterlist sg[/*sg_elems*/];
49};
50
51struct virtblk_req
52{
53 struct request *req;
54 struct virtio_blk_outhdr out_hdr;
55 struct virtio_scsi_inhdr in_hdr;
56 u8 status;
57};
58
59static void blk_done(struct virtqueue *vq)
60{
61 struct virtio_blk *vblk = vq->vdev->priv;
62 struct virtblk_req *vbr;
63 unsigned int len;
64 unsigned long flags;
65
66 spin_lock_irqsave(vblk->disk->queue->queue_lock, flags);
67 while ((vbr = virtqueue_get_buf(vblk->vq, &len)) != NULL) {
68 int error;
69
70 switch (vbr->status) {
71 case VIRTIO_BLK_S_OK:
72 error = 0;
73 break;
74 case VIRTIO_BLK_S_UNSUPP:
75 error = -ENOTTY;
76 break;
77 default:
78 error = -EIO;
79 break;
80 }
81
82 switch (vbr->req->cmd_type) {
83 case REQ_TYPE_BLOCK_PC:
84 vbr->req->resid_len = vbr->in_hdr.residual;
85 vbr->req->sense_len = vbr->in_hdr.sense_len;
86 vbr->req->errors = vbr->in_hdr.errors;
87 break;
88 case REQ_TYPE_SPECIAL:
89 vbr->req->errors = (error != 0);
90 break;
91 default:
92 break;
93 }
94
95 __blk_end_request_all(vbr->req, error);
96 mempool_free(vbr, vblk->pool);
97 }
98 /* In case queue is stopped waiting for more buffers. */
99 blk_start_queue(vblk->disk->queue);
100 spin_unlock_irqrestore(vblk->disk->queue->queue_lock, flags);
101}
102
103static bool do_req(struct request_queue *q, struct virtio_blk *vblk,
104 struct request *req)
105{
106 unsigned long num, out = 0, in = 0;
107 struct virtblk_req *vbr;
108
109 vbr = mempool_alloc(vblk->pool, GFP_ATOMIC);
110 if (!vbr)
111 /* When another request finishes we'll try again. */
112 return false;
113
114 vbr->req = req;
115
116 if (req->cmd_flags & REQ_FLUSH) {
117 vbr->out_hdr.type = VIRTIO_BLK_T_FLUSH;
118 vbr->out_hdr.sector = 0;
119 vbr->out_hdr.ioprio = req_get_ioprio(vbr->req);
120 } else {
121 switch (req->cmd_type) {
122 case REQ_TYPE_FS:
123 vbr->out_hdr.type = 0;
124 vbr->out_hdr.sector = blk_rq_pos(vbr->req);
125 vbr->out_hdr.ioprio = req_get_ioprio(vbr->req);
126 break;
127 case REQ_TYPE_BLOCK_PC:
128 vbr->out_hdr.type = VIRTIO_BLK_T_SCSI_CMD;
129 vbr->out_hdr.sector = 0;
130 vbr->out_hdr.ioprio = req_get_ioprio(vbr->req);
131 break;
132 case REQ_TYPE_SPECIAL:
133 vbr->out_hdr.type = VIRTIO_BLK_T_GET_ID;
134 vbr->out_hdr.sector = 0;
135 vbr->out_hdr.ioprio = req_get_ioprio(vbr->req);
136 break;
137 default:
138 /* We don't put anything else in the queue. */
139 BUG();
140 }
141 }
142
143 sg_set_buf(&vblk->sg[out++], &vbr->out_hdr, sizeof(vbr->out_hdr));
144
145 /*
146 * If this is a packet command we need a couple of additional headers.
147 * Behind the normal outhdr we put a segment with the scsi command
148 * block, and before the normal inhdr we put the sense data and the
149 * inhdr with additional status information before the normal inhdr.
150 */
151 if (vbr->req->cmd_type == REQ_TYPE_BLOCK_PC)
152 sg_set_buf(&vblk->sg[out++], vbr->req->cmd, vbr->req->cmd_len);
153
154 num = blk_rq_map_sg(q, vbr->req, vblk->sg + out);
155
156 if (vbr->req->cmd_type == REQ_TYPE_BLOCK_PC) {
157 sg_set_buf(&vblk->sg[num + out + in++], vbr->req->sense, SCSI_SENSE_BUFFERSIZE);
158 sg_set_buf(&vblk->sg[num + out + in++], &vbr->in_hdr,
159 sizeof(vbr->in_hdr));
160 }
161
162 sg_set_buf(&vblk->sg[num + out + in++], &vbr->status,
163 sizeof(vbr->status));
164
165 if (num) {
166 if (rq_data_dir(vbr->req) == WRITE) {
167 vbr->out_hdr.type |= VIRTIO_BLK_T_OUT;
168 out += num;
169 } else {
170 vbr->out_hdr.type |= VIRTIO_BLK_T_IN;
171 in += num;
172 }
173 }
174
175 if (virtqueue_add_buf(vblk->vq, vblk->sg, out, in, vbr, GFP_ATOMIC)<0) {
176 mempool_free(vbr, vblk->pool);
177 return false;
178 }
179
180 return true;
181}
182
183static void do_virtblk_request(struct request_queue *q)
184{
185 struct virtio_blk *vblk = q->queuedata;
186 struct request *req;
187 unsigned int issued = 0;
188
189 while ((req = blk_peek_request(q)) != NULL) {
190 BUG_ON(req->nr_phys_segments + 2 > vblk->sg_elems);
191
192 /* If this request fails, stop queue and wait for something to
193 finish to restart it. */
194 if (!do_req(q, vblk, req)) {
195 blk_stop_queue(q);
196 break;
197 }
198 blk_start_request(req);
199 issued++;
200 }
201
202 if (issued)
203 virtqueue_kick(vblk->vq);
204}
205
206/* return id (s/n) string for *disk to *id_str
207 */
208static int virtblk_get_id(struct gendisk *disk, char *id_str)
209{
210 struct virtio_blk *vblk = disk->private_data;
211 struct request *req;
212 struct bio *bio;
213 int err;
214
215 bio = bio_map_kern(vblk->disk->queue, id_str, VIRTIO_BLK_ID_BYTES,
216 GFP_KERNEL);
217 if (IS_ERR(bio))
218 return PTR_ERR(bio);
219
220 req = blk_make_request(vblk->disk->queue, bio, GFP_KERNEL);
221 if (IS_ERR(req)) {
222 bio_put(bio);
223 return PTR_ERR(req);
224 }
225
226 req->cmd_type = REQ_TYPE_SPECIAL;
227 err = blk_execute_rq(vblk->disk->queue, vblk->disk, req, false);
228 blk_put_request(req);
229
230 return err;
231}
232
233static int virtblk_ioctl(struct block_device *bdev, fmode_t mode,
234 unsigned int cmd, unsigned long data)
235{
236 struct gendisk *disk = bdev->bd_disk;
237 struct virtio_blk *vblk = disk->private_data;
238
239 /*
240 * Only allow the generic SCSI ioctls if the host can support it.
241 */
242 if (!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_SCSI))
243 return -ENOTTY;
244
245 return scsi_cmd_blk_ioctl(bdev, mode, cmd,
246 (void __user *)data);
247}
248
249/* We provide getgeo only to please some old bootloader/partitioning tools */
250static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
251{
252 struct virtio_blk *vblk = bd->bd_disk->private_data;
253 struct virtio_blk_geometry vgeo;
254 int err;
255
256 /* see if the host passed in geometry config */
257 err = virtio_config_val(vblk->vdev, VIRTIO_BLK_F_GEOMETRY,
258 offsetof(struct virtio_blk_config, geometry),
259 &vgeo);
260
261 if (!err) {
262 geo->heads = vgeo.heads;
263 geo->sectors = vgeo.sectors;
264 geo->cylinders = vgeo.cylinders;
265 } else {
266 /* some standard values, similar to sd */
267 geo->heads = 1 << 6;
268 geo->sectors = 1 << 5;
269 geo->cylinders = get_capacity(bd->bd_disk) >> 11;
270 }
271 return 0;
272}
273
274static const struct block_device_operations virtblk_fops = {
275 .ioctl = virtblk_ioctl,
276 .owner = THIS_MODULE,
277 .getgeo = virtblk_getgeo,
278};
279
280static int index_to_minor(int index)
281{
282 return index << PART_BITS;
283}
284
285static int minor_to_index(int minor)
286{
287 return minor >> PART_BITS;
288}
289
290static ssize_t virtblk_serial_show(struct device *dev,
291 struct device_attribute *attr, char *buf)
292{
293 struct gendisk *disk = dev_to_disk(dev);
294 int err;
295
296 /* sysfs gives us a PAGE_SIZE buffer */
297 BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES);
298
299 buf[VIRTIO_BLK_ID_BYTES] = '\0';
300 err = virtblk_get_id(disk, buf);
301 if (!err)
302 return strlen(buf);
303
304 if (err == -EIO) /* Unsupported? Make it empty. */
305 return 0;
306
307 return err;
308}
309DEVICE_ATTR(serial, S_IRUGO, virtblk_serial_show, NULL);
310
311static void virtblk_config_changed_work(struct work_struct *work)
312{
313 struct virtio_blk *vblk =
314 container_of(work, struct virtio_blk, config_work);
315 struct virtio_device *vdev = vblk->vdev;
316 struct request_queue *q = vblk->disk->queue;
317 char cap_str_2[10], cap_str_10[10];
318 u64 capacity, size;
319
320 mutex_lock(&vblk->config_lock);
321 if (!vblk->config_enable)
322 goto done;
323
324 /* Host must always specify the capacity. */
325 vdev->config->get(vdev, offsetof(struct virtio_blk_config, capacity),
326 &capacity, sizeof(capacity));
327
328 /* If capacity is too big, truncate with warning. */
329 if ((sector_t)capacity != capacity) {
330 dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n",
331 (unsigned long long)capacity);
332 capacity = (sector_t)-1;
333 }
334
335 size = capacity * queue_logical_block_size(q);
336 string_get_size(size, STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
337 string_get_size(size, STRING_UNITS_10, cap_str_10, sizeof(cap_str_10));
338
339 dev_notice(&vdev->dev,
340 "new size: %llu %d-byte logical blocks (%s/%s)\n",
341 (unsigned long long)capacity,
342 queue_logical_block_size(q),
343 cap_str_10, cap_str_2);
344
345 set_capacity(vblk->disk, capacity);
346 revalidate_disk(vblk->disk);
347done:
348 mutex_unlock(&vblk->config_lock);
349}
350
351static void virtblk_config_changed(struct virtio_device *vdev)
352{
353 struct virtio_blk *vblk = vdev->priv;
354
355 queue_work(virtblk_wq, &vblk->config_work);
356}
357
358static int init_vq(struct virtio_blk *vblk)
359{
360 int err = 0;
361
362 /* We expect one virtqueue, for output. */
363 vblk->vq = virtio_find_single_vq(vblk->vdev, blk_done, "requests");
364 if (IS_ERR(vblk->vq))
365 err = PTR_ERR(vblk->vq);
366
367 return err;
368}
369
370/*
371 * Legacy naming scheme used for virtio devices. We are stuck with it for
372 * virtio blk but don't ever use it for any new driver.
373 */
374static int virtblk_name_format(char *prefix, int index, char *buf, int buflen)
375{
376 const int base = 'z' - 'a' + 1;
377 char *begin = buf + strlen(prefix);
378 char *end = buf + buflen;
379 char *p;
380 int unit;
381
382 p = end - 1;
383 *p = '\0';
384 unit = base;
385 do {
386 if (p == begin)
387 return -EINVAL;
388 *--p = 'a' + (index % unit);
389 index = (index / unit) - 1;
390 } while (index >= 0);
391
392 memmove(begin, p, end - p);
393 memcpy(buf, prefix, strlen(prefix));
394
395 return 0;
396}
397
398static int __devinit virtblk_probe(struct virtio_device *vdev)
399{
400 struct virtio_blk *vblk;
401 struct request_queue *q;
402 int err, index;
403 u64 cap;
404 u32 v, blk_size, sg_elems, opt_io_size;
405 u16 min_io_size;
406 u8 physical_block_exp, alignment_offset;
407
408 err = ida_simple_get(&vd_index_ida, 0, minor_to_index(1 << MINORBITS),
409 GFP_KERNEL);
410 if (err < 0)
411 goto out;
412 index = err;
413
414 /* We need to know how many segments before we allocate. */
415 err = virtio_config_val(vdev, VIRTIO_BLK_F_SEG_MAX,
416 offsetof(struct virtio_blk_config, seg_max),
417 &sg_elems);
418
419 /* We need at least one SG element, whatever they say. */
420 if (err || !sg_elems)
421 sg_elems = 1;
422
423 /* We need an extra sg elements at head and tail. */
424 sg_elems += 2;
425 vdev->priv = vblk = kmalloc(sizeof(*vblk) +
426 sizeof(vblk->sg[0]) * sg_elems, GFP_KERNEL);
427 if (!vblk) {
428 err = -ENOMEM;
429 goto out_free_index;
430 }
431
432 vblk->vdev = vdev;
433 vblk->sg_elems = sg_elems;
434 sg_init_table(vblk->sg, vblk->sg_elems);
435 mutex_init(&vblk->config_lock);
436 INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
437 vblk->config_enable = true;
438
439 err = init_vq(vblk);
440 if (err)
441 goto out_free_vblk;
442
443 vblk->pool = mempool_create_kmalloc_pool(1,sizeof(struct virtblk_req));
444 if (!vblk->pool) {
445 err = -ENOMEM;
446 goto out_free_vq;
447 }
448
449 /* FIXME: How many partitions? How long is a piece of string? */
450 vblk->disk = alloc_disk(1 << PART_BITS);
451 if (!vblk->disk) {
452 err = -ENOMEM;
453 goto out_mempool;
454 }
455
456 q = vblk->disk->queue = blk_init_queue(do_virtblk_request, NULL);
457 if (!q) {
458 err = -ENOMEM;
459 goto out_put_disk;
460 }
461
462 q->queuedata = vblk;
463
464 virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);
465
466 vblk->disk->major = major;
467 vblk->disk->first_minor = index_to_minor(index);
468 vblk->disk->private_data = vblk;
469 vblk->disk->fops = &virtblk_fops;
470 vblk->disk->driverfs_dev = &vdev->dev;
471 vblk->index = index;
472
473 /* configure queue flush support */
474 if (virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH))
475 blk_queue_flush(q, REQ_FLUSH);
476
477 /* If disk is read-only in the host, the guest should obey */
478 if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO))
479 set_disk_ro(vblk->disk, 1);
480
481 /* Host must always specify the capacity. */
482 vdev->config->get(vdev, offsetof(struct virtio_blk_config, capacity),
483 &cap, sizeof(cap));
484
485 /* If capacity is too big, truncate with warning. */
486 if ((sector_t)cap != cap) {
487 dev_warn(&vdev->dev, "Capacity %llu too large: truncating\n",
488 (unsigned long long)cap);
489 cap = (sector_t)-1;
490 }
491 set_capacity(vblk->disk, cap);
492
493 /* We can handle whatever the host told us to handle. */
494 blk_queue_max_segments(q, vblk->sg_elems-2);
495
496 /* No need to bounce any requests */
497 blk_queue_bounce_limit(q, BLK_BOUNCE_ANY);
498
499 /* No real sector limit. */
500 blk_queue_max_hw_sectors(q, -1U);
501
502 /* Host can optionally specify maximum segment size and number of
503 * segments. */
504 err = virtio_config_val(vdev, VIRTIO_BLK_F_SIZE_MAX,
505 offsetof(struct virtio_blk_config, size_max),
506 &v);
507 if (!err)
508 blk_queue_max_segment_size(q, v);
509 else
510 blk_queue_max_segment_size(q, -1U);
511
512 /* Host can optionally specify the block size of the device */
513 err = virtio_config_val(vdev, VIRTIO_BLK_F_BLK_SIZE,
514 offsetof(struct virtio_blk_config, blk_size),
515 &blk_size);
516 if (!err)
517 blk_queue_logical_block_size(q, blk_size);
518 else
519 blk_size = queue_logical_block_size(q);
520
521 /* Use topology information if available */
522 err = virtio_config_val(vdev, VIRTIO_BLK_F_TOPOLOGY,
523 offsetof(struct virtio_blk_config, physical_block_exp),
524 &physical_block_exp);
525 if (!err && physical_block_exp)
526 blk_queue_physical_block_size(q,
527 blk_size * (1 << physical_block_exp));
528
529 err = virtio_config_val(vdev, VIRTIO_BLK_F_TOPOLOGY,
530 offsetof(struct virtio_blk_config, alignment_offset),
531 &alignment_offset);
532 if (!err && alignment_offset)
533 blk_queue_alignment_offset(q, blk_size * alignment_offset);
534
535 err = virtio_config_val(vdev, VIRTIO_BLK_F_TOPOLOGY,
536 offsetof(struct virtio_blk_config, min_io_size),
537 &min_io_size);
538 if (!err && min_io_size)
539 blk_queue_io_min(q, blk_size * min_io_size);
540
541 err = virtio_config_val(vdev, VIRTIO_BLK_F_TOPOLOGY,
542 offsetof(struct virtio_blk_config, opt_io_size),
543 &opt_io_size);
544 if (!err && opt_io_size)
545 blk_queue_io_opt(q, blk_size * opt_io_size);
546
547
548 add_disk(vblk->disk);
549 err = device_create_file(disk_to_dev(vblk->disk), &dev_attr_serial);
550 if (err)
551 goto out_del_disk;
552
553 return 0;
554
555out_del_disk:
556 del_gendisk(vblk->disk);
557 blk_cleanup_queue(vblk->disk->queue);
558out_put_disk:
559 put_disk(vblk->disk);
560out_mempool:
561 mempool_destroy(vblk->pool);
562out_free_vq:
563 vdev->config->del_vqs(vdev);
564out_free_vblk:
565 kfree(vblk);
566out_free_index:
567 ida_simple_remove(&vd_index_ida, index);
568out:
569 return err;
570}
571
572static void __devexit virtblk_remove(struct virtio_device *vdev)
573{
574 struct virtio_blk *vblk = vdev->priv;
575 int index = vblk->index;
576
577 /* Prevent config work handler from accessing the device. */
578 mutex_lock(&vblk->config_lock);
579 vblk->config_enable = false;
580 mutex_unlock(&vblk->config_lock);
581
582 del_gendisk(vblk->disk);
583 blk_cleanup_queue(vblk->disk->queue);
584
585 /* Stop all the virtqueues. */
586 vdev->config->reset(vdev);
587
588 flush_work(&vblk->config_work);
589
590 put_disk(vblk->disk);
591 mempool_destroy(vblk->pool);
592 vdev->config->del_vqs(vdev);
593 kfree(vblk);
594 ida_simple_remove(&vd_index_ida, index);
595}
596
597#ifdef CONFIG_PM
598static int virtblk_freeze(struct virtio_device *vdev)
599{
600 struct virtio_blk *vblk = vdev->priv;
601
602 /* Ensure we don't receive any more interrupts */
603 vdev->config->reset(vdev);
604
605 /* Prevent config work handler from accessing the device. */
606 mutex_lock(&vblk->config_lock);
607 vblk->config_enable = false;
608 mutex_unlock(&vblk->config_lock);
609
610 flush_work(&vblk->config_work);
611
612 spin_lock_irq(vblk->disk->queue->queue_lock);
613 blk_stop_queue(vblk->disk->queue);
614 spin_unlock_irq(vblk->disk->queue->queue_lock);
615 blk_sync_queue(vblk->disk->queue);
616
617 vdev->config->del_vqs(vdev);
618 return 0;
619}
620
621static int virtblk_restore(struct virtio_device *vdev)
622{
623 struct virtio_blk *vblk = vdev->priv;
624 int ret;
625
626 vblk->config_enable = true;
627 ret = init_vq(vdev->priv);
628 if (!ret) {
629 spin_lock_irq(vblk->disk->queue->queue_lock);
630 blk_start_queue(vblk->disk->queue);
631 spin_unlock_irq(vblk->disk->queue->queue_lock);
632 }
633 return ret;
634}
635#endif
636
637static const struct virtio_device_id id_table[] = {
638 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
639 { 0 },
640};
641
642static unsigned int features[] = {
643 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
644 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE, VIRTIO_BLK_F_SCSI,
645 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY
646};
647
648/*
649 * virtio_blk causes spurious section mismatch warning by
650 * simultaneously referring to a __devinit and a __devexit function.
651 * Use __refdata to avoid this warning.
652 */
653static struct virtio_driver __refdata virtio_blk = {
654 .feature_table = features,
655 .feature_table_size = ARRAY_SIZE(features),
656 .driver.name = KBUILD_MODNAME,
657 .driver.owner = THIS_MODULE,
658 .id_table = id_table,
659 .probe = virtblk_probe,
660 .remove = __devexit_p(virtblk_remove),
661 .config_changed = virtblk_config_changed,
662#ifdef CONFIG_PM
663 .freeze = virtblk_freeze,
664 .restore = virtblk_restore,
665#endif
666};
667
668static int __init init(void)
669{
670 int error;
671
672 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0);
673 if (!virtblk_wq)
674 return -ENOMEM;
675
676 major = register_blkdev(0, "virtblk");
677 if (major < 0) {
678 error = major;
679 goto out_destroy_workqueue;
680 }
681
682 error = register_virtio_driver(&virtio_blk);
683 if (error)
684 goto out_unregister_blkdev;
685 return 0;
686
687out_unregister_blkdev:
688 unregister_blkdev(major, "virtblk");
689out_destroy_workqueue:
690 destroy_workqueue(virtblk_wq);
691 return error;
692}
693
694static void __exit fini(void)
695{
696 unregister_blkdev(major, "virtblk");
697 unregister_virtio_driver(&virtio_blk);
698 destroy_workqueue(virtblk_wq);
699}
700module_init(init);
701module_exit(fini);
702
703MODULE_DEVICE_TABLE(virtio, id_table);
704MODULE_DESCRIPTION("Virtio block driver");
705MODULE_LICENSE("GPL");
1// SPDX-License-Identifier: GPL-2.0-only
2//#define DEBUG
3#include <linux/spinlock.h>
4#include <linux/slab.h>
5#include <linux/blkdev.h>
6#include <linux/hdreg.h>
7#include <linux/module.h>
8#include <linux/mutex.h>
9#include <linux/interrupt.h>
10#include <linux/virtio.h>
11#include <linux/virtio_blk.h>
12#include <linux/scatterlist.h>
13#include <linux/string_helpers.h>
14#include <linux/idr.h>
15#include <linux/blk-mq.h>
16#include <linux/blk-mq-virtio.h>
17#include <linux/numa.h>
18#include <uapi/linux/virtio_ring.h>
19
20#define PART_BITS 4
21#define VQ_NAME_LEN 16
22#define MAX_DISCARD_SEGMENTS 256u
23
24/* The maximum number of sg elements that fit into a virtqueue */
25#define VIRTIO_BLK_MAX_SG_ELEMS 32768
26
27#ifdef CONFIG_ARCH_NO_SG_CHAIN
28#define VIRTIO_BLK_INLINE_SG_CNT 0
29#else
30#define VIRTIO_BLK_INLINE_SG_CNT 2
31#endif
32
33static unsigned int num_request_queues;
34module_param(num_request_queues, uint, 0644);
35MODULE_PARM_DESC(num_request_queues,
36 "Limit the number of request queues to use for blk device. "
37 "0 for no limit. "
38 "Values > nr_cpu_ids truncated to nr_cpu_ids.");
39
40static unsigned int poll_queues;
41module_param(poll_queues, uint, 0644);
42MODULE_PARM_DESC(poll_queues, "The number of dedicated virtqueues for polling I/O");
43
44static int major;
45static DEFINE_IDA(vd_index_ida);
46
47static struct workqueue_struct *virtblk_wq;
48
49struct virtio_blk_vq {
50 struct virtqueue *vq;
51 spinlock_t lock;
52 char name[VQ_NAME_LEN];
53} ____cacheline_aligned_in_smp;
54
55struct virtio_blk {
56 /*
57 * This mutex must be held by anything that may run after
58 * virtblk_remove() sets vblk->vdev to NULL.
59 *
60 * blk-mq, virtqueue processing, and sysfs attribute code paths are
61 * shut down before vblk->vdev is set to NULL and therefore do not need
62 * to hold this mutex.
63 */
64 struct mutex vdev_mutex;
65 struct virtio_device *vdev;
66
67 /* The disk structure for the kernel. */
68 struct gendisk *disk;
69
70 /* Block layer tags. */
71 struct blk_mq_tag_set tag_set;
72
73 /* Process context for config space updates */
74 struct work_struct config_work;
75
76 /* Ida index - used to track minor number allocations. */
77 int index;
78
79 /* num of vqs */
80 int num_vqs;
81 int io_queues[HCTX_MAX_TYPES];
82 struct virtio_blk_vq *vqs;
83};
84
85struct virtblk_req {
86 struct virtio_blk_outhdr out_hdr;
87 u8 status;
88 struct sg_table sg_table;
89 struct scatterlist sg[];
90};
91
92static inline blk_status_t virtblk_result(struct virtblk_req *vbr)
93{
94 switch (vbr->status) {
95 case VIRTIO_BLK_S_OK:
96 return BLK_STS_OK;
97 case VIRTIO_BLK_S_UNSUPP:
98 return BLK_STS_NOTSUPP;
99 default:
100 return BLK_STS_IOERR;
101 }
102}
103
104static inline struct virtio_blk_vq *get_virtio_blk_vq(struct blk_mq_hw_ctx *hctx)
105{
106 struct virtio_blk *vblk = hctx->queue->queuedata;
107 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];
108
109 return vq;
110}
111
112static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr)
113{
114 struct scatterlist hdr, status, *sgs[3];
115 unsigned int num_out = 0, num_in = 0;
116
117 sg_init_one(&hdr, &vbr->out_hdr, sizeof(vbr->out_hdr));
118 sgs[num_out++] = &hdr;
119
120 if (vbr->sg_table.nents) {
121 if (vbr->out_hdr.type & cpu_to_virtio32(vq->vdev, VIRTIO_BLK_T_OUT))
122 sgs[num_out++] = vbr->sg_table.sgl;
123 else
124 sgs[num_out + num_in++] = vbr->sg_table.sgl;
125 }
126
127 sg_init_one(&status, &vbr->status, sizeof(vbr->status));
128 sgs[num_out + num_in++] = &status;
129
130 return virtqueue_add_sgs(vq, sgs, num_out, num_in, vbr, GFP_ATOMIC);
131}
132
133static int virtblk_setup_discard_write_zeroes_erase(struct request *req, bool unmap)
134{
135 unsigned short segments = blk_rq_nr_discard_segments(req);
136 unsigned short n = 0;
137 struct virtio_blk_discard_write_zeroes *range;
138 struct bio *bio;
139 u32 flags = 0;
140
141 if (unmap)
142 flags |= VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP;
143
144 range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC);
145 if (!range)
146 return -ENOMEM;
147
148 /*
149 * Single max discard segment means multi-range discard isn't
150 * supported, and block layer only runs contiguity merge like
151 * normal RW request. So we can't reply on bio for retrieving
152 * each range info.
153 */
154 if (queue_max_discard_segments(req->q) == 1) {
155 range[0].flags = cpu_to_le32(flags);
156 range[0].num_sectors = cpu_to_le32(blk_rq_sectors(req));
157 range[0].sector = cpu_to_le64(blk_rq_pos(req));
158 n = 1;
159 } else {
160 __rq_for_each_bio(bio, req) {
161 u64 sector = bio->bi_iter.bi_sector;
162 u32 num_sectors = bio->bi_iter.bi_size >> SECTOR_SHIFT;
163
164 range[n].flags = cpu_to_le32(flags);
165 range[n].num_sectors = cpu_to_le32(num_sectors);
166 range[n].sector = cpu_to_le64(sector);
167 n++;
168 }
169 }
170
171 WARN_ON_ONCE(n != segments);
172
173 req->special_vec.bv_page = virt_to_page(range);
174 req->special_vec.bv_offset = offset_in_page(range);
175 req->special_vec.bv_len = sizeof(*range) * segments;
176 req->rq_flags |= RQF_SPECIAL_PAYLOAD;
177
178 return 0;
179}
180
181static void virtblk_unmap_data(struct request *req, struct virtblk_req *vbr)
182{
183 if (blk_rq_nr_phys_segments(req))
184 sg_free_table_chained(&vbr->sg_table,
185 VIRTIO_BLK_INLINE_SG_CNT);
186}
187
188static int virtblk_map_data(struct blk_mq_hw_ctx *hctx, struct request *req,
189 struct virtblk_req *vbr)
190{
191 int err;
192
193 if (!blk_rq_nr_phys_segments(req))
194 return 0;
195
196 vbr->sg_table.sgl = vbr->sg;
197 err = sg_alloc_table_chained(&vbr->sg_table,
198 blk_rq_nr_phys_segments(req),
199 vbr->sg_table.sgl,
200 VIRTIO_BLK_INLINE_SG_CNT);
201 if (unlikely(err))
202 return -ENOMEM;
203
204 return blk_rq_map_sg(hctx->queue, req, vbr->sg_table.sgl);
205}
206
207static void virtblk_cleanup_cmd(struct request *req)
208{
209 if (req->rq_flags & RQF_SPECIAL_PAYLOAD)
210 kfree(bvec_virt(&req->special_vec));
211}
212
213static blk_status_t virtblk_setup_cmd(struct virtio_device *vdev,
214 struct request *req,
215 struct virtblk_req *vbr)
216{
217 bool unmap = false;
218 u32 type;
219
220 vbr->out_hdr.sector = 0;
221
222 switch (req_op(req)) {
223 case REQ_OP_READ:
224 type = VIRTIO_BLK_T_IN;
225 vbr->out_hdr.sector = cpu_to_virtio64(vdev,
226 blk_rq_pos(req));
227 break;
228 case REQ_OP_WRITE:
229 type = VIRTIO_BLK_T_OUT;
230 vbr->out_hdr.sector = cpu_to_virtio64(vdev,
231 blk_rq_pos(req));
232 break;
233 case REQ_OP_FLUSH:
234 type = VIRTIO_BLK_T_FLUSH;
235 break;
236 case REQ_OP_DISCARD:
237 type = VIRTIO_BLK_T_DISCARD;
238 break;
239 case REQ_OP_WRITE_ZEROES:
240 type = VIRTIO_BLK_T_WRITE_ZEROES;
241 unmap = !(req->cmd_flags & REQ_NOUNMAP);
242 break;
243 case REQ_OP_SECURE_ERASE:
244 type = VIRTIO_BLK_T_SECURE_ERASE;
245 break;
246 case REQ_OP_DRV_IN:
247 type = VIRTIO_BLK_T_GET_ID;
248 break;
249 default:
250 WARN_ON_ONCE(1);
251 return BLK_STS_IOERR;
252 }
253
254 vbr->out_hdr.type = cpu_to_virtio32(vdev, type);
255 vbr->out_hdr.ioprio = cpu_to_virtio32(vdev, req_get_ioprio(req));
256
257 if (type == VIRTIO_BLK_T_DISCARD || type == VIRTIO_BLK_T_WRITE_ZEROES ||
258 type == VIRTIO_BLK_T_SECURE_ERASE) {
259 if (virtblk_setup_discard_write_zeroes_erase(req, unmap))
260 return BLK_STS_RESOURCE;
261 }
262
263 return 0;
264}
265
266static inline void virtblk_request_done(struct request *req)
267{
268 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
269
270 virtblk_unmap_data(req, vbr);
271 virtblk_cleanup_cmd(req);
272 blk_mq_end_request(req, virtblk_result(vbr));
273}
274
275static void virtblk_done(struct virtqueue *vq)
276{
277 struct virtio_blk *vblk = vq->vdev->priv;
278 bool req_done = false;
279 int qid = vq->index;
280 struct virtblk_req *vbr;
281 unsigned long flags;
282 unsigned int len;
283
284 spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
285 do {
286 virtqueue_disable_cb(vq);
287 while ((vbr = virtqueue_get_buf(vblk->vqs[qid].vq, &len)) != NULL) {
288 struct request *req = blk_mq_rq_from_pdu(vbr);
289
290 if (likely(!blk_should_fake_timeout(req->q)))
291 blk_mq_complete_request(req);
292 req_done = true;
293 }
294 if (unlikely(virtqueue_is_broken(vq)))
295 break;
296 } while (!virtqueue_enable_cb(vq));
297
298 /* In case queue is stopped waiting for more buffers. */
299 if (req_done)
300 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
301 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
302}
303
304static void virtio_commit_rqs(struct blk_mq_hw_ctx *hctx)
305{
306 struct virtio_blk *vblk = hctx->queue->queuedata;
307 struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num];
308 bool kick;
309
310 spin_lock_irq(&vq->lock);
311 kick = virtqueue_kick_prepare(vq->vq);
312 spin_unlock_irq(&vq->lock);
313
314 if (kick)
315 virtqueue_notify(vq->vq);
316}
317
318static blk_status_t virtblk_fail_to_queue(struct request *req, int rc)
319{
320 virtblk_cleanup_cmd(req);
321 switch (rc) {
322 case -ENOSPC:
323 return BLK_STS_DEV_RESOURCE;
324 case -ENOMEM:
325 return BLK_STS_RESOURCE;
326 default:
327 return BLK_STS_IOERR;
328 }
329}
330
331static blk_status_t virtblk_prep_rq(struct blk_mq_hw_ctx *hctx,
332 struct virtio_blk *vblk,
333 struct request *req,
334 struct virtblk_req *vbr)
335{
336 blk_status_t status;
337 int num;
338
339 status = virtblk_setup_cmd(vblk->vdev, req, vbr);
340 if (unlikely(status))
341 return status;
342
343 num = virtblk_map_data(hctx, req, vbr);
344 if (unlikely(num < 0))
345 return virtblk_fail_to_queue(req, -ENOMEM);
346 vbr->sg_table.nents = num;
347
348 blk_mq_start_request(req);
349
350 return BLK_STS_OK;
351}
352
353static blk_status_t virtio_queue_rq(struct blk_mq_hw_ctx *hctx,
354 const struct blk_mq_queue_data *bd)
355{
356 struct virtio_blk *vblk = hctx->queue->queuedata;
357 struct request *req = bd->rq;
358 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
359 unsigned long flags;
360 int qid = hctx->queue_num;
361 bool notify = false;
362 blk_status_t status;
363 int err;
364
365 status = virtblk_prep_rq(hctx, vblk, req, vbr);
366 if (unlikely(status))
367 return status;
368
369 spin_lock_irqsave(&vblk->vqs[qid].lock, flags);
370 err = virtblk_add_req(vblk->vqs[qid].vq, vbr);
371 if (err) {
372 virtqueue_kick(vblk->vqs[qid].vq);
373 /* Don't stop the queue if -ENOMEM: we may have failed to
374 * bounce the buffer due to global resource outage.
375 */
376 if (err == -ENOSPC)
377 blk_mq_stop_hw_queue(hctx);
378 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
379 virtblk_unmap_data(req, vbr);
380 return virtblk_fail_to_queue(req, err);
381 }
382
383 if (bd->last && virtqueue_kick_prepare(vblk->vqs[qid].vq))
384 notify = true;
385 spin_unlock_irqrestore(&vblk->vqs[qid].lock, flags);
386
387 if (notify)
388 virtqueue_notify(vblk->vqs[qid].vq);
389 return BLK_STS_OK;
390}
391
392static bool virtblk_prep_rq_batch(struct request *req)
393{
394 struct virtio_blk *vblk = req->mq_hctx->queue->queuedata;
395 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
396
397 req->mq_hctx->tags->rqs[req->tag] = req;
398
399 return virtblk_prep_rq(req->mq_hctx, vblk, req, vbr) == BLK_STS_OK;
400}
401
402static bool virtblk_add_req_batch(struct virtio_blk_vq *vq,
403 struct request **rqlist)
404{
405 unsigned long flags;
406 int err;
407 bool kick;
408
409 spin_lock_irqsave(&vq->lock, flags);
410
411 while (!rq_list_empty(*rqlist)) {
412 struct request *req = rq_list_pop(rqlist);
413 struct virtblk_req *vbr = blk_mq_rq_to_pdu(req);
414
415 err = virtblk_add_req(vq->vq, vbr);
416 if (err) {
417 virtblk_unmap_data(req, vbr);
418 virtblk_cleanup_cmd(req);
419 blk_mq_requeue_request(req, true);
420 }
421 }
422
423 kick = virtqueue_kick_prepare(vq->vq);
424 spin_unlock_irqrestore(&vq->lock, flags);
425
426 return kick;
427}
428
429static void virtio_queue_rqs(struct request **rqlist)
430{
431 struct request *req, *next, *prev = NULL;
432 struct request *requeue_list = NULL;
433
434 rq_list_for_each_safe(rqlist, req, next) {
435 struct virtio_blk_vq *vq = get_virtio_blk_vq(req->mq_hctx);
436 bool kick;
437
438 if (!virtblk_prep_rq_batch(req)) {
439 rq_list_move(rqlist, &requeue_list, req, prev);
440 req = prev;
441 if (!req)
442 continue;
443 }
444
445 if (!next || req->mq_hctx != next->mq_hctx) {
446 req->rq_next = NULL;
447 kick = virtblk_add_req_batch(vq, rqlist);
448 if (kick)
449 virtqueue_notify(vq->vq);
450
451 *rqlist = next;
452 prev = NULL;
453 } else
454 prev = req;
455 }
456
457 *rqlist = requeue_list;
458}
459
460/* return id (s/n) string for *disk to *id_str
461 */
462static int virtblk_get_id(struct gendisk *disk, char *id_str)
463{
464 struct virtio_blk *vblk = disk->private_data;
465 struct request_queue *q = vblk->disk->queue;
466 struct request *req;
467 int err;
468
469 req = blk_mq_alloc_request(q, REQ_OP_DRV_IN, 0);
470 if (IS_ERR(req))
471 return PTR_ERR(req);
472
473 err = blk_rq_map_kern(q, req, id_str, VIRTIO_BLK_ID_BYTES, GFP_KERNEL);
474 if (err)
475 goto out;
476
477 blk_execute_rq(req, false);
478 err = blk_status_to_errno(virtblk_result(blk_mq_rq_to_pdu(req)));
479out:
480 blk_mq_free_request(req);
481 return err;
482}
483
484/* We provide getgeo only to please some old bootloader/partitioning tools */
485static int virtblk_getgeo(struct block_device *bd, struct hd_geometry *geo)
486{
487 struct virtio_blk *vblk = bd->bd_disk->private_data;
488 int ret = 0;
489
490 mutex_lock(&vblk->vdev_mutex);
491
492 if (!vblk->vdev) {
493 ret = -ENXIO;
494 goto out;
495 }
496
497 /* see if the host passed in geometry config */
498 if (virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_GEOMETRY)) {
499 virtio_cread(vblk->vdev, struct virtio_blk_config,
500 geometry.cylinders, &geo->cylinders);
501 virtio_cread(vblk->vdev, struct virtio_blk_config,
502 geometry.heads, &geo->heads);
503 virtio_cread(vblk->vdev, struct virtio_blk_config,
504 geometry.sectors, &geo->sectors);
505 } else {
506 /* some standard values, similar to sd */
507 geo->heads = 1 << 6;
508 geo->sectors = 1 << 5;
509 geo->cylinders = get_capacity(bd->bd_disk) >> 11;
510 }
511out:
512 mutex_unlock(&vblk->vdev_mutex);
513 return ret;
514}
515
516static void virtblk_free_disk(struct gendisk *disk)
517{
518 struct virtio_blk *vblk = disk->private_data;
519
520 ida_free(&vd_index_ida, vblk->index);
521 mutex_destroy(&vblk->vdev_mutex);
522 kfree(vblk);
523}
524
525static const struct block_device_operations virtblk_fops = {
526 .owner = THIS_MODULE,
527 .getgeo = virtblk_getgeo,
528 .free_disk = virtblk_free_disk,
529};
530
531static int index_to_minor(int index)
532{
533 return index << PART_BITS;
534}
535
536static int minor_to_index(int minor)
537{
538 return minor >> PART_BITS;
539}
540
541static ssize_t serial_show(struct device *dev,
542 struct device_attribute *attr, char *buf)
543{
544 struct gendisk *disk = dev_to_disk(dev);
545 int err;
546
547 /* sysfs gives us a PAGE_SIZE buffer */
548 BUILD_BUG_ON(PAGE_SIZE < VIRTIO_BLK_ID_BYTES);
549
550 buf[VIRTIO_BLK_ID_BYTES] = '\0';
551 err = virtblk_get_id(disk, buf);
552 if (!err)
553 return strlen(buf);
554
555 if (err == -EIO) /* Unsupported? Make it empty. */
556 return 0;
557
558 return err;
559}
560
561static DEVICE_ATTR_RO(serial);
562
563/* The queue's logical block size must be set before calling this */
564static void virtblk_update_capacity(struct virtio_blk *vblk, bool resize)
565{
566 struct virtio_device *vdev = vblk->vdev;
567 struct request_queue *q = vblk->disk->queue;
568 char cap_str_2[10], cap_str_10[10];
569 unsigned long long nblocks;
570 u64 capacity;
571
572 /* Host must always specify the capacity. */
573 virtio_cread(vdev, struct virtio_blk_config, capacity, &capacity);
574
575 nblocks = DIV_ROUND_UP_ULL(capacity, queue_logical_block_size(q) >> 9);
576
577 string_get_size(nblocks, queue_logical_block_size(q),
578 STRING_UNITS_2, cap_str_2, sizeof(cap_str_2));
579 string_get_size(nblocks, queue_logical_block_size(q),
580 STRING_UNITS_10, cap_str_10, sizeof(cap_str_10));
581
582 dev_notice(&vdev->dev,
583 "[%s] %s%llu %d-byte logical blocks (%s/%s)\n",
584 vblk->disk->disk_name,
585 resize ? "new size: " : "",
586 nblocks,
587 queue_logical_block_size(q),
588 cap_str_10,
589 cap_str_2);
590
591 set_capacity_and_notify(vblk->disk, capacity);
592}
593
594static void virtblk_config_changed_work(struct work_struct *work)
595{
596 struct virtio_blk *vblk =
597 container_of(work, struct virtio_blk, config_work);
598
599 virtblk_update_capacity(vblk, true);
600}
601
602static void virtblk_config_changed(struct virtio_device *vdev)
603{
604 struct virtio_blk *vblk = vdev->priv;
605
606 queue_work(virtblk_wq, &vblk->config_work);
607}
608
609static int init_vq(struct virtio_blk *vblk)
610{
611 int err;
612 int i;
613 vq_callback_t **callbacks;
614 const char **names;
615 struct virtqueue **vqs;
616 unsigned short num_vqs;
617 unsigned int num_poll_vqs;
618 struct virtio_device *vdev = vblk->vdev;
619 struct irq_affinity desc = { 0, };
620
621 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_MQ,
622 struct virtio_blk_config, num_queues,
623 &num_vqs);
624 if (err)
625 num_vqs = 1;
626
627 if (!err && !num_vqs) {
628 dev_err(&vdev->dev, "MQ advertised but zero queues reported\n");
629 return -EINVAL;
630 }
631
632 num_vqs = min_t(unsigned int,
633 min_not_zero(num_request_queues, nr_cpu_ids),
634 num_vqs);
635
636 num_poll_vqs = min_t(unsigned int, poll_queues, num_vqs - 1);
637
638 vblk->io_queues[HCTX_TYPE_DEFAULT] = num_vqs - num_poll_vqs;
639 vblk->io_queues[HCTX_TYPE_READ] = 0;
640 vblk->io_queues[HCTX_TYPE_POLL] = num_poll_vqs;
641
642 dev_info(&vdev->dev, "%d/%d/%d default/read/poll queues\n",
643 vblk->io_queues[HCTX_TYPE_DEFAULT],
644 vblk->io_queues[HCTX_TYPE_READ],
645 vblk->io_queues[HCTX_TYPE_POLL]);
646
647 vblk->vqs = kmalloc_array(num_vqs, sizeof(*vblk->vqs), GFP_KERNEL);
648 if (!vblk->vqs)
649 return -ENOMEM;
650
651 names = kmalloc_array(num_vqs, sizeof(*names), GFP_KERNEL);
652 callbacks = kmalloc_array(num_vqs, sizeof(*callbacks), GFP_KERNEL);
653 vqs = kmalloc_array(num_vqs, sizeof(*vqs), GFP_KERNEL);
654 if (!names || !callbacks || !vqs) {
655 err = -ENOMEM;
656 goto out;
657 }
658
659 for (i = 0; i < num_vqs - num_poll_vqs; i++) {
660 callbacks[i] = virtblk_done;
661 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req.%d", i);
662 names[i] = vblk->vqs[i].name;
663 }
664
665 for (; i < num_vqs; i++) {
666 callbacks[i] = NULL;
667 snprintf(vblk->vqs[i].name, VQ_NAME_LEN, "req_poll.%d", i);
668 names[i] = vblk->vqs[i].name;
669 }
670
671 /* Discover virtqueues and write information to configuration. */
672 err = virtio_find_vqs(vdev, num_vqs, vqs, callbacks, names, &desc);
673 if (err)
674 goto out;
675
676 for (i = 0; i < num_vqs; i++) {
677 spin_lock_init(&vblk->vqs[i].lock);
678 vblk->vqs[i].vq = vqs[i];
679 }
680 vblk->num_vqs = num_vqs;
681
682out:
683 kfree(vqs);
684 kfree(callbacks);
685 kfree(names);
686 if (err)
687 kfree(vblk->vqs);
688 return err;
689}
690
691/*
692 * Legacy naming scheme used for virtio devices. We are stuck with it for
693 * virtio blk but don't ever use it for any new driver.
694 */
695static int virtblk_name_format(char *prefix, int index, char *buf, int buflen)
696{
697 const int base = 'z' - 'a' + 1;
698 char *begin = buf + strlen(prefix);
699 char *end = buf + buflen;
700 char *p;
701 int unit;
702
703 p = end - 1;
704 *p = '\0';
705 unit = base;
706 do {
707 if (p == begin)
708 return -EINVAL;
709 *--p = 'a' + (index % unit);
710 index = (index / unit) - 1;
711 } while (index >= 0);
712
713 memmove(begin, p, end - p);
714 memcpy(buf, prefix, strlen(prefix));
715
716 return 0;
717}
718
719static int virtblk_get_cache_mode(struct virtio_device *vdev)
720{
721 u8 writeback;
722 int err;
723
724 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE,
725 struct virtio_blk_config, wce,
726 &writeback);
727
728 /*
729 * If WCE is not configurable and flush is not available,
730 * assume no writeback cache is in use.
731 */
732 if (err)
733 writeback = virtio_has_feature(vdev, VIRTIO_BLK_F_FLUSH);
734
735 return writeback;
736}
737
738static void virtblk_update_cache_mode(struct virtio_device *vdev)
739{
740 u8 writeback = virtblk_get_cache_mode(vdev);
741 struct virtio_blk *vblk = vdev->priv;
742
743 blk_queue_write_cache(vblk->disk->queue, writeback, false);
744}
745
746static const char *const virtblk_cache_types[] = {
747 "write through", "write back"
748};
749
750static ssize_t
751cache_type_store(struct device *dev, struct device_attribute *attr,
752 const char *buf, size_t count)
753{
754 struct gendisk *disk = dev_to_disk(dev);
755 struct virtio_blk *vblk = disk->private_data;
756 struct virtio_device *vdev = vblk->vdev;
757 int i;
758
759 BUG_ON(!virtio_has_feature(vblk->vdev, VIRTIO_BLK_F_CONFIG_WCE));
760 i = sysfs_match_string(virtblk_cache_types, buf);
761 if (i < 0)
762 return i;
763
764 virtio_cwrite8(vdev, offsetof(struct virtio_blk_config, wce), i);
765 virtblk_update_cache_mode(vdev);
766 return count;
767}
768
769static ssize_t
770cache_type_show(struct device *dev, struct device_attribute *attr, char *buf)
771{
772 struct gendisk *disk = dev_to_disk(dev);
773 struct virtio_blk *vblk = disk->private_data;
774 u8 writeback = virtblk_get_cache_mode(vblk->vdev);
775
776 BUG_ON(writeback >= ARRAY_SIZE(virtblk_cache_types));
777 return sysfs_emit(buf, "%s\n", virtblk_cache_types[writeback]);
778}
779
780static DEVICE_ATTR_RW(cache_type);
781
782static struct attribute *virtblk_attrs[] = {
783 &dev_attr_serial.attr,
784 &dev_attr_cache_type.attr,
785 NULL,
786};
787
788static umode_t virtblk_attrs_are_visible(struct kobject *kobj,
789 struct attribute *a, int n)
790{
791 struct device *dev = kobj_to_dev(kobj);
792 struct gendisk *disk = dev_to_disk(dev);
793 struct virtio_blk *vblk = disk->private_data;
794 struct virtio_device *vdev = vblk->vdev;
795
796 if (a == &dev_attr_cache_type.attr &&
797 !virtio_has_feature(vdev, VIRTIO_BLK_F_CONFIG_WCE))
798 return S_IRUGO;
799
800 return a->mode;
801}
802
803static const struct attribute_group virtblk_attr_group = {
804 .attrs = virtblk_attrs,
805 .is_visible = virtblk_attrs_are_visible,
806};
807
808static const struct attribute_group *virtblk_attr_groups[] = {
809 &virtblk_attr_group,
810 NULL,
811};
812
813static void virtblk_map_queues(struct blk_mq_tag_set *set)
814{
815 struct virtio_blk *vblk = set->driver_data;
816 int i, qoff;
817
818 for (i = 0, qoff = 0; i < set->nr_maps; i++) {
819 struct blk_mq_queue_map *map = &set->map[i];
820
821 map->nr_queues = vblk->io_queues[i];
822 map->queue_offset = qoff;
823 qoff += map->nr_queues;
824
825 if (map->nr_queues == 0)
826 continue;
827
828 /*
829 * Regular queues have interrupts and hence CPU affinity is
830 * defined by the core virtio code, but polling queues have
831 * no interrupts so we let the block layer assign CPU affinity.
832 */
833 if (i == HCTX_TYPE_POLL)
834 blk_mq_map_queues(&set->map[i]);
835 else
836 blk_mq_virtio_map_queues(&set->map[i], vblk->vdev, 0);
837 }
838}
839
840static void virtblk_complete_batch(struct io_comp_batch *iob)
841{
842 struct request *req;
843
844 rq_list_for_each(&iob->req_list, req) {
845 virtblk_unmap_data(req, blk_mq_rq_to_pdu(req));
846 virtblk_cleanup_cmd(req);
847 }
848 blk_mq_end_request_batch(iob);
849}
850
851static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
852{
853 struct virtio_blk *vblk = hctx->queue->queuedata;
854 struct virtio_blk_vq *vq = get_virtio_blk_vq(hctx);
855 struct virtblk_req *vbr;
856 unsigned long flags;
857 unsigned int len;
858 int found = 0;
859
860 spin_lock_irqsave(&vq->lock, flags);
861
862 while ((vbr = virtqueue_get_buf(vq->vq, &len)) != NULL) {
863 struct request *req = blk_mq_rq_from_pdu(vbr);
864
865 found++;
866 if (!blk_mq_add_to_batch(req, iob, vbr->status,
867 virtblk_complete_batch))
868 blk_mq_complete_request(req);
869 }
870
871 if (found)
872 blk_mq_start_stopped_hw_queues(vblk->disk->queue, true);
873
874 spin_unlock_irqrestore(&vq->lock, flags);
875
876 return found;
877}
878
879static const struct blk_mq_ops virtio_mq_ops = {
880 .queue_rq = virtio_queue_rq,
881 .queue_rqs = virtio_queue_rqs,
882 .commit_rqs = virtio_commit_rqs,
883 .complete = virtblk_request_done,
884 .map_queues = virtblk_map_queues,
885 .poll = virtblk_poll,
886};
887
888static unsigned int virtblk_queue_depth;
889module_param_named(queue_depth, virtblk_queue_depth, uint, 0444);
890
891static int virtblk_probe(struct virtio_device *vdev)
892{
893 struct virtio_blk *vblk;
894 struct request_queue *q;
895 int err, index;
896
897 u32 v, blk_size, max_size, sg_elems, opt_io_size;
898 u32 max_discard_segs = 0;
899 u32 discard_granularity = 0;
900 u16 min_io_size;
901 u8 physical_block_exp, alignment_offset;
902 unsigned int queue_depth;
903
904 if (!vdev->config->get) {
905 dev_err(&vdev->dev, "%s failure: config access disabled\n",
906 __func__);
907 return -EINVAL;
908 }
909
910 err = ida_alloc_range(&vd_index_ida, 0,
911 minor_to_index(1 << MINORBITS) - 1, GFP_KERNEL);
912 if (err < 0)
913 goto out;
914 index = err;
915
916 /* We need to know how many segments before we allocate. */
917 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SEG_MAX,
918 struct virtio_blk_config, seg_max,
919 &sg_elems);
920
921 /* We need at least one SG element, whatever they say. */
922 if (err || !sg_elems)
923 sg_elems = 1;
924
925 /* Prevent integer overflows and honor max vq size */
926 sg_elems = min_t(u32, sg_elems, VIRTIO_BLK_MAX_SG_ELEMS - 2);
927
928 vdev->priv = vblk = kmalloc(sizeof(*vblk), GFP_KERNEL);
929 if (!vblk) {
930 err = -ENOMEM;
931 goto out_free_index;
932 }
933
934 mutex_init(&vblk->vdev_mutex);
935
936 vblk->vdev = vdev;
937
938 INIT_WORK(&vblk->config_work, virtblk_config_changed_work);
939
940 err = init_vq(vblk);
941 if (err)
942 goto out_free_vblk;
943
944 /* Default queue sizing is to fill the ring. */
945 if (!virtblk_queue_depth) {
946 queue_depth = vblk->vqs[0].vq->num_free;
947 /* ... but without indirect descs, we use 2 descs per req */
948 if (!virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC))
949 queue_depth /= 2;
950 } else {
951 queue_depth = virtblk_queue_depth;
952 }
953
954 memset(&vblk->tag_set, 0, sizeof(vblk->tag_set));
955 vblk->tag_set.ops = &virtio_mq_ops;
956 vblk->tag_set.queue_depth = queue_depth;
957 vblk->tag_set.numa_node = NUMA_NO_NODE;
958 vblk->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
959 vblk->tag_set.cmd_size =
960 sizeof(struct virtblk_req) +
961 sizeof(struct scatterlist) * VIRTIO_BLK_INLINE_SG_CNT;
962 vblk->tag_set.driver_data = vblk;
963 vblk->tag_set.nr_hw_queues = vblk->num_vqs;
964 vblk->tag_set.nr_maps = 1;
965 if (vblk->io_queues[HCTX_TYPE_POLL])
966 vblk->tag_set.nr_maps = 3;
967
968 err = blk_mq_alloc_tag_set(&vblk->tag_set);
969 if (err)
970 goto out_free_vq;
971
972 vblk->disk = blk_mq_alloc_disk(&vblk->tag_set, vblk);
973 if (IS_ERR(vblk->disk)) {
974 err = PTR_ERR(vblk->disk);
975 goto out_free_tags;
976 }
977 q = vblk->disk->queue;
978
979 virtblk_name_format("vd", index, vblk->disk->disk_name, DISK_NAME_LEN);
980
981 vblk->disk->major = major;
982 vblk->disk->first_minor = index_to_minor(index);
983 vblk->disk->minors = 1 << PART_BITS;
984 vblk->disk->private_data = vblk;
985 vblk->disk->fops = &virtblk_fops;
986 vblk->index = index;
987
988 /* configure queue flush support */
989 virtblk_update_cache_mode(vdev);
990
991 /* If disk is read-only in the host, the guest should obey */
992 if (virtio_has_feature(vdev, VIRTIO_BLK_F_RO))
993 set_disk_ro(vblk->disk, 1);
994
995 /* We can handle whatever the host told us to handle. */
996 blk_queue_max_segments(q, sg_elems);
997
998 /* No real sector limit. */
999 blk_queue_max_hw_sectors(q, UINT_MAX);
1000
1001 max_size = virtio_max_dma_size(vdev);
1002
1003 /* Host can optionally specify maximum segment size and number of
1004 * segments. */
1005 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_SIZE_MAX,
1006 struct virtio_blk_config, size_max, &v);
1007 if (!err)
1008 max_size = min(max_size, v);
1009
1010 blk_queue_max_segment_size(q, max_size);
1011
1012 /* Host can optionally specify the block size of the device */
1013 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_BLK_SIZE,
1014 struct virtio_blk_config, blk_size,
1015 &blk_size);
1016 if (!err) {
1017 err = blk_validate_block_size(blk_size);
1018 if (err) {
1019 dev_err(&vdev->dev,
1020 "virtio_blk: invalid block size: 0x%x\n",
1021 blk_size);
1022 goto out_cleanup_disk;
1023 }
1024
1025 blk_queue_logical_block_size(q, blk_size);
1026 } else
1027 blk_size = queue_logical_block_size(q);
1028
1029 /* Use topology information if available */
1030 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
1031 struct virtio_blk_config, physical_block_exp,
1032 &physical_block_exp);
1033 if (!err && physical_block_exp)
1034 blk_queue_physical_block_size(q,
1035 blk_size * (1 << physical_block_exp));
1036
1037 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
1038 struct virtio_blk_config, alignment_offset,
1039 &alignment_offset);
1040 if (!err && alignment_offset)
1041 blk_queue_alignment_offset(q, blk_size * alignment_offset);
1042
1043 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
1044 struct virtio_blk_config, min_io_size,
1045 &min_io_size);
1046 if (!err && min_io_size)
1047 blk_queue_io_min(q, blk_size * min_io_size);
1048
1049 err = virtio_cread_feature(vdev, VIRTIO_BLK_F_TOPOLOGY,
1050 struct virtio_blk_config, opt_io_size,
1051 &opt_io_size);
1052 if (!err && opt_io_size)
1053 blk_queue_io_opt(q, blk_size * opt_io_size);
1054
1055 if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD)) {
1056 virtio_cread(vdev, struct virtio_blk_config,
1057 discard_sector_alignment, &discard_granularity);
1058
1059 virtio_cread(vdev, struct virtio_blk_config,
1060 max_discard_sectors, &v);
1061 blk_queue_max_discard_sectors(q, v ? v : UINT_MAX);
1062
1063 virtio_cread(vdev, struct virtio_blk_config, max_discard_seg,
1064 &max_discard_segs);
1065 }
1066
1067 if (virtio_has_feature(vdev, VIRTIO_BLK_F_WRITE_ZEROES)) {
1068 virtio_cread(vdev, struct virtio_blk_config,
1069 max_write_zeroes_sectors, &v);
1070 blk_queue_max_write_zeroes_sectors(q, v ? v : UINT_MAX);
1071 }
1072
1073 /* The discard and secure erase limits are combined since the Linux
1074 * block layer uses the same limit for both commands.
1075 *
1076 * If both VIRTIO_BLK_F_SECURE_ERASE and VIRTIO_BLK_F_DISCARD features
1077 * are negotiated, we will use the minimum between the limits.
1078 *
1079 * discard sector alignment is set to the minimum between discard_sector_alignment
1080 * and secure_erase_sector_alignment.
1081 *
1082 * max discard sectors is set to the minimum between max_discard_seg and
1083 * max_secure_erase_seg.
1084 */
1085 if (virtio_has_feature(vdev, VIRTIO_BLK_F_SECURE_ERASE)) {
1086
1087 virtio_cread(vdev, struct virtio_blk_config,
1088 secure_erase_sector_alignment, &v);
1089
1090 /* secure_erase_sector_alignment should not be zero, the device should set a
1091 * valid number of sectors.
1092 */
1093 if (!v) {
1094 dev_err(&vdev->dev,
1095 "virtio_blk: secure_erase_sector_alignment can't be 0\n");
1096 err = -EINVAL;
1097 goto out_cleanup_disk;
1098 }
1099
1100 discard_granularity = min_not_zero(discard_granularity, v);
1101
1102 virtio_cread(vdev, struct virtio_blk_config,
1103 max_secure_erase_sectors, &v);
1104
1105 /* max_secure_erase_sectors should not be zero, the device should set a
1106 * valid number of sectors.
1107 */
1108 if (!v) {
1109 dev_err(&vdev->dev,
1110 "virtio_blk: max_secure_erase_sectors can't be 0\n");
1111 err = -EINVAL;
1112 goto out_cleanup_disk;
1113 }
1114
1115 blk_queue_max_secure_erase_sectors(q, v);
1116
1117 virtio_cread(vdev, struct virtio_blk_config,
1118 max_secure_erase_seg, &v);
1119
1120 /* max_secure_erase_seg should not be zero, the device should set a
1121 * valid number of segments
1122 */
1123 if (!v) {
1124 dev_err(&vdev->dev,
1125 "virtio_blk: max_secure_erase_seg can't be 0\n");
1126 err = -EINVAL;
1127 goto out_cleanup_disk;
1128 }
1129
1130 max_discard_segs = min_not_zero(max_discard_segs, v);
1131 }
1132
1133 if (virtio_has_feature(vdev, VIRTIO_BLK_F_DISCARD) ||
1134 virtio_has_feature(vdev, VIRTIO_BLK_F_SECURE_ERASE)) {
1135 /* max_discard_seg and discard_granularity will be 0 only
1136 * if max_discard_seg and discard_sector_alignment fields in the virtio
1137 * config are 0 and VIRTIO_BLK_F_SECURE_ERASE feature is not negotiated.
1138 * In this case, we use default values.
1139 */
1140 if (!max_discard_segs)
1141 max_discard_segs = sg_elems;
1142
1143 blk_queue_max_discard_segments(q,
1144 min(max_discard_segs, MAX_DISCARD_SEGMENTS));
1145
1146 if (discard_granularity)
1147 q->limits.discard_granularity = discard_granularity << SECTOR_SHIFT;
1148 else
1149 q->limits.discard_granularity = blk_size;
1150 }
1151
1152 virtblk_update_capacity(vblk, false);
1153 virtio_device_ready(vdev);
1154
1155 err = device_add_disk(&vdev->dev, vblk->disk, virtblk_attr_groups);
1156 if (err)
1157 goto out_cleanup_disk;
1158
1159 return 0;
1160
1161out_cleanup_disk:
1162 put_disk(vblk->disk);
1163out_free_tags:
1164 blk_mq_free_tag_set(&vblk->tag_set);
1165out_free_vq:
1166 vdev->config->del_vqs(vdev);
1167 kfree(vblk->vqs);
1168out_free_vblk:
1169 kfree(vblk);
1170out_free_index:
1171 ida_free(&vd_index_ida, index);
1172out:
1173 return err;
1174}
1175
1176static void virtblk_remove(struct virtio_device *vdev)
1177{
1178 struct virtio_blk *vblk = vdev->priv;
1179
1180 /* Make sure no work handler is accessing the device. */
1181 flush_work(&vblk->config_work);
1182
1183 del_gendisk(vblk->disk);
1184 blk_mq_free_tag_set(&vblk->tag_set);
1185
1186 mutex_lock(&vblk->vdev_mutex);
1187
1188 /* Stop all the virtqueues. */
1189 virtio_reset_device(vdev);
1190
1191 /* Virtqueues are stopped, nothing can use vblk->vdev anymore. */
1192 vblk->vdev = NULL;
1193
1194 vdev->config->del_vqs(vdev);
1195 kfree(vblk->vqs);
1196
1197 mutex_unlock(&vblk->vdev_mutex);
1198
1199 put_disk(vblk->disk);
1200}
1201
1202#ifdef CONFIG_PM_SLEEP
1203static int virtblk_freeze(struct virtio_device *vdev)
1204{
1205 struct virtio_blk *vblk = vdev->priv;
1206
1207 /* Ensure we don't receive any more interrupts */
1208 virtio_reset_device(vdev);
1209
1210 /* Make sure no work handler is accessing the device. */
1211 flush_work(&vblk->config_work);
1212
1213 blk_mq_quiesce_queue(vblk->disk->queue);
1214
1215 vdev->config->del_vqs(vdev);
1216 kfree(vblk->vqs);
1217
1218 return 0;
1219}
1220
1221static int virtblk_restore(struct virtio_device *vdev)
1222{
1223 struct virtio_blk *vblk = vdev->priv;
1224 int ret;
1225
1226 ret = init_vq(vdev->priv);
1227 if (ret)
1228 return ret;
1229
1230 virtio_device_ready(vdev);
1231
1232 blk_mq_unquiesce_queue(vblk->disk->queue);
1233 return 0;
1234}
1235#endif
1236
1237static const struct virtio_device_id id_table[] = {
1238 { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1239 { 0 },
1240};
1241
1242static unsigned int features_legacy[] = {
1243 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
1244 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
1245 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
1246 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
1247 VIRTIO_BLK_F_SECURE_ERASE,
1248}
1249;
1250static unsigned int features[] = {
1251 VIRTIO_BLK_F_SEG_MAX, VIRTIO_BLK_F_SIZE_MAX, VIRTIO_BLK_F_GEOMETRY,
1252 VIRTIO_BLK_F_RO, VIRTIO_BLK_F_BLK_SIZE,
1253 VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_TOPOLOGY, VIRTIO_BLK_F_CONFIG_WCE,
1254 VIRTIO_BLK_F_MQ, VIRTIO_BLK_F_DISCARD, VIRTIO_BLK_F_WRITE_ZEROES,
1255 VIRTIO_BLK_F_SECURE_ERASE,
1256};
1257
1258static struct virtio_driver virtio_blk = {
1259 .feature_table = features,
1260 .feature_table_size = ARRAY_SIZE(features),
1261 .feature_table_legacy = features_legacy,
1262 .feature_table_size_legacy = ARRAY_SIZE(features_legacy),
1263 .driver.name = KBUILD_MODNAME,
1264 .driver.owner = THIS_MODULE,
1265 .id_table = id_table,
1266 .probe = virtblk_probe,
1267 .remove = virtblk_remove,
1268 .config_changed = virtblk_config_changed,
1269#ifdef CONFIG_PM_SLEEP
1270 .freeze = virtblk_freeze,
1271 .restore = virtblk_restore,
1272#endif
1273};
1274
1275static int __init virtio_blk_init(void)
1276{
1277 int error;
1278
1279 virtblk_wq = alloc_workqueue("virtio-blk", 0, 0);
1280 if (!virtblk_wq)
1281 return -ENOMEM;
1282
1283 major = register_blkdev(0, "virtblk");
1284 if (major < 0) {
1285 error = major;
1286 goto out_destroy_workqueue;
1287 }
1288
1289 error = register_virtio_driver(&virtio_blk);
1290 if (error)
1291 goto out_unregister_blkdev;
1292 return 0;
1293
1294out_unregister_blkdev:
1295 unregister_blkdev(major, "virtblk");
1296out_destroy_workqueue:
1297 destroy_workqueue(virtblk_wq);
1298 return error;
1299}
1300
1301static void __exit virtio_blk_fini(void)
1302{
1303 unregister_virtio_driver(&virtio_blk);
1304 unregister_blkdev(major, "virtblk");
1305 destroy_workqueue(virtblk_wq);
1306}
1307module_init(virtio_blk_init);
1308module_exit(virtio_blk_fini);
1309
1310MODULE_DEVICE_TABLE(virtio, id_table);
1311MODULE_DESCRIPTION("Virtio block driver");
1312MODULE_LICENSE("GPL");