1/******************************************************************************
  2 *
  3 * Back-end of the driver for virtual block devices. This portion of the
  4 * driver exports a 'unified' block-device interface that can be accessed
  5 * by any operating system that implements a compatible front end. A
  6 * reference front-end implementation can be found in:
  7 *  drivers/block/xen-blkfront.c
  8 *
  9 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
 10 * Copyright (c) 2005, Christopher Clark
 11 *
 12 * This program is free software; you can redistribute it and/or
 13 * modify it under the terms of the GNU General Public License version 2
 14 * as published by the Free Software Foundation; or, when distributed
 15 * separately from the Linux kernel or incorporated into other
 16 * software packages, subject to the following license:
 17 *
 18 * Permission is hereby granted, free of charge, to any person obtaining a copy
 19 * of this source file (the "Software"), to deal in the Software without
 20 * restriction, including without limitation the rights to use, copy, modify,
 21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
 22 * and to permit persons to whom the Software is furnished to do so, subject to
 23 * the following conditions:
 24 *
 25 * The above copyright notice and this permission notice shall be included in
 26 * all copies or substantial portions of the Software.
 27 *
 28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 34 * IN THE SOFTWARE.
 35 */
 36
 
 
 37#include <linux/spinlock.h>
 38#include <linux/kthread.h>
 39#include <linux/list.h>
 40#include <linux/delay.h>
 41#include <linux/freezer.h>
 
 42
 43#include <xen/events.h>
 44#include <xen/page.h>
 
 45#include <asm/xen/hypervisor.h>
 46#include <asm/xen/hypercall.h>
 
 
 47#include "common.h"
 48
 49/*
 50 * These are rather arbitrary. They are fairly large because adjacent requests
 51 * pulled from a communication ring are quite likely to end up being part of
 52 * the same scatter/gather request at the disc.
 53 *
 54 * ** TRY INCREASING 'xen_blkif_reqs' IF WRITE SPEEDS SEEM TOO LOW **
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 55 *
 56 * This will increase the chances of being able to write whole tracks.
 57 * 64 should be enough to keep us competitive with Linux.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 58 */
 59static int xen_blkif_reqs = 64;
 60module_param_named(reqs, xen_blkif_reqs, int, 0);
 61MODULE_PARM_DESC(reqs, "Number of blkback requests to allocate");
 
 
 
 
 
 62
 63/* Run-time switchable: /sys/module/blkback/parameters/ */
 64static unsigned int log_stats;
 65module_param(log_stats, int, 0644);
 66
 67/*
 68 * Each outstanding request that we've passed to the lower device layers has a
 69 * 'pending_req' allocated to it. Each buffer_head that completes decrements
 70 * the pendcnt towards zero. When it hits zero, the specified domain has a
 71 * response queued for it, with the saved 'id' passed back.
 72 */
 73struct pending_req {
 74	struct xen_blkif	*blkif;
 75	u64			id;
 76	int			nr_pages;
 77	atomic_t		pendcnt;
 78	unsigned short		operation;
 79	int			status;
 80	struct list_head	free_list;
 81};
 82
 83#define BLKBACK_INVALID_HANDLE (~0)
 84
 85struct xen_blkbk {
 86	struct pending_req	*pending_reqs;
 87	/* List of all 'pending_req' available */
 88	struct list_head	pending_free;
 89	/* And its spinlock. */
 90	spinlock_t		pending_free_lock;
 91	wait_queue_head_t	pending_free_wq;
 92	/* The list of all pages that are available. */
 93	struct page		**pending_pages;
 94	/* And the grant handles that are available. */
 95	grant_handle_t		*pending_grant_handles;
 96};
 
 
 
 
 
 
 
 
 
 97
 98static struct xen_blkbk *blkbk;
 99
100/*
101 * Little helpful macro to figure out the index and virtual address of the
102 * pending_pages[..]. For each 'pending_req' we have have up to
103 * BLKIF_MAX_SEGMENTS_PER_REQUEST (11) pages. The seg would be from 0 through
104 * 10 and would index in the pending_pages[..].
 
 
 
 
105 */
106static inline int vaddr_pagenr(struct pending_req *req, int seg)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
107{
108	return (req - blkbk->pending_reqs) *
109		BLKIF_MAX_SEGMENTS_PER_REQUEST + seg;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110}
111
112#define pending_page(req, seg) pending_pages[vaddr_pagenr(req, seg)]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
113
114static inline unsigned long vaddr(struct pending_req *req, int seg)
 
 
 
 
115{
116	unsigned long pfn = page_to_pfn(blkbk->pending_page(req, seg));
117	return (unsigned long)pfn_to_kaddr(pfn);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118}
119
120#define pending_handle(_req, _seg) \
121	(blkbk->pending_grant_handles[vaddr_pagenr(_req, _seg)])
 
 
 
 
 
122
 
 
 
 
123
124static int do_block_io_op(struct xen_blkif *blkif);
125static int dispatch_rw_block_io(struct xen_blkif *blkif,
126				struct blkif_request *req,
127				struct pending_req *pending_req);
128static void make_response(struct xen_blkif *blkif, u64 id,
129			  unsigned short op, int st);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
130
131/*
132 * Retrieve from the 'pending_reqs' a free pending_req structure to be used.
133 */
134static struct pending_req *alloc_req(void)
135{
136	struct pending_req *req = NULL;
137	unsigned long flags;
138
139	spin_lock_irqsave(&blkbk->pending_free_lock, flags);
140	if (!list_empty(&blkbk->pending_free)) {
141		req = list_entry(blkbk->pending_free.next, struct pending_req,
142				 free_list);
143		list_del(&req->free_list);
144	}
145	spin_unlock_irqrestore(&blkbk->pending_free_lock, flags);
146	return req;
147}
148
149/*
150 * Return the 'pending_req' structure back to the freepool. We also
151 * wake up the thread if it was waiting for a free page.
152 */
153static void free_req(struct pending_req *req)
154{
155	unsigned long flags;
156	int was_empty;
157
158	spin_lock_irqsave(&blkbk->pending_free_lock, flags);
159	was_empty = list_empty(&blkbk->pending_free);
160	list_add(&req->free_list, &blkbk->pending_free);
161	spin_unlock_irqrestore(&blkbk->pending_free_lock, flags);
162	if (was_empty)
163		wake_up(&blkbk->pending_free_wq);
164}
165
166/*
167 * Routines for managing virtual block devices (vbds).
168 */
169static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif,
170			     int operation)
171{
172	struct xen_vbd *vbd = &blkif->vbd;
173	int rc = -EACCES;
174
175	if ((operation != READ) && vbd->readonly)
176		goto out;
177
178	if (likely(req->nr_sects)) {
179		blkif_sector_t end = req->sector_number + req->nr_sects;
180
181		if (unlikely(end < req->sector_number))
182			goto out;
183		if (unlikely(end > vbd_sz(vbd)))
184			goto out;
185	}
186
187	req->dev  = vbd->pdevice;
188	req->bdev = vbd->bdev;
189	rc = 0;
190
191 out:
192	return rc;
193}
194
195static void xen_vbd_resize(struct xen_blkif *blkif)
196{
197	struct xen_vbd *vbd = &blkif->vbd;
198	struct xenbus_transaction xbt;
199	int err;
200	struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be);
201	unsigned long long new_size = vbd_sz(vbd);
202
203	pr_info(DRV_PFX "VBD Resize: Domid: %d, Device: (%d, %d)\n",
204		blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice));
205	pr_info(DRV_PFX "VBD Resize: new size %llu\n", new_size);
206	vbd->size = new_size;
207again:
208	err = xenbus_transaction_start(&xbt);
209	if (err) {
210		pr_warn(DRV_PFX "Error starting transaction");
211		return;
212	}
213	err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
214			    (unsigned long long)vbd_sz(vbd));
215	if (err) {
216		pr_warn(DRV_PFX "Error writing new size");
217		goto abort;
218	}
219	/*
220	 * Write the current state; we will use this to synchronize
221	 * the front-end. If the current state is "connected" the
222	 * front-end will get the new size information online.
223	 */
224	err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state);
225	if (err) {
226		pr_warn(DRV_PFX "Error writing the state");
227		goto abort;
228	}
229
230	err = xenbus_transaction_end(xbt, 0);
231	if (err == -EAGAIN)
232		goto again;
233	if (err)
234		pr_warn(DRV_PFX "Error ending transaction");
235	return;
236abort:
237	xenbus_transaction_end(xbt, 1);
238}
239
240/*
241 * Notification from the guest OS.
242 */
243static void blkif_notify_work(struct xen_blkif *blkif)
244{
245	blkif->waiting_reqs = 1;
246	wake_up(&blkif->wq);
247}
248
249irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
250{
251	blkif_notify_work(dev_id);
252	return IRQ_HANDLED;
253}
254
255/*
256 * SCHEDULER FUNCTIONS
257 */
258
259static void print_stats(struct xen_blkif *blkif)
260{
261	pr_info("xen-blkback (%s): oo %3d  |  rd %4d  |  wr %4d  |  f %4d\n",
262		 current->comm, blkif->st_oo_req,
263		 blkif->st_rd_req, blkif->st_wr_req, blkif->st_f_req);
264	blkif->st_print = jiffies + msecs_to_jiffies(10 * 1000);
265	blkif->st_rd_req = 0;
266	blkif->st_wr_req = 0;
267	blkif->st_oo_req = 0;
 
 
 
 
268}
269
270int xen_blkif_schedule(void *arg)
271{
272	struct xen_blkif *blkif = arg;
 
273	struct xen_vbd *vbd = &blkif->vbd;
 
 
 
 
274
275	xen_blkif_get(blkif);
276
277	while (!kthread_should_stop()) {
278		if (try_to_freeze())
279			continue;
280		if (unlikely(vbd->size != vbd_sz(vbd)))
281			xen_vbd_resize(blkif);
282
283		wait_event_interruptible(
284			blkif->wq,
285			blkif->waiting_reqs || kthread_should_stop());
286		wait_event_interruptible(
287			blkbk->pending_free_wq,
288			!list_empty(&blkbk->pending_free) ||
289			kthread_should_stop());
 
 
 
 
 
 
 
 
290
291		blkif->waiting_reqs = 0;
 
 
292		smp_mb(); /* clear flag *before* checking for work */
293
294		if (do_block_io_op(blkif))
295			blkif->waiting_reqs = 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
296
297		if (log_stats && time_after(jiffies, blkif->st_print))
298			print_stats(blkif);
 
 
 
 
 
 
 
299	}
300
 
 
 
301	if (log_stats)
302		print_stats(blkif);
303
304	blkif->xenblkd = NULL;
305	xen_blkif_put(blkif);
306
307	return 0;
308}
309
310struct seg_buf {
311	unsigned long buf;
312	unsigned int nsec;
313};
314/*
315 * Unmap the grant references, and also remove the M2P over-rides
316 * used in the 'pending_req'.
317 */
318static void xen_blkbk_unmap(struct pending_req *req)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
319{
320	struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
321	unsigned int i, invcount = 0;
322	grant_handle_t handle;
323	int ret;
324
325	for (i = 0; i < req->nr_pages; i++) {
326		handle = pending_handle(req, i);
327		if (handle == BLKBACK_INVALID_HANDLE)
 
 
 
328			continue;
329		gnttab_set_unmap_op(&unmap[invcount], vaddr(req, i),
330				    GNTMAP_host_map, handle);
331		pending_handle(req, i) = BLKBACK_INVALID_HANDLE;
 
332		invcount++;
333	}
334
335	ret = HYPERVISOR_grant_table_op(
336		GNTTABOP_unmap_grant_ref, unmap, invcount);
337	BUG_ON(ret);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
338	/*
339	 * Note, we use invcount, so nr->pages, so we can't index
340	 * using vaddr(req, i).
 
 
 
 
 
 
 
 
341	 */
342	for (i = 0; i < invcount; i++) {
343		ret = m2p_remove_override(
344			virt_to_page(unmap[i].host_addr), false);
345		if (ret) {
346			pr_alert(DRV_PFX "Failed to remove M2P override for %lx\n",
347				 (unsigned long)unmap[i].host_addr);
348			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
349		}
 
 
350	}
351}
352
353static int xen_blkbk_map(struct blkif_request *req,
354			 struct pending_req *pending_req,
355			 struct seg_buf seg[])
356{
357	struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
358	int i;
359	int nseg = req->nr_segments;
 
 
 
360	int ret = 0;
 
 
 
 
 
361
362	/*
363	 * Fill out preq.nr_sects with proper amount of sectors, and setup
364	 * assign map[..] with the PFN of the page in our domain with the
365	 * corresponding grant reference for each page.
366	 */
367	for (i = 0; i < nseg; i++) {
 
368		uint32_t flags;
369
370		flags = GNTMAP_host_map;
371		if (pending_req->operation != BLKIF_OP_READ)
372			flags |= GNTMAP_readonly;
373		gnttab_set_map_op(&map[i], vaddr(pending_req, i), flags,
374				  req->u.rw.seg[i].gref,
375				  pending_req->blkif->domid);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
376	}
377
378	ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map, nseg);
379	BUG_ON(ret);
380
381	/*
382	 * Now swizzle the MFN in our domain with the MFN from the other domain
383	 * so that when we access vaddr(pending_req,i) it has the contents of
384	 * the page from the other domain.
385	 */
386	for (i = 0; i < nseg; i++) {
387		if (unlikely(map[i].status != 0)) {
388			pr_debug(DRV_PFX "invalid buffer -- could not remap it\n");
389			map[i].handle = BLKBACK_INVALID_HANDLE;
390			ret |= 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
391		}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
392
393		pending_handle(pending_req, i) = map[i].handle;
 
 
 
 
 
 
394
395		if (ret)
396			continue;
397
398		ret = m2p_add_override(PFN_DOWN(map[i].dev_bus_addr),
399			blkbk->pending_page(pending_req, i), false);
400		if (ret) {
401			pr_alert(DRV_PFX "Failed to install M2P override for %lx (ret: %d)\n",
402				 (unsigned long)map[i].dev_bus_addr, ret);
403			/* We could switch over to GNTTABOP_copy */
404			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
405		}
406
407		seg[i].buf  = map[i].dev_bus_addr |
408			(req->u.rw.seg[i].first_sect << 9);
 
409	}
410	return ret;
 
 
 
 
 
411}
412
413/*
414 * Completion callback on the bio's. Called as bh->b_end_io()
415 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
416
417static void __end_block_io_op(struct pending_req *pending_req, int error)
 
418{
419	/* An error fails the entire request. */
420	if ((pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE) &&
421	    (error == -EOPNOTSUPP)) {
422		pr_debug(DRV_PFX "flush diskcache op failed, not supported\n");
423		xen_blkbk_flush_diskcache(XBT_NIL, pending_req->blkif->be, 0);
 
 
 
 
 
424		pending_req->status = BLKIF_RSP_EOPNOTSUPP;
425	} else if (error) {
426		pr_debug(DRV_PFX "Buffer not up-to-date at end of operation,"
427			 " error=%d\n", error);
428		pending_req->status = BLKIF_RSP_ERROR;
429	}
430
431	/*
432	 * If all of the bio's have completed it is time to unmap
433	 * the grant references associated with 'request' and provide
434	 * the proper response on the ring.
435	 */
436	if (atomic_dec_and_test(&pending_req->pendcnt)) {
437		xen_blkbk_unmap(pending_req);
438		make_response(pending_req->blkif, pending_req->id,
439			      pending_req->operation, pending_req->status);
440		xen_blkif_put(pending_req->blkif);
441		free_req(pending_req);
442	}
443}
444
445/*
446 * bio callback.
447 */
448static void end_block_io_op(struct bio *bio, int error)
449{
450	__end_block_io_op(bio->bi_private, error);
451	bio_put(bio);
452}
453
 
 
 
 
454
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
455
456/*
457 * Function to copy the from the ring buffer the 'struct blkif_request'
458 * (which has the sectors we want, number of them, grant references, etc),
459 * and transmute  it to the block API to hand it over to the proper block disk.
460 */
461static int
462__do_block_io_op(struct xen_blkif *blkif)
463{
464	union blkif_back_rings *blk_rings = &blkif->blk_rings;
465	struct blkif_request req;
466	struct pending_req *pending_req;
467	RING_IDX rc, rp;
468	int more_to_do = 0;
469
470	rc = blk_rings->common.req_cons;
471	rp = blk_rings->common.sring->req_prod;
472	rmb(); /* Ensure we see queued requests up to 'rp'. */
473
 
 
 
 
 
 
474	while (rc != rp) {
475
476		if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc))
477			break;
478
 
 
 
479		if (kthread_should_stop()) {
480			more_to_do = 1;
481			break;
482		}
483
484		pending_req = alloc_req();
485		if (NULL == pending_req) {
486			blkif->st_oo_req++;
487			more_to_do = 1;
488			break;
489		}
490
491		switch (blkif->blk_protocol) {
492		case BLKIF_PROTOCOL_NATIVE:
493			memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));
494			break;
495		case BLKIF_PROTOCOL_X86_32:
496			blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc));
497			break;
498		case BLKIF_PROTOCOL_X86_64:
499			blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc));
500			break;
501		default:
502			BUG();
503		}
504		blk_rings->common.req_cons = ++rc; /* before make_response() */
505
506		/* Apply all sanity checks to /private copy/ of request. */
507		barrier();
508
509		if (dispatch_rw_block_io(blkif, &req, pending_req))
 
 
 
 
 
 
 
510			break;
 
 
 
 
 
 
 
 
 
 
511
512		/* Yield point for this unbounded loop. */
513		cond_resched();
514	}
515
516	return more_to_do;
517}
518
519static int
520do_block_io_op(struct xen_blkif *blkif)
521{
522	union blkif_back_rings *blk_rings = &blkif->blk_rings;
523	int more_to_do;
524
525	do {
526		more_to_do = __do_block_io_op(blkif);
527		if (more_to_do)
528			break;
529
530		RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do);
531	} while (more_to_do);
532
533	return more_to_do;
534}
535
536/*
537 * Transmutation of the 'struct blkif_request' to a proper 'struct bio'
538 * and call the 'submit_bio' to pass it to the underlying storage.
539 */
540static int dispatch_rw_block_io(struct xen_blkif *blkif,
541				struct blkif_request *req,
542				struct pending_req *pending_req)
543{
544	struct phys_req preq;
545	struct seg_buf seg[BLKIF_MAX_SEGMENTS_PER_REQUEST];
546	unsigned int nseg;
547	struct bio *bio = NULL;
548	struct bio *biolist[BLKIF_MAX_SEGMENTS_PER_REQUEST];
549	int i, nbio = 0;
550	int operation;
 
551	struct blk_plug plug;
 
 
 
 
 
 
 
 
 
 
 
 
 
552
553	switch (req->operation) {
554	case BLKIF_OP_READ:
555		blkif->st_rd_req++;
556		operation = READ;
557		break;
558	case BLKIF_OP_WRITE:
559		blkif->st_wr_req++;
560		operation = WRITE_ODIRECT;
 
561		break;
 
 
 
562	case BLKIF_OP_FLUSH_DISKCACHE:
563		blkif->st_f_req++;
564		operation = WRITE_FLUSH;
 
565		break;
566	case BLKIF_OP_WRITE_BARRIER:
567	default:
568		operation = 0; /* make gcc happy */
569		goto fail_response;
570		break;
571	}
572
573	/* Check that the number of segments is sane. */
574	nseg = req->nr_segments;
575	if (unlikely(nseg == 0 && operation != WRITE_FLUSH) ||
576	    unlikely(nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) {
577		pr_debug(DRV_PFX "Bad number of segments in request (%d)\n",
578			 nseg);
 
 
 
 
579		/* Haven't submitted any bio's yet. */
580		goto fail_response;
581	}
582
583	preq.dev           = req->handle;
584	preq.sector_number = req->u.rw.sector_number;
585	preq.nr_sects      = 0;
586
587	pending_req->blkif     = blkif;
588	pending_req->id        = req->id;
589	pending_req->operation = req->operation;
590	pending_req->status    = BLKIF_RSP_OKAY;
591	pending_req->nr_pages  = nseg;
592
593	for (i = 0; i < nseg; i++) {
594		seg[i].nsec = req->u.rw.seg[i].last_sect -
595			req->u.rw.seg[i].first_sect + 1;
596		if ((req->u.rw.seg[i].last_sect >= (PAGE_SIZE >> 9)) ||
597		    (req->u.rw.seg[i].last_sect < req->u.rw.seg[i].first_sect))
 
 
 
 
 
 
 
 
 
 
 
 
 
598			goto fail_response;
599		preq.nr_sects += seg[i].nsec;
600
601	}
602
603	if (xen_vbd_translate(&preq, blkif, operation) != 0) {
604		pr_debug(DRV_PFX "access denied: %s of [%llu,%llu] on dev=%04x\n",
605			 operation == READ ? "read" : "write",
606			 preq.sector_number,
607			 preq.sector_number + preq.nr_sects, preq.dev);
 
608		goto fail_response;
609	}
610
611	/*
612	 * This check _MUST_ be done after xen_vbd_translate as the preq.bdev
613	 * is set there.
614	 */
615	for (i = 0; i < nseg; i++) {
616		if (((int)preq.sector_number|(int)seg[i].nsec) &
617		    ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {
618			pr_debug(DRV_PFX "Misaligned I/O request from domain %d",
619				 blkif->domid);
620			goto fail_response;
621		}
622	}
623
 
 
 
 
 
 
624	/*
625	 * If we have failed at this point, we need to undo the M2P override,
626	 * set gnttab_set_unmap_op on all of the grant references and perform
627	 * the hypercall to unmap the grants - that is all done in
628	 * xen_blkbk_unmap.
629	 */
630	if (xen_blkbk_map(req, pending_req, seg))
631		goto fail_flush;
632
633	/* This corresponding xen_blkif_put is done in __end_block_io_op */
634	xen_blkif_get(blkif);
 
 
 
 
635
636	for (i = 0; i < nseg; i++) {
637		while ((bio == NULL) ||
638		       (bio_add_page(bio,
639				     blkbk->pending_page(pending_req, i),
640				     seg[i].nsec << 9,
641				     seg[i].buf & ~PAGE_MASK) == 0)) {
642
643			bio = bio_alloc(GFP_KERNEL, nseg-i);
644			if (unlikely(bio == NULL))
645				goto fail_put_bio;
646
647			biolist[nbio++] = bio;
648			bio->bi_bdev    = preq.bdev;
649			bio->bi_private = pending_req;
650			bio->bi_end_io  = end_block_io_op;
651			bio->bi_sector  = preq.sector_number;
652		}
653
654		preq.sector_number += seg[i].nsec;
655	}
656
657	/* This will be hit if the operation was a flush. */
658	if (!bio) {
659		BUG_ON(operation != WRITE_FLUSH);
660
661		bio = bio_alloc(GFP_KERNEL, 0);
662		if (unlikely(bio == NULL))
663			goto fail_put_bio;
664
 
 
665		biolist[nbio++] = bio;
666		bio->bi_bdev    = preq.bdev;
667		bio->bi_private = pending_req;
668		bio->bi_end_io  = end_block_io_op;
669	}
670
671	/*
672	 * We set it one so that the last submit_bio does not have to call
673	 * atomic_inc.
674	 */
675	atomic_set(&pending_req->pendcnt, nbio);
676
677	/* Get a reference count for the disk queue and start sending I/O */
678	blk_start_plug(&plug);
679
680	for (i = 0; i < nbio; i++)
681		submit_bio(operation, biolist[i]);
682
683	/* Let the I/Os go.. */
684	blk_finish_plug(&plug);
685
686	if (operation == READ)
687		blkif->st_rd_sect += preq.nr_sects;
688	else if (operation == WRITE || operation == WRITE_FLUSH)
689		blkif->st_wr_sect += preq.nr_sects;
690
691	return 0;
692
693 fail_flush:
694	xen_blkbk_unmap(pending_req);
 
695 fail_response:
696	/* Haven't submitted any bio's yet. */
697	make_response(blkif, req->id, req->operation, BLKIF_RSP_ERROR);
698	free_req(pending_req);
699	msleep(1); /* back off a bit */
700	return -EIO;
701
702 fail_put_bio:
703	for (i = 0; i < nbio; i++)
704		bio_put(biolist[i]);
705	__end_block_io_op(pending_req, -EINVAL);
706	msleep(1); /* back off a bit */
707	return -EIO;
708}
709
710
711
712/*
713 * Put a response on the ring on how the operation fared.
714 */
715static void make_response(struct xen_blkif *blkif, u64 id,
716			  unsigned short op, int st)
717{
718	struct blkif_response  resp;
719	unsigned long     flags;
720	union blkif_back_rings *blk_rings = &blkif->blk_rings;
721	int notify;
722
723	resp.id        = id;
724	resp.operation = op;
725	resp.status    = st;
726
727	spin_lock_irqsave(&blkif->blk_ring_lock, flags);
728	/* Place on the response ring for the relevant domain. */
729	switch (blkif->blk_protocol) {
730	case BLKIF_PROTOCOL_NATIVE:
731		memcpy(RING_GET_RESPONSE(&blk_rings->native, blk_rings->native.rsp_prod_pvt),
732		       &resp, sizeof(resp));
733		break;
734	case BLKIF_PROTOCOL_X86_32:
735		memcpy(RING_GET_RESPONSE(&blk_rings->x86_32, blk_rings->x86_32.rsp_prod_pvt),
736		       &resp, sizeof(resp));
737		break;
738	case BLKIF_PROTOCOL_X86_64:
739		memcpy(RING_GET_RESPONSE(&blk_rings->x86_64, blk_rings->x86_64.rsp_prod_pvt),
740		       &resp, sizeof(resp));
741		break;
742	default:
743		BUG();
744	}
 
 
 
 
 
745	blk_rings->common.rsp_prod_pvt++;
746	RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);
747	spin_unlock_irqrestore(&blkif->blk_ring_lock, flags);
748	if (notify)
749		notify_remote_via_irq(blkif->irq);
750}
751
752static int __init xen_blkif_init(void)
753{
754	int i, mmap_pages;
755	int rc = 0;
756
757	if (!xen_pv_domain())
758		return -ENODEV;
759
760	blkbk = kzalloc(sizeof(struct xen_blkbk), GFP_KERNEL);
761	if (!blkbk) {
762		pr_alert(DRV_PFX "%s: out of memory!\n", __func__);
763		return -ENOMEM;
764	}
765
766	mmap_pages = xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST;
767
768	blkbk->pending_reqs          = kmalloc(sizeof(blkbk->pending_reqs[0]) *
769					xen_blkif_reqs, GFP_KERNEL);
770	blkbk->pending_grant_handles = kzalloc(sizeof(blkbk->pending_grant_handles[0]) *
771					mmap_pages, GFP_KERNEL);
772	blkbk->pending_pages         = kzalloc(sizeof(blkbk->pending_pages[0]) *
773					mmap_pages, GFP_KERNEL);
774
775	if (!blkbk->pending_reqs || !blkbk->pending_grant_handles ||
776	    !blkbk->pending_pages) {
777		rc = -ENOMEM;
778		goto out_of_memory;
779	}
780
781	for (i = 0; i < mmap_pages; i++) {
782		blkbk->pending_grant_handles[i] = BLKBACK_INVALID_HANDLE;
783		blkbk->pending_pages[i] = alloc_page(GFP_KERNEL);
784		if (blkbk->pending_pages[i] == NULL) {
785			rc = -ENOMEM;
786			goto out_of_memory;
787		}
788	}
 
 
 
 
789	rc = xen_blkif_interface_init();
790	if (rc)
791		goto failed_init;
792
793	memset(blkbk->pending_reqs, 0, sizeof(blkbk->pending_reqs));
794
795	INIT_LIST_HEAD(&blkbk->pending_free);
796	spin_lock_init(&blkbk->pending_free_lock);
797	init_waitqueue_head(&blkbk->pending_free_wq);
798
799	for (i = 0; i < xen_blkif_reqs; i++)
800		list_add_tail(&blkbk->pending_reqs[i].free_list,
801			      &blkbk->pending_free);
802
803	rc = xen_blkif_xenbus_init();
804	if (rc)
805		goto failed_init;
806
807	return 0;
808
809 out_of_memory:
810	pr_alert(DRV_PFX "%s: out of memory\n", __func__);
811 failed_init:
812	kfree(blkbk->pending_reqs);
813	kfree(blkbk->pending_grant_handles);
814	if (blkbk->pending_pages) {
815		for (i = 0; i < mmap_pages; i++) {
816			if (blkbk->pending_pages[i])
817				__free_page(blkbk->pending_pages[i]);
818		}
819		kfree(blkbk->pending_pages);
820	}
821	kfree(blkbk);
822	blkbk = NULL;
823	return rc;
824}
825
826module_init(xen_blkif_init);
827
 
 
 
 
 
 
 
 
 
828MODULE_LICENSE("Dual BSD/GPL");
829MODULE_ALIAS("xen-backend:vbd");