1/*
  2 * Common Block IO controller cgroup interface
  3 *
  4 * Based on ideas and code from CFQ, CFS and BFQ:
  5 * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
  6 *
  7 * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
  8 *		      Paolo Valente <paolo.valente@unimore.it>
  9 *
 10 * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
 11 * 	              Nauman Rafique <nauman@google.com>
 12 */
 13#include <linux/ioprio.h>
 
 14#include <linux/kdev_t.h>
 15#include <linux/module.h>
 16#include <linux/err.h>
 17#include <linux/blkdev.h>
 18#include <linux/slab.h>
 19#include <linux/genhd.h>
 20#include <linux/delay.h>
 21#include <linux/atomic.h>
 22#include "blk-cgroup.h"
 23#include "blk.h"
 24
 25#define MAX_KEY_LEN 100
 26
 27static DEFINE_MUTEX(blkcg_pol_mutex);
 
 28
 29struct blkcg blkcg_root = { .cfq_weight = 2 * CFQ_WEIGHT_DEFAULT };
 30EXPORT_SYMBOL_GPL(blkcg_root);
 31
 32static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 33
 34struct blkcg *cgroup_to_blkcg(struct cgroup *cgroup)
 
 35{
 36	return container_of(cgroup_subsys_state(cgroup, blkio_subsys_id),
 37			    struct blkcg, css);
 38}
 39EXPORT_SYMBOL_GPL(cgroup_to_blkcg);
 40
 41static struct blkcg *task_blkcg(struct task_struct *tsk)
 
 42{
 43	return container_of(task_subsys_state(tsk, blkio_subsys_id),
 44			    struct blkcg, css);
 
 
 
 
 45}
 46
 47struct blkcg *bio_blkcg(struct bio *bio)
 
 
 48{
 49	if (bio && bio->bi_css)
 50		return container_of(bio->bi_css, struct blkcg, css);
 51	return task_blkcg(current);
 
 52}
 53EXPORT_SYMBOL_GPL(bio_blkcg);
 54
 55static bool blkcg_policy_enabled(struct request_queue *q,
 56				 const struct blkcg_policy *pol)
 57{
 58	return pol && test_bit(pol->plid, q->blkcg_pols);
 59}
 60
 61/**
 62 * blkg_free - free a blkg
 63 * @blkg: blkg to free
 64 *
 65 * Free @blkg which may be partially allocated.
 66 */
 67static void blkg_free(struct blkcg_gq *blkg)
 68{
 69	int i;
 70
 71	if (!blkg)
 72		return;
 
 
 73
 74	for (i = 0; i < BLKCG_MAX_POLS; i++) {
 75		struct blkcg_policy *pol = blkcg_policy[i];
 76		struct blkg_policy_data *pd = blkg->pd[i];
 77
 78		if (!pd)
 79			continue;
 
 
 
 
 80
 81		if (pol && pol->pd_exit_fn)
 82			pol->pd_exit_fn(blkg);
 
 
 
 
 83
 84		kfree(pd);
 85	}
 
 
 86
 87	kfree(blkg);
 
 
 
 
 
 
 
 88}
 89
 90/**
 91 * blkg_alloc - allocate a blkg
 92 * @blkcg: block cgroup the new blkg is associated with
 93 * @q: request_queue the new blkg is associated with
 94 *
 95 * Allocate a new blkg assocating @blkcg and @q.
 96 */
 97static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q)
 98{
 99	struct blkcg_gq *blkg;
100	int i;
101
102	/* alloc and init base part */
103	blkg = kzalloc_node(sizeof(*blkg), GFP_ATOMIC, q->node);
104	if (!blkg)
105		return NULL;
106
107	blkg->q = q;
108	INIT_LIST_HEAD(&blkg->q_node);
109	blkg->blkcg = blkcg;
110	blkg->refcnt = 1;
111
112	for (i = 0; i < BLKCG_MAX_POLS; i++) {
113		struct blkcg_policy *pol = blkcg_policy[i];
114		struct blkg_policy_data *pd;
115
116		if (!blkcg_policy_enabled(q, pol))
 
117			continue;
118
119		/* alloc per-policy data and attach it to blkg */
120		pd = kzalloc_node(pol->pd_size, GFP_ATOMIC, q->node);
121		if (!pd) {
122			blkg_free(blkg);
123			return NULL;
124		}
 
 
 
 
 
 
 
 
 
 
125
126		blkg->pd[i] = pd;
127		pd->blkg = blkg;
128
129		/* invoke per-policy init */
130		if (blkcg_policy_enabled(blkg->q, pol))
131			pol->pd_init_fn(blkg);
 
 
 
 
 
 
 
 
 
 
132	}
 
133
134	return blkg;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
135}
136
137static struct blkcg_gq *__blkg_lookup(struct blkcg *blkcg,
138				      struct request_queue *q)
 
 
 
 
139{
140	struct blkcg_gq *blkg;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
141
142	blkg = rcu_dereference(blkcg->blkg_hint);
143	if (blkg && blkg->q == q)
144		return blkg;
 
145
146	/*
147	 * Hint didn't match.  Look up from the radix tree.  Note that we
148	 * may not be holding queue_lock and thus are not sure whether
149	 * @blkg from blkg_tree has already been removed or not, so we
150	 * can't update hint to the lookup result.  Leave it to the caller.
151	 */
152	blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
153	if (blkg && blkg->q == q)
154		return blkg;
155
156	return NULL;
 
 
 
157}
158
159/**
160 * blkg_lookup - lookup blkg for the specified blkcg - q pair
161 * @blkcg: blkcg of interest
162 * @q: request_queue of interest
163 *
164 * Lookup blkg for the @blkcg - @q pair.  This function should be called
165 * under RCU read lock and is guaranteed to return %NULL if @q is bypassing
166 * - see blk_queue_bypass_start() for details.
167 */
168struct blkcg_gq *blkg_lookup(struct blkcg *blkcg, struct request_queue *q)
169{
170	WARN_ON_ONCE(!rcu_read_lock_held());
 
 
 
171
172	if (unlikely(blk_queue_bypass(q)))
173		return NULL;
174	return __blkg_lookup(blkcg, q);
 
175}
176EXPORT_SYMBOL_GPL(blkg_lookup);
177
178static struct blkcg_gq *__blkg_lookup_create(struct blkcg *blkcg,
179					     struct request_queue *q)
180	__releases(q->queue_lock) __acquires(q->queue_lock)
181{
182	struct blkcg_gq *blkg;
183	int ret;
184
185	WARN_ON_ONCE(!rcu_read_lock_held());
186	lockdep_assert_held(q->queue_lock);
187
188	/* lookup and update hint on success, see __blkg_lookup() for details */
189	blkg = __blkg_lookup(blkcg, q);
190	if (blkg) {
191		rcu_assign_pointer(blkcg->blkg_hint, blkg);
192		return blkg;
193	}
194
195	/* blkg holds a reference to blkcg */
196	if (!css_tryget(&blkcg->css))
197		return ERR_PTR(-EINVAL);
198
199	/* allocate */
200	ret = -ENOMEM;
201	blkg = blkg_alloc(blkcg, q);
202	if (unlikely(!blkg))
203		goto err_put;
204
205	/* insert */
206	ret = radix_tree_preload(GFP_ATOMIC);
207	if (ret)
208		goto err_free;
 
209
210	spin_lock(&blkcg->lock);
211	ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
212	if (likely(!ret)) {
213		hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
214		list_add(&blkg->q_node, &q->blkg_list);
 
 
215	}
216	spin_unlock(&blkcg->lock);
 
 
217
218	radix_tree_preload_end();
 
 
 
219
220	if (!ret)
221		return blkg;
222err_free:
223	blkg_free(blkg);
224err_put:
225	css_put(&blkcg->css);
226	return ERR_PTR(ret);
 
227}
 
228
229struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
230				    struct request_queue *q)
231{
 
 
 
 
 
 
 
 
 
 
 
 
232	/*
233	 * This could be the first entry point of blkcg implementation and
234	 * we shouldn't allow anything to go through for a bypassing queue.
 
235	 */
236	if (unlikely(blk_queue_bypass(q)))
237		return ERR_PTR(blk_queue_dead(q) ? -EINVAL : -EBUSY);
238	return __blkg_lookup_create(blkcg, q);
 
 
 
 
 
239}
240EXPORT_SYMBOL_GPL(blkg_lookup_create);
241
242static void blkg_destroy(struct blkcg_gq *blkg)
 
243{
244	struct blkcg *blkcg = blkg->blkcg;
 
 
 
 
 
 
 
245
246	lockdep_assert_held(blkg->q->queue_lock);
247	lockdep_assert_held(&blkcg->lock);
 
 
 
248
249	/* Something wrong if we are trying to remove same group twice */
250	WARN_ON_ONCE(list_empty(&blkg->q_node));
251	WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
 
 
 
 
 
252
253	radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
254	list_del_init(&blkg->q_node);
255	hlist_del_init_rcu(&blkg->blkcg_node);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
256
257	/*
258	 * Both setting lookup hint to and clearing it from @blkg are done
259	 * under queue_lock.  If it's not pointing to @blkg now, it never
260	 * will.  Hint assignment itself can race safely.
261	 */
262	if (rcu_dereference_raw(blkcg->blkg_hint) == blkg)
263		rcu_assign_pointer(blkcg->blkg_hint, NULL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
264
265	/*
266	 * Put the reference taken at the time of creation so that when all
267	 * queues are gone, group can be destroyed.
 
268	 */
269	blkg_put(blkg);
 
 
 
 
 
 
 
 
270}
 
271
272/**
273 * blkg_destroy_all - destroy all blkgs associated with a request_queue
274 * @q: request_queue of interest
275 *
276 * Destroy all blkgs associated with @q.
277 */
278static void blkg_destroy_all(struct request_queue *q)
279{
280	struct blkcg_gq *blkg, *n;
 
 
 
 
 
 
281
282	lockdep_assert_held(q->queue_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
283
284	list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
285		struct blkcg *blkcg = blkg->blkcg;
 
 
 
286
287		spin_lock(&blkcg->lock);
288		blkg_destroy(blkg);
289		spin_unlock(&blkcg->lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
290	}
 
 
 
291}
 
292
293static void blkg_rcu_free(struct rcu_head *rcu_head)
 
294{
295	blkg_free(container_of(rcu_head, struct blkcg_gq, rcu_head));
 
 
 
 
 
 
 
 
 
 
296}
 
297
298void __blkg_release(struct blkcg_gq *blkg)
299{
300	/* release the extra blkcg reference this blkg has been holding */
301	css_put(&blkg->blkcg->css);
302
303	/*
304	 * A group is freed in rcu manner. But having an rcu lock does not
305	 * mean that one can access all the fields of blkg and assume these
306	 * are valid. For example, don't try to follow throtl_data and
307	 * request queue links.
 
308	 *
309	 * Having a reference to blkg under an rcu allows acess to only
310	 * values local to groups like group stats and group rate limits
 
311	 */
312	call_rcu(&blkg->rcu_head, blkg_rcu_free);
 
 
 
 
 
 
313}
314EXPORT_SYMBOL_GPL(__blkg_release);
315
316static int blkcg_reset_stats(struct cgroup *cgroup, struct cftype *cftype,
317			     u64 val)
318{
319	struct blkcg *blkcg = cgroup_to_blkcg(cgroup);
320	struct blkcg_gq *blkg;
 
321	struct hlist_node *n;
 
322	int i;
 
 
 
 
323
324	mutex_lock(&blkcg_pol_mutex);
325	spin_lock_irq(&blkcg->lock);
326
327	/*
328	 * Note that stat reset is racy - it doesn't synchronize against
329	 * stat updates.  This is a debug feature which shouldn't exist
330	 * anyway.  If you get hit by a race, retry.
331	 */
332	hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node) {
333		for (i = 0; i < BLKCG_MAX_POLS; i++) {
334			struct blkcg_policy *pol = blkcg_policy[i];
335
336			if (blkcg_policy_enabled(blkg->q, pol) &&
337			    pol->pd_reset_stats_fn)
338				pol->pd_reset_stats_fn(blkg);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
339		}
 
 
 
 
 
 
 
 
 
340	}
341
342	spin_unlock_irq(&blkcg->lock);
343	mutex_unlock(&blkcg_pol_mutex);
344	return 0;
345}
346
347static const char *blkg_dev_name(struct blkcg_gq *blkg)
 
348{
349	/* some drivers (floppy) instantiate a queue w/o disk registered */
350	if (blkg->q->backing_dev_info.dev)
351		return dev_name(blkg->q->backing_dev_info.dev);
352	return NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
353}
354
355/**
356 * blkcg_print_blkgs - helper for printing per-blkg data
357 * @sf: seq_file to print to
358 * @blkcg: blkcg of interest
359 * @prfill: fill function to print out a blkg
360 * @pol: policy in question
361 * @data: data to be passed to @prfill
362 * @show_total: to print out sum of prfill return values or not
363 *
364 * This function invokes @prfill on each blkg of @blkcg if pd for the
365 * policy specified by @pol exists.  @prfill is invoked with @sf, the
366 * policy data and @data.  If @show_total is %true, the sum of the return
367 * values from @prfill is printed with "Total" label at the end.
368 *
369 * This is to be used to construct print functions for
370 * cftype->read_seq_string method.
371 */
372void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
373		       u64 (*prfill)(struct seq_file *,
374				     struct blkg_policy_data *, int),
375		       const struct blkcg_policy *pol, int data,
376		       bool show_total)
377{
378	struct blkcg_gq *blkg;
379	struct hlist_node *n;
380	u64 total = 0;
381
382	spin_lock_irq(&blkcg->lock);
383	hlist_for_each_entry(blkg, n, &blkcg->blkg_list, blkcg_node)
384		if (blkcg_policy_enabled(blkg->q, pol))
385			total += prfill(sf, blkg->pd[pol->plid], data);
386	spin_unlock_irq(&blkcg->lock);
387
388	if (show_total)
389		seq_printf(sf, "Total %llu\n", (unsigned long long)total);
390}
391EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
392
393/**
394 * __blkg_prfill_u64 - prfill helper for a single u64 value
395 * @sf: seq_file to print to
396 * @pd: policy private data of interest
397 * @v: value to print
398 *
399 * Print @v to @sf for the device assocaited with @pd.
400 */
401u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
402{
403	const char *dname = blkg_dev_name(pd->blkg);
 
 
 
 
 
 
404
405	if (!dname)
406		return 0;
 
 
 
 
 
407
408	seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
409	return v;
 
 
410}
411EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
412
413/**
414 * __blkg_prfill_rwstat - prfill helper for a blkg_rwstat
415 * @sf: seq_file to print to
416 * @pd: policy private data of interest
417 * @rwstat: rwstat to print
418 *
419 * Print @rwstat to @sf for the device assocaited with @pd.
420 */
421u64 __blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
422			 const struct blkg_rwstat *rwstat)
423{
424	static const char *rwstr[] = {
425		[BLKG_RWSTAT_READ]	= "Read",
426		[BLKG_RWSTAT_WRITE]	= "Write",
427		[BLKG_RWSTAT_SYNC]	= "Sync",
428		[BLKG_RWSTAT_ASYNC]	= "Async",
429	};
430	const char *dname = blkg_dev_name(pd->blkg);
431	u64 v;
432	int i;
433
434	if (!dname)
435		return 0;
 
 
436
437	for (i = 0; i < BLKG_RWSTAT_NR; i++)
438		seq_printf(sf, "%s %s %llu\n", dname, rwstr[i],
439			   (unsigned long long)rwstat->cnt[i]);
440
441	v = rwstat->cnt[BLKG_RWSTAT_READ] + rwstat->cnt[BLKG_RWSTAT_WRITE];
442	seq_printf(sf, "%s Total %llu\n", dname, (unsigned long long)v);
443	return v;
444}
445
446/**
447 * blkg_prfill_stat - prfill callback for blkg_stat
448 * @sf: seq_file to print to
449 * @pd: policy private data of interest
450 * @off: offset to the blkg_stat in @pd
451 *
452 * prfill callback for printing a blkg_stat.
453 */
454u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd, int off)
455{
456	return __blkg_prfill_u64(sf, pd, blkg_stat_read((void *)pd + off));
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
457}
458EXPORT_SYMBOL_GPL(blkg_prfill_stat);
459
460/**
461 * blkg_prfill_rwstat - prfill callback for blkg_rwstat
462 * @sf: seq_file to print to
463 * @pd: policy private data of interest
464 * @off: offset to the blkg_rwstat in @pd
465 *
466 * prfill callback for printing a blkg_rwstat.
467 */
468u64 blkg_prfill_rwstat(struct seq_file *sf, struct blkg_policy_data *pd,
469		       int off)
470{
471	struct blkg_rwstat rwstat = blkg_rwstat_read((void *)pd + off);
 
472
473	return __blkg_prfill_rwstat(sf, pd, &rwstat);
 
 
 
 
474}
475EXPORT_SYMBOL_GPL(blkg_prfill_rwstat);
476
477/**
478 * blkg_conf_prep - parse and prepare for per-blkg config update
479 * @blkcg: target block cgroup
480 * @pol: target policy
481 * @input: input string
482 * @ctx: blkg_conf_ctx to be filled
483 *
484 * Parse per-blkg config update from @input and initialize @ctx with the
485 * result.  @ctx->blkg points to the blkg to be updated and @ctx->v the new
486 * value.  This function returns with RCU read lock and queue lock held and
487 * must be paired with blkg_conf_finish().
488 */
489int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
490		   const char *input, struct blkg_conf_ctx *ctx)
491	__acquires(rcu) __acquires(disk->queue->queue_lock)
492{
493	struct gendisk *disk;
494	struct blkcg_gq *blkg;
495	unsigned int major, minor;
496	unsigned long long v;
497	int part, ret;
 
498
499	if (sscanf(input, "%u:%u %llu", &major, &minor, &v) != 3)
 
 
 
 
 
 
 
 
 
 
 
 
 
500		return -EINVAL;
501
502	disk = get_gendisk(MKDEV(major, minor), &part);
503	if (!disk || part)
 
 
504		return -EINVAL;
505
506	rcu_read_lock();
507	spin_lock_irq(disk->queue->queue_lock);
 
508
509	if (blkcg_policy_enabled(disk->queue, pol))
510		blkg = blkg_lookup_create(blkcg, disk->queue);
511	else
512		blkg = ERR_PTR(-EINVAL);
513
514	if (IS_ERR(blkg)) {
515		ret = PTR_ERR(blkg);
516		rcu_read_unlock();
517		spin_unlock_irq(disk->queue->queue_lock);
518		put_disk(disk);
519		/*
520		 * If queue was bypassing, we should retry.  Do so after a
521		 * short msleep().  It isn't strictly necessary but queue
522		 * can be bypassing for some time and it's always nice to
523		 * avoid busy looping.
524		 */
525		if (ret == -EBUSY) {
526			msleep(10);
527			ret = restart_syscall();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
528		}
529		return ret;
 
 
530	}
531
532	ctx->disk = disk;
533	ctx->blkg = blkg;
534	ctx->v = v;
535	return 0;
536}
537EXPORT_SYMBOL_GPL(blkg_conf_prep);
538
539/**
540 * blkg_conf_finish - finish up per-blkg config update
541 * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
542 *
543 * Finish up after per-blkg config update.  This function must be paired
544 * with blkg_conf_prep().
545 */
546void blkg_conf_finish(struct blkg_conf_ctx *ctx)
547	__releases(ctx->disk->queue->queue_lock) __releases(rcu)
548{
549	spin_unlock_irq(ctx->disk->queue->queue_lock);
550	rcu_read_unlock();
551	put_disk(ctx->disk);
552}
553EXPORT_SYMBOL_GPL(blkg_conf_finish);
554
555struct cftype blkcg_files[] = {
556	{
557		.name = "reset_stats",
558		.write_u64 = blkcg_reset_stats,
559	},
560	{ }	/* terminate */
561};
 
562
563/**
564 * blkcg_pre_destroy - cgroup pre_destroy callback
565 * @cgroup: cgroup of interest
566 *
567 * This function is called when @cgroup is about to go away and responsible
568 * for shooting down all blkgs associated with @cgroup.  blkgs should be
569 * removed while holding both q and blkcg locks.  As blkcg lock is nested
570 * inside q lock, this function performs reverse double lock dancing.
571 *
572 * This is the blkcg counterpart of ioc_release_fn().
573 */
574static int blkcg_pre_destroy(struct cgroup *cgroup)
575{
576	struct blkcg *blkcg = cgroup_to_blkcg(cgroup);
577
578	spin_lock_irq(&blkcg->lock);
579
580	while (!hlist_empty(&blkcg->blkg_list)) {
581		struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
582						struct blkcg_gq, blkcg_node);
583		struct request_queue *q = blkg->q;
584
585		if (spin_trylock(q->queue_lock)) {
586			blkg_destroy(blkg);
587			spin_unlock(q->queue_lock);
588		} else {
589			spin_unlock_irq(&blkcg->lock);
590			cpu_relax();
591			spin_lock_irq(&blkcg->lock);
592		}
593	}
594
595	spin_unlock_irq(&blkcg->lock);
596	return 0;
 
 
 
 
 
 
 
597}
598
599static void blkcg_destroy(struct cgroup *cgroup)
600{
601	struct blkcg *blkcg = cgroup_to_blkcg(cgroup);
602
603	if (blkcg != &blkcg_root)
604		kfree(blkcg);
 
 
 
 
605}
606
607static struct cgroup_subsys_state *blkcg_create(struct cgroup *cgroup)
608{
609	static atomic64_t id_seq = ATOMIC64_INIT(0);
610	struct blkcg *blkcg;
611	struct cgroup *parent = cgroup->parent;
 
 
 
 
 
612
613	if (!parent) {
614		blkcg = &blkcg_root;
615		goto done;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
616	}
617
618	blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
619	if (!blkcg)
620		return ERR_PTR(-ENOMEM);
621
622	blkcg->cfq_weight = CFQ_WEIGHT_DEFAULT;
623	blkcg->id = atomic64_inc_return(&id_seq); /* root is 0, start from 1 */
624done:
625	spin_lock_init(&blkcg->lock);
626	INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_ATOMIC);
627	INIT_HLIST_HEAD(&blkcg->blkg_list);
628
629	return &blkcg->css;
630}
631
632/**
633 * blkcg_init_queue - initialize blkcg part of request queue
634 * @q: request_queue to initialize
635 *
636 * Called from blk_alloc_queue_node(). Responsible for initializing blkcg
637 * part of new request_queue @q.
638 *
639 * RETURNS:
640 * 0 on success, -errno on failure.
641 */
642int blkcg_init_queue(struct request_queue *q)
643{
644	might_sleep();
645
646	return blk_throtl_init(q);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
647}
648
649/**
650 * blkcg_drain_queue - drain blkcg part of request_queue
651 * @q: request_queue to drain
652 *
653 * Called from blk_drain_queue().  Responsible for draining blkcg part.
654 */
655void blkcg_drain_queue(struct request_queue *q)
 
656{
657	lockdep_assert_held(q->queue_lock);
 
658
659	blk_throtl_drain(q);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
660}
661
662/**
663 * blkcg_exit_queue - exit and release blkcg part of request_queue
664 * @q: request_queue being released
665 *
666 * Called from blk_release_queue().  Responsible for exiting blkcg part.
667 */
668void blkcg_exit_queue(struct request_queue *q)
 
669{
670	spin_lock_irq(q->queue_lock);
671	blkg_destroy_all(q);
672	spin_unlock_irq(q->queue_lock);
673
674	blk_throtl_exit(q);
 
 
 
 
 
 
 
 
 
 
675}
676
677/*
678 * We cannot support shared io contexts, as we have no mean to support
679 * two tasks with the same ioc in two different groups without major rework
680 * of the main cic data structures.  For now we allow a task to change
681 * its cgroup only if it's the only owner of its ioc.
682 */
683static int blkcg_can_attach(struct cgroup *cgrp, struct cgroup_taskset *tset)
684{
685	struct task_struct *task;
686	struct io_context *ioc;
687	int ret = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
688
689	/* task_lock() is needed to avoid races with exit_io_context() */
690	cgroup_taskset_for_each(task, cgrp, tset) {
691		task_lock(task);
692		ioc = task->io_context;
693		if (ioc && atomic_read(&ioc->nr_tasks) > 1)
694			ret = -EINVAL;
695		task_unlock(task);
696		if (ret)
697			break;
 
 
 
 
 
 
 
 
 
 
 
 
 
698	}
 
 
 
 
 
 
 
 
 
 
 
 
699	return ret;
700}
701
702struct cgroup_subsys blkio_subsys = {
703	.name = "blkio",
704	.create = blkcg_create,
705	.can_attach = blkcg_can_attach,
706	.pre_destroy = blkcg_pre_destroy,
707	.destroy = blkcg_destroy,
708	.subsys_id = blkio_subsys_id,
709	.base_cftypes = blkcg_files,
710	.module = THIS_MODULE,
711};
712EXPORT_SYMBOL_GPL(blkio_subsys);
713
714/**
715 * blkcg_activate_policy - activate a blkcg policy on a request_queue
716 * @q: request_queue of interest
717 * @pol: blkcg policy to activate
718 *
719 * Activate @pol on @q.  Requires %GFP_KERNEL context.  @q goes through
720 * bypass mode to populate its blkgs with policy_data for @pol.
721 *
722 * Activation happens with @q bypassed, so nobody would be accessing blkgs
723 * from IO path.  Update of each blkg is protected by both queue and blkcg
724 * locks so that holding either lock and testing blkcg_policy_enabled() is
725 * always enough for dereferencing policy data.
726 *
727 * The caller is responsible for synchronizing [de]activations and policy
728 * [un]registerations.  Returns 0 on success, -errno on failure.
729 */
730int blkcg_activate_policy(struct request_queue *q,
731			  const struct blkcg_policy *pol)
732{
733	LIST_HEAD(pds);
734	struct blkcg_gq *blkg;
735	struct blkg_policy_data *pd, *n;
736	int cnt = 0, ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
737
738	if (blkcg_policy_enabled(q, pol))
739		return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
740
741	blk_queue_bypass_start(q);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
742
743	/* make sure the root blkg exists and count the existing blkgs */
744	spin_lock_irq(q->queue_lock);
 
 
 
 
 
 
 
 
745
746	rcu_read_lock();
747	blkg = __blkg_lookup_create(&blkcg_root, q);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
748	rcu_read_unlock();
 
 
749
750	if (IS_ERR(blkg)) {
751		ret = PTR_ERR(blkg);
752		goto out_unlock;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
753	}
754	q->root_blkg = blkg;
755
756	list_for_each_entry(blkg, &q->blkg_list, q_node)
757		cnt++;
 
 
 
 
 
 
758
759	spin_unlock_irq(q->queue_lock);
 
 
 
 
 
760
761	/* allocate policy_data for all existing blkgs */
762	while (cnt--) {
763		pd = kzalloc_node(pol->pd_size, GFP_KERNEL, q->node);
764		if (!pd) {
765			ret = -ENOMEM;
766			goto out_free;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
767		}
768		list_add_tail(&pd->alloc_node, &pds);
 
 
769	}
 
 
770
771	/*
772	 * Install the allocated pds.  With @q bypassing, no new blkg
773	 * should have been created while the queue lock was dropped.
774	 */
775	spin_lock_irq(q->queue_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
776
777	list_for_each_entry(blkg, &q->blkg_list, q_node) {
778		if (WARN_ON(list_empty(&pds))) {
779			/* umm... this shouldn't happen, just abort */
780			ret = -ENOMEM;
781			goto out_unlock;
782		}
783		pd = list_first_entry(&pds, struct blkg_policy_data, alloc_node);
784		list_del_init(&pd->alloc_node);
785
786		/* grab blkcg lock too while installing @pd on @blkg */
787		spin_lock(&blkg->blkcg->lock);
788
789		blkg->pd[pol->plid] = pd;
790		pd->blkg = blkg;
791		pol->pd_init_fn(blkg);
792
793		spin_unlock(&blkg->blkcg->lock);
794	}
795
796	__set_bit(pol->plid, q->blkcg_pols);
797	ret = 0;
798out_unlock:
799	spin_unlock_irq(q->queue_lock);
800out_free:
801	blk_queue_bypass_end(q);
802	list_for_each_entry_safe(pd, n, &pds, alloc_node)
803		kfree(pd);
804	return ret;
805}
806EXPORT_SYMBOL_GPL(blkcg_activate_policy);
807
808/**
809 * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
810 * @q: request_queue of interest
811 * @pol: blkcg policy to deactivate
812 *
813 * Deactivate @pol on @q.  Follows the same synchronization rules as
814 * blkcg_activate_policy().
815 */
816void blkcg_deactivate_policy(struct request_queue *q,
817			     const struct blkcg_policy *pol)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
818{
819	struct blkcg_gq *blkg;
 
 
820
821	if (!blkcg_policy_enabled(q, pol))
822		return;
 
 
 
 
 
 
823
824	blk_queue_bypass_start(q);
825	spin_lock_irq(q->queue_lock);
 
826
827	__clear_bit(pol->plid, q->blkcg_pols);
 
 
 
828
829	/* if no policy is left, no need for blkgs - shoot them down */
830	if (bitmap_empty(q->blkcg_pols, BLKCG_MAX_POLS))
831		blkg_destroy_all(q);
 
832
833	list_for_each_entry(blkg, &q->blkg_list, q_node) {
834		/* grab blkcg lock too while removing @pd from @blkg */
835		spin_lock(&blkg->blkcg->lock);
836
837		if (pol->pd_exit_fn)
838			pol->pd_exit_fn(blkg);
 
 
 
 
 
 
 
 
 
 
 
839
840		kfree(blkg->pd[pol->plid]);
841		blkg->pd[pol->plid] = NULL;
 
 
842
843		spin_unlock(&blkg->blkcg->lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
844	}
845
846	spin_unlock_irq(q->queue_lock);
847	blk_queue_bypass_end(q);
 
 
 
 
 
 
 
 
 
848}
849EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
850
851/**
852 * blkcg_policy_register - register a blkcg policy
853 * @pol: blkcg policy to register
854 *
855 * Register @pol with blkcg core.  Might sleep and @pol may be modified on
856 * successful registration.  Returns 0 on success and -errno on failure.
857 */
858int blkcg_policy_register(struct blkcg_policy *pol)
859{
860	int i, ret;
 
861
862	if (WARN_ON(pol->pd_size < sizeof(struct blkg_policy_data)))
863		return -EINVAL;
 
 
 
 
864
865	mutex_lock(&blkcg_pol_mutex);
 
866
867	/* find an empty slot */
868	ret = -ENOSPC;
869	for (i = 0; i < BLKCG_MAX_POLS; i++)
870		if (!blkcg_policy[i])
871			break;
872	if (i >= BLKCG_MAX_POLS)
873		goto out_unlock;
874
875	/* register and update blkgs */
876	pol->plid = i;
877	blkcg_policy[i] = pol;
878
879	/* everything is in place, add intf files for the new policy */
880	if (pol->cftypes)
881		WARN_ON(cgroup_add_cftypes(&blkio_subsys, pol->cftypes));
882	ret = 0;
883out_unlock:
884	mutex_unlock(&blkcg_pol_mutex);
885	return ret;
886}
887EXPORT_SYMBOL_GPL(blkcg_policy_register);
888
889/**
890 * blkcg_policy_unregister - unregister a blkcg policy
891 * @pol: blkcg policy to unregister
892 *
893 * Undo blkcg_policy_register(@pol).  Might sleep.
894 */
895void blkcg_policy_unregister(struct blkcg_policy *pol)
896{
897	mutex_lock(&blkcg_pol_mutex);
 
 
 
 
898
899	if (WARN_ON(blkcg_policy[pol->plid] != pol))
900		goto out_unlock;
 
 
 
 
 
901
902	/* kill the intf files first */
903	if (pol->cftypes)
904		cgroup_rm_cftypes(&blkio_subsys, pol->cftypes);
 
905
906	/* unregister and update blkgs */
907	blkcg_policy[pol->plid] = NULL;
908out_unlock:
909	mutex_unlock(&blkcg_pol_mutex);
910}
911EXPORT_SYMBOL_GPL(blkcg_policy_unregister);