1/* Kernel thread helper functions.
   2 *   Copyright (C) 2004 IBM Corporation, Rusty Russell.
   3 *
   4 * Creation is done via kthreadd, so that we get a clean environment
   5 * even if we're invoked from userspace (think modprobe, hotplug cpu,
   6 * etc.).
   7 */
   8#include <uapi/linux/sched/types.h>
   9#include <linux/sched.h>
  10#include <linux/sched/task.h>
  11#include <linux/kthread.h>
  12#include <linux/completion.h>
  13#include <linux/err.h>
  14#include <linux/cpuset.h>
  15#include <linux/unistd.h>
  16#include <linux/file.h>
  17#include <linux/export.h>
  18#include <linux/mutex.h>
  19#include <linux/slab.h>
  20#include <linux/freezer.h>
  21#include <linux/ptrace.h>
  22#include <linux/uaccess.h>
  23#include <trace/events/sched.h>
  24
  25static DEFINE_SPINLOCK(kthread_create_lock);
  26static LIST_HEAD(kthread_create_list);
  27struct task_struct *kthreadd_task;
  28
  29struct kthread_create_info
  30{
  31	/* Information passed to kthread() from kthreadd. */
  32	int (*threadfn)(void *data);
  33	void *data;
  34	int node;
  35
  36	/* Result passed back to kthread_create() from kthreadd. */
  37	struct task_struct *result;
  38	struct completion *done;
  39
  40	struct list_head list;
  41};
  42
  43struct kthread {
  44	unsigned long flags;
  45	unsigned int cpu;
  46	void *data;
  47	struct completion parked;
  48	struct completion exited;
  49#ifdef CONFIG_BLK_CGROUP
  50	struct cgroup_subsys_state *blkcg_css;
  51#endif
  52};
  53
  54enum KTHREAD_BITS {
  55	KTHREAD_IS_PER_CPU = 0,
  56	KTHREAD_SHOULD_STOP,
  57	KTHREAD_SHOULD_PARK,
  58};
  59
  60static inline void set_kthread_struct(void *kthread)
  61{
  62	/*
  63	 * We abuse ->set_child_tid to avoid the new member and because it
  64	 * can't be wrongly copied by copy_process(). We also rely on fact
  65	 * that the caller can't exec, so PF_KTHREAD can't be cleared.
  66	 */
  67	current->set_child_tid = (__force void __user *)kthread;
  68}
  69
  70static inline struct kthread *to_kthread(struct task_struct *k)
  71{
  72	WARN_ON(!(k->flags & PF_KTHREAD));
  73	return (__force void *)k->set_child_tid;
  74}
  75
  76void free_kthread_struct(struct task_struct *k)
  77{
  78	struct kthread *kthread;
  79
  80	/*
  81	 * Can be NULL if this kthread was created by kernel_thread()
  82	 * or if kmalloc() in kthread() failed.
  83	 */
  84	kthread = to_kthread(k);
  85#ifdef CONFIG_BLK_CGROUP
  86	WARN_ON_ONCE(kthread && kthread->blkcg_css);
  87#endif
  88	kfree(kthread);
  89}
  90
  91/**
  92 * kthread_should_stop - should this kthread return now?
  93 *
  94 * When someone calls kthread_stop() on your kthread, it will be woken
  95 * and this will return true.  You should then return, and your return
  96 * value will be passed through to kthread_stop().
  97 */
  98bool kthread_should_stop(void)
  99{
 100	return test_bit(KTHREAD_SHOULD_STOP, &to_kthread(current)->flags);
 101}
 102EXPORT_SYMBOL(kthread_should_stop);
 103
 104/**
 105 * kthread_should_park - should this kthread park now?
 106 *
 107 * When someone calls kthread_park() on your kthread, it will be woken
 108 * and this will return true.  You should then do the necessary
 109 * cleanup and call kthread_parkme()
 110 *
 111 * Similar to kthread_should_stop(), but this keeps the thread alive
 112 * and in a park position. kthread_unpark() "restarts" the thread and
 113 * calls the thread function again.
 114 */
 115bool kthread_should_park(void)
 116{
 117	return test_bit(KTHREAD_SHOULD_PARK, &to_kthread(current)->flags);
 118}
 119EXPORT_SYMBOL_GPL(kthread_should_park);
 120
 121/**
 122 * kthread_freezable_should_stop - should this freezable kthread return now?
 123 * @was_frozen: optional out parameter, indicates whether %current was frozen
 124 *
 125 * kthread_should_stop() for freezable kthreads, which will enter
 126 * refrigerator if necessary.  This function is safe from kthread_stop() /
 127 * freezer deadlock and freezable kthreads should use this function instead
 128 * of calling try_to_freeze() directly.
 129 */
 130bool kthread_freezable_should_stop(bool *was_frozen)
 131{
 132	bool frozen = false;
 133
 134	might_sleep();
 135
 136	if (unlikely(freezing(current)))
 137		frozen = __refrigerator(true);
 138
 139	if (was_frozen)
 140		*was_frozen = frozen;
 141
 142	return kthread_should_stop();
 143}
 144EXPORT_SYMBOL_GPL(kthread_freezable_should_stop);
 145
 146/**
 147 * kthread_data - return data value specified on kthread creation
 148 * @task: kthread task in question
 149 *
 150 * Return the data value specified when kthread @task was created.
 151 * The caller is responsible for ensuring the validity of @task when
 152 * calling this function.
 153 */
 154void *kthread_data(struct task_struct *task)
 155{
 156	return to_kthread(task)->data;
 157}
 158
 159/**
 160 * kthread_probe_data - speculative version of kthread_data()
 161 * @task: possible kthread task in question
 162 *
 163 * @task could be a kthread task.  Return the data value specified when it
 164 * was created if accessible.  If @task isn't a kthread task or its data is
 165 * inaccessible for any reason, %NULL is returned.  This function requires
 166 * that @task itself is safe to dereference.
 167 */
 168void *kthread_probe_data(struct task_struct *task)
 169{
 170	struct kthread *kthread = to_kthread(task);
 171	void *data = NULL;
 172
 173	probe_kernel_read(&data, &kthread->data, sizeof(data));
 174	return data;
 175}
 176
 177static void __kthread_parkme(struct kthread *self)
 178{
 179	for (;;) {
 180		set_current_state(TASK_PARKED);
 181		if (!test_bit(KTHREAD_SHOULD_PARK, &self->flags))
 182			break;
 183		schedule();
 184	}
 185	__set_current_state(TASK_RUNNING);
 186}
 187
 188void kthread_parkme(void)
 189{
 190	__kthread_parkme(to_kthread(current));
 191}
 192EXPORT_SYMBOL_GPL(kthread_parkme);
 193
 194void kthread_park_complete(struct task_struct *k)
 195{
 196	complete_all(&to_kthread(k)->parked);
 197}
 198
 199static int kthread(void *_create)
 200{
 201	/* Copy data: it's on kthread's stack */
 202	struct kthread_create_info *create = _create;
 203	int (*threadfn)(void *data) = create->threadfn;
 204	void *data = create->data;
 205	struct completion *done;
 206	struct kthread *self;
 207	int ret;
 208
 209	self = kzalloc(sizeof(*self), GFP_KERNEL);
 210	set_kthread_struct(self);
 211
 212	/* If user was SIGKILLed, I release the structure. */
 213	done = xchg(&create->done, NULL);
 214	if (!done) {
 215		kfree(create);
 216		do_exit(-EINTR);
 217	}
 218
 219	if (!self) {
 220		create->result = ERR_PTR(-ENOMEM);
 221		complete(done);
 222		do_exit(-ENOMEM);
 223	}
 224
 225	self->data = data;
 226	init_completion(&self->exited);
 227	init_completion(&self->parked);
 228	current->vfork_done = &self->exited;
 229
 230	/* OK, tell user we're spawned, wait for stop or wakeup */
 231	__set_current_state(TASK_UNINTERRUPTIBLE);
 232	create->result = current;
 233	complete(done);
 234	schedule();
 235
 236	ret = -EINTR;
 237	if (!test_bit(KTHREAD_SHOULD_STOP, &self->flags)) {
 238		cgroup_kthread_ready();
 239		__kthread_parkme(self);
 240		ret = threadfn(data);
 241	}
 
 242	do_exit(ret);
 243}
 244
 245/* called from do_fork() to get node information for about to be created task */
 246int tsk_fork_get_node(struct task_struct *tsk)
 247{
 248#ifdef CONFIG_NUMA
 249	if (tsk == kthreadd_task)
 250		return tsk->pref_node_fork;
 251#endif
 252	return NUMA_NO_NODE;
 253}
 254
 255static void create_kthread(struct kthread_create_info *create)
 256{
 257	int pid;
 258
 259#ifdef CONFIG_NUMA
 260	current->pref_node_fork = create->node;
 261#endif
 262	/* We want our own signal handler (we take no signals by default). */
 263	pid = kernel_thread(kthread, create, CLONE_FS | CLONE_FILES | SIGCHLD);
 264	if (pid < 0) {
 265		/* If user was SIGKILLed, I release the structure. */
 266		struct completion *done = xchg(&create->done, NULL);
 267
 268		if (!done) {
 269			kfree(create);
 270			return;
 271		}
 272		create->result = ERR_PTR(pid);
 273		complete(done);
 274	}
 275}
 276
 277static __printf(4, 0)
 278struct task_struct *__kthread_create_on_node(int (*threadfn)(void *data),
 279						    void *data, int node,
 280						    const char namefmt[],
 281						    va_list args)
 282{
 283	DECLARE_COMPLETION_ONSTACK(done);
 284	struct task_struct *task;
 285	struct kthread_create_info *create = kmalloc(sizeof(*create),
 286						     GFP_KERNEL);
 287
 288	if (!create)
 289		return ERR_PTR(-ENOMEM);
 290	create->threadfn = threadfn;
 291	create->data = data;
 292	create->node = node;
 293	create->done = &done;
 294
 295	spin_lock(&kthread_create_lock);
 296	list_add_tail(&create->list, &kthread_create_list);
 297	spin_unlock(&kthread_create_lock);
 298
 299	wake_up_process(kthreadd_task);
 300	/*
 301	 * Wait for completion in killable state, for I might be chosen by
 302	 * the OOM killer while kthreadd is trying to allocate memory for
 303	 * new kernel thread.
 304	 */
 305	if (unlikely(wait_for_completion_killable(&done))) {
 306		/*
 307		 * If I was SIGKILLed before kthreadd (or new kernel thread)
 308		 * calls complete(), leave the cleanup of this structure to
 309		 * that thread.
 310		 */
 311		if (xchg(&create->done, NULL))
 312			return ERR_PTR(-EINTR);
 313		/*
 314		 * kthreadd (or new kernel thread) will call complete()
 315		 * shortly.
 316		 */
 317		wait_for_completion(&done);
 318	}
 319	task = create->result;
 320	if (!IS_ERR(task)) {
 321		static const struct sched_param param = { .sched_priority = 0 };
 322
 323		vsnprintf(task->comm, sizeof(task->comm), namefmt, args);
 324		/*
 325		 * root may have changed our (kthreadd's) priority or CPU mask.
 326		 * The kernel thread should not inherit these properties.
 327		 */
 328		sched_setscheduler_nocheck(task, SCHED_NORMAL, &param);
 329		set_cpus_allowed_ptr(task, cpu_all_mask);
 330	}
 331	kfree(create);
 332	return task;
 333}
 334
 335/**
 336 * kthread_create_on_node - create a kthread.
 337 * @threadfn: the function to run until signal_pending(current).
 338 * @data: data ptr for @threadfn.
 339 * @node: task and thread structures for the thread are allocated on this node
 340 * @namefmt: printf-style name for the thread.
 341 *
 342 * Description: This helper function creates and names a kernel
 343 * thread.  The thread will be stopped: use wake_up_process() to start
 344 * it.  See also kthread_run().  The new thread has SCHED_NORMAL policy and
 345 * is affine to all CPUs.
 346 *
 347 * If thread is going to be bound on a particular cpu, give its node
 348 * in @node, to get NUMA affinity for kthread stack, or else give NUMA_NO_NODE.
 349 * When woken, the thread will run @threadfn() with @data as its
 350 * argument. @threadfn() can either call do_exit() directly if it is a
 351 * standalone thread for which no one will call kthread_stop(), or
 352 * return when 'kthread_should_stop()' is true (which means
 353 * kthread_stop() has been called).  The return value should be zero
 354 * or a negative error number; it will be passed to kthread_stop().
 355 *
 356 * Returns a task_struct or ERR_PTR(-ENOMEM) or ERR_PTR(-EINTR).
 357 */
 358struct task_struct *kthread_create_on_node(int (*threadfn)(void *data),
 359					   void *data, int node,
 
 360					   const char namefmt[],
 361					   ...)
 362{
 363	struct task_struct *task;
 364	va_list args;
 365
 366	va_start(args, namefmt);
 367	task = __kthread_create_on_node(threadfn, data, node, namefmt, args);
 368	va_end(args);
 
 369
 370	return task;
 371}
 372EXPORT_SYMBOL(kthread_create_on_node);
 373
 374static void __kthread_bind_mask(struct task_struct *p, const struct cpumask *mask, long state)
 375{
 376	unsigned long flags;
 377
 378	if (!wait_task_inactive(p, state)) {
 379		WARN_ON(1);
 380		return;
 381	}
 382
 383	/* It's safe because the task is inactive. */
 384	raw_spin_lock_irqsave(&p->pi_lock, flags);
 385	do_set_cpus_allowed(p, mask);
 386	p->flags |= PF_NO_SETAFFINITY;
 387	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
 388}
 389
 390static void __kthread_bind(struct task_struct *p, unsigned int cpu, long state)
 391{
 392	__kthread_bind_mask(p, cpumask_of(cpu), state);
 393}
 394
 395void kthread_bind_mask(struct task_struct *p, const struct cpumask *mask)
 396{
 397	__kthread_bind_mask(p, mask, TASK_UNINTERRUPTIBLE);
 
 
 
 
 
 
 
 
 
 398}
 
 399
 400/**
 401 * kthread_bind - bind a just-created kthread to a cpu.
 402 * @p: thread created by kthread_create().
 403 * @cpu: cpu (might not be online, must be possible) for @k to run on.
 404 *
 405 * Description: This function is equivalent to set_cpus_allowed(),
 406 * except that @cpu doesn't need to be online, and the thread must be
 407 * stopped (i.e., just returned from kthread_create()).
 408 */
 409void kthread_bind(struct task_struct *p, unsigned int cpu)
 410{
 411	__kthread_bind(p, cpu, TASK_UNINTERRUPTIBLE);
 412}
 413EXPORT_SYMBOL(kthread_bind);
 414
 415/**
 416 * kthread_create_on_cpu - Create a cpu bound kthread
 417 * @threadfn: the function to run until signal_pending(current).
 418 * @data: data ptr for @threadfn.
 419 * @cpu: The cpu on which the thread should be bound,
 420 * @namefmt: printf-style name for the thread. Format is restricted
 421 *	     to "name.*%u". Code fills in cpu number.
 422 *
 423 * Description: This helper function creates and names a kernel thread
 424 * The thread will be woken and put into park mode.
 425 */
 426struct task_struct *kthread_create_on_cpu(int (*threadfn)(void *data),
 427					  void *data, unsigned int cpu,
 428					  const char *namefmt)
 429{
 430	struct task_struct *p;
 431
 432	p = kthread_create_on_node(threadfn, data, cpu_to_node(cpu), namefmt,
 433				   cpu);
 434	if (IS_ERR(p))
 435		return p;
 436	kthread_bind(p, cpu);
 437	/* CPU hotplug need to bind once again when unparking the thread. */
 438	set_bit(KTHREAD_IS_PER_CPU, &to_kthread(p)->flags);
 439	to_kthread(p)->cpu = cpu;
 440	return p;
 441}
 442
 443/**
 444 * kthread_unpark - unpark a thread created by kthread_create().
 445 * @k:		thread created by kthread_create().
 446 *
 447 * Sets kthread_should_park() for @k to return false, wakes it, and
 448 * waits for it to return. If the thread is marked percpu then its
 449 * bound to the cpu again.
 450 */
 451void kthread_unpark(struct task_struct *k)
 452{
 453	struct kthread *kthread = to_kthread(k);
 454
 455	/*
 456	 * Newly created kthread was parked when the CPU was offline.
 457	 * The binding was lost and we need to set it again.
 458	 */
 459	if (test_bit(KTHREAD_IS_PER_CPU, &kthread->flags))
 460		__kthread_bind(k, kthread->cpu, TASK_PARKED);
 461
 462	reinit_completion(&kthread->parked);
 463	clear_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
 464	wake_up_state(k, TASK_PARKED);
 465}
 466EXPORT_SYMBOL_GPL(kthread_unpark);
 467
 468/**
 469 * kthread_park - park a thread created by kthread_create().
 470 * @k: thread created by kthread_create().
 471 *
 472 * Sets kthread_should_park() for @k to return true, wakes it, and
 473 * waits for it to return. This can also be called after kthread_create()
 474 * instead of calling wake_up_process(): the thread will park without
 475 * calling threadfn().
 476 *
 477 * Returns 0 if the thread is parked, -ENOSYS if the thread exited.
 478 * If called by the kthread itself just the park bit is set.
 479 */
 480int kthread_park(struct task_struct *k)
 481{
 482	struct kthread *kthread = to_kthread(k);
 483
 484	if (WARN_ON(k->flags & PF_EXITING))
 485		return -ENOSYS;
 486
 487	set_bit(KTHREAD_SHOULD_PARK, &kthread->flags);
 488	if (k != current) {
 489		wake_up_process(k);
 490		wait_for_completion(&kthread->parked);
 491	}
 492
 493	return 0;
 
 
 494}
 495EXPORT_SYMBOL_GPL(kthread_park);
 496
 497/**
 498 * kthread_stop - stop a thread created by kthread_create().
 499 * @k: thread created by kthread_create().
 500 *
 501 * Sets kthread_should_stop() for @k to return true, wakes it, and
 502 * waits for it to exit. This can also be called after kthread_create()
 503 * instead of calling wake_up_process(): the thread will exit without
 504 * calling threadfn().
 505 *
 506 * If threadfn() may call do_exit() itself, the caller must ensure
 507 * task_struct can't go away.
 508 *
 509 * Returns the result of threadfn(), or %-EINTR if wake_up_process()
 510 * was never called.
 511 */
 512int kthread_stop(struct task_struct *k)
 513{
 514	struct kthread *kthread;
 515	int ret;
 516
 517	trace_sched_kthread_stop(k);
 518
 519	get_task_struct(k);
 
 520	kthread = to_kthread(k);
 521	set_bit(KTHREAD_SHOULD_STOP, &kthread->flags);
 522	kthread_unpark(k);
 523	wake_up_process(k);
 524	wait_for_completion(&kthread->exited);
 
 
 525	ret = k->exit_code;
 526	put_task_struct(k);
 527
 
 528	trace_sched_kthread_stop_ret(ret);
 
 529	return ret;
 530}
 531EXPORT_SYMBOL(kthread_stop);
 532
 533int kthreadd(void *unused)
 534{
 535	struct task_struct *tsk = current;
 536
 537	/* Setup a clean context for our children to inherit. */
 538	set_task_comm(tsk, "kthreadd");
 539	ignore_signals(tsk);
 540	set_cpus_allowed_ptr(tsk, cpu_all_mask);
 541	set_mems_allowed(node_states[N_MEMORY]);
 542
 543	current->flags |= PF_NOFREEZE;
 544	cgroup_init_kthreadd();
 545
 546	for (;;) {
 547		set_current_state(TASK_INTERRUPTIBLE);
 548		if (list_empty(&kthread_create_list))
 549			schedule();
 550		__set_current_state(TASK_RUNNING);
 551
 552		spin_lock(&kthread_create_lock);
 553		while (!list_empty(&kthread_create_list)) {
 554			struct kthread_create_info *create;
 555
 556			create = list_entry(kthread_create_list.next,
 557					    struct kthread_create_info, list);
 558			list_del_init(&create->list);
 559			spin_unlock(&kthread_create_lock);
 560
 561			create_kthread(create);
 562
 563			spin_lock(&kthread_create_lock);
 564		}
 565		spin_unlock(&kthread_create_lock);
 566	}
 567
 568	return 0;
 569}
 570
 571void __kthread_init_worker(struct kthread_worker *worker,
 572				const char *name,
 573				struct lock_class_key *key)
 574{
 575	memset(worker, 0, sizeof(struct kthread_worker));
 576	spin_lock_init(&worker->lock);
 577	lockdep_set_class_and_name(&worker->lock, key, name);
 578	INIT_LIST_HEAD(&worker->work_list);
 579	INIT_LIST_HEAD(&worker->delayed_work_list);
 580}
 581EXPORT_SYMBOL_GPL(__kthread_init_worker);
 582
 583/**
 584 * kthread_worker_fn - kthread function to process kthread_worker
 585 * @worker_ptr: pointer to initialized kthread_worker
 586 *
 587 * This function implements the main cycle of kthread worker. It processes
 588 * work_list until it is stopped with kthread_stop(). It sleeps when the queue
 589 * is empty.
 590 *
 591 * The works are not allowed to keep any locks, disable preemption or interrupts
 592 * when they finish. There is defined a safe point for freezing when one work
 593 * finishes and before a new one is started.
 594 *
 595 * Also the works must not be handled by more than one worker at the same time,
 596 * see also kthread_queue_work().
 597 */
 598int kthread_worker_fn(void *worker_ptr)
 599{
 600	struct kthread_worker *worker = worker_ptr;
 601	struct kthread_work *work;
 602
 603	/*
 604	 * FIXME: Update the check and remove the assignment when all kthread
 605	 * worker users are created using kthread_create_worker*() functions.
 606	 */
 607	WARN_ON(worker->task && worker->task != current);
 608	worker->task = current;
 609
 610	if (worker->flags & KTW_FREEZABLE)
 611		set_freezable();
 612
 613repeat:
 614	set_current_state(TASK_INTERRUPTIBLE);	/* mb paired w/ kthread_stop */
 615
 616	if (kthread_should_stop()) {
 617		__set_current_state(TASK_RUNNING);
 618		spin_lock_irq(&worker->lock);
 619		worker->task = NULL;
 620		spin_unlock_irq(&worker->lock);
 621		return 0;
 622	}
 623
 624	work = NULL;
 625	spin_lock_irq(&worker->lock);
 626	if (!list_empty(&worker->work_list)) {
 627		work = list_first_entry(&worker->work_list,
 628					struct kthread_work, node);
 629		list_del_init(&work->node);
 630	}
 631	worker->current_work = work;
 632	spin_unlock_irq(&worker->lock);
 633
 634	if (work) {
 635		__set_current_state(TASK_RUNNING);
 636		work->func(work);
 637	} else if (!freezing(current))
 638		schedule();
 639
 640	try_to_freeze();
 641	cond_resched();
 642	goto repeat;
 643}
 644EXPORT_SYMBOL_GPL(kthread_worker_fn);
 645
 646static __printf(3, 0) struct kthread_worker *
 647__kthread_create_worker(int cpu, unsigned int flags,
 648			const char namefmt[], va_list args)
 649{
 650	struct kthread_worker *worker;
 651	struct task_struct *task;
 652	int node = -1;
 653
 654	worker = kzalloc(sizeof(*worker), GFP_KERNEL);
 655	if (!worker)
 656		return ERR_PTR(-ENOMEM);
 657
 658	kthread_init_worker(worker);
 659
 660	if (cpu >= 0)
 661		node = cpu_to_node(cpu);
 662
 663	task = __kthread_create_on_node(kthread_worker_fn, worker,
 664						node, namefmt, args);
 665	if (IS_ERR(task))
 666		goto fail_task;
 667
 668	if (cpu >= 0)
 669		kthread_bind(task, cpu);
 670
 671	worker->flags = flags;
 672	worker->task = task;
 673	wake_up_process(task);
 674	return worker;
 675
 676fail_task:
 677	kfree(worker);
 678	return ERR_CAST(task);
 679}
 680
 681/**
 682 * kthread_create_worker - create a kthread worker
 683 * @flags: flags modifying the default behavior of the worker
 684 * @namefmt: printf-style name for the kthread worker (task).
 685 *
 686 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
 687 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
 688 * when the worker was SIGKILLed.
 689 */
 690struct kthread_worker *
 691kthread_create_worker(unsigned int flags, const char namefmt[], ...)
 692{
 693	struct kthread_worker *worker;
 694	va_list args;
 695
 696	va_start(args, namefmt);
 697	worker = __kthread_create_worker(-1, flags, namefmt, args);
 698	va_end(args);
 699
 700	return worker;
 701}
 702EXPORT_SYMBOL(kthread_create_worker);
 703
 704/**
 705 * kthread_create_worker_on_cpu - create a kthread worker and bind it
 706 *	it to a given CPU and the associated NUMA node.
 707 * @cpu: CPU number
 708 * @flags: flags modifying the default behavior of the worker
 709 * @namefmt: printf-style name for the kthread worker (task).
 710 *
 711 * Use a valid CPU number if you want to bind the kthread worker
 712 * to the given CPU and the associated NUMA node.
 713 *
 714 * A good practice is to add the cpu number also into the worker name.
 715 * For example, use kthread_create_worker_on_cpu(cpu, "helper/%d", cpu).
 716 *
 717 * Returns a pointer to the allocated worker on success, ERR_PTR(-ENOMEM)
 718 * when the needed structures could not get allocated, and ERR_PTR(-EINTR)
 719 * when the worker was SIGKILLed.
 720 */
 721struct kthread_worker *
 722kthread_create_worker_on_cpu(int cpu, unsigned int flags,
 723			     const char namefmt[], ...)
 724{
 725	struct kthread_worker *worker;
 726	va_list args;
 727
 728	va_start(args, namefmt);
 729	worker = __kthread_create_worker(cpu, flags, namefmt, args);
 730	va_end(args);
 731
 732	return worker;
 733}
 734EXPORT_SYMBOL(kthread_create_worker_on_cpu);
 735
 736/*
 737 * Returns true when the work could not be queued at the moment.
 738 * It happens when it is already pending in a worker list
 739 * or when it is being cancelled.
 740 */
 741static inline bool queuing_blocked(struct kthread_worker *worker,
 742				   struct kthread_work *work)
 743{
 744	lockdep_assert_held(&worker->lock);
 745
 746	return !list_empty(&work->node) || work->canceling;
 747}
 748
 749static void kthread_insert_work_sanity_check(struct kthread_worker *worker,
 750					     struct kthread_work *work)
 751{
 752	lockdep_assert_held(&worker->lock);
 753	WARN_ON_ONCE(!list_empty(&work->node));
 754	/* Do not use a work with >1 worker, see kthread_queue_work() */
 755	WARN_ON_ONCE(work->worker && work->worker != worker);
 756}
 757
 758/* insert @work before @pos in @worker */
 759static void kthread_insert_work(struct kthread_worker *worker,
 760				struct kthread_work *work,
 761				struct list_head *pos)
 762{
 763	kthread_insert_work_sanity_check(worker, work);
 764
 765	list_add_tail(&work->node, pos);
 766	work->worker = worker;
 767	if (!worker->current_work && likely(worker->task))
 768		wake_up_process(worker->task);
 769}
 770
 771/**
 772 * kthread_queue_work - queue a kthread_work
 773 * @worker: target kthread_worker
 774 * @work: kthread_work to queue
 775 *
 776 * Queue @work to work processor @task for async execution.  @task
 777 * must have been created with kthread_worker_create().  Returns %true
 778 * if @work was successfully queued, %false if it was already pending.
 779 *
 780 * Reinitialize the work if it needs to be used by another worker.
 781 * For example, when the worker was stopped and started again.
 782 */
 783bool kthread_queue_work(struct kthread_worker *worker,
 784			struct kthread_work *work)
 785{
 786	bool ret = false;
 787	unsigned long flags;
 788
 789	spin_lock_irqsave(&worker->lock, flags);
 790	if (!queuing_blocked(worker, work)) {
 791		kthread_insert_work(worker, work, &worker->work_list);
 792		ret = true;
 793	}
 794	spin_unlock_irqrestore(&worker->lock, flags);
 795	return ret;
 796}
 797EXPORT_SYMBOL_GPL(kthread_queue_work);
 798
 799/**
 800 * kthread_delayed_work_timer_fn - callback that queues the associated kthread
 801 *	delayed work when the timer expires.
 802 * @t: pointer to the expired timer
 803 *
 804 * The format of the function is defined by struct timer_list.
 805 * It should have been called from irqsafe timer with irq already off.
 806 */
 807void kthread_delayed_work_timer_fn(struct timer_list *t)
 808{
 809	struct kthread_delayed_work *dwork = from_timer(dwork, t, timer);
 810	struct kthread_work *work = &dwork->work;
 811	struct kthread_worker *worker = work->worker;
 812
 813	/*
 814	 * This might happen when a pending work is reinitialized.
 815	 * It means that it is used a wrong way.
 816	 */
 817	if (WARN_ON_ONCE(!worker))
 818		return;
 819
 820	spin_lock(&worker->lock);
 821	/* Work must not be used with >1 worker, see kthread_queue_work(). */
 822	WARN_ON_ONCE(work->worker != worker);
 823
 824	/* Move the work from worker->delayed_work_list. */
 825	WARN_ON_ONCE(list_empty(&work->node));
 826	list_del_init(&work->node);
 827	kthread_insert_work(worker, work, &worker->work_list);
 828
 829	spin_unlock(&worker->lock);
 830}
 831EXPORT_SYMBOL(kthread_delayed_work_timer_fn);
 832
 833void __kthread_queue_delayed_work(struct kthread_worker *worker,
 834				  struct kthread_delayed_work *dwork,
 835				  unsigned long delay)
 836{
 837	struct timer_list *timer = &dwork->timer;
 838	struct kthread_work *work = &dwork->work;
 839
 840	WARN_ON_ONCE(timer->function != kthread_delayed_work_timer_fn);
 841
 842	/*
 843	 * If @delay is 0, queue @dwork->work immediately.  This is for
 844	 * both optimization and correctness.  The earliest @timer can
 845	 * expire is on the closest next tick and delayed_work users depend
 846	 * on that there's no such delay when @delay is 0.
 847	 */
 848	if (!delay) {
 849		kthread_insert_work(worker, work, &worker->work_list);
 850		return;
 851	}
 852
 853	/* Be paranoid and try to detect possible races already now. */
 854	kthread_insert_work_sanity_check(worker, work);
 855
 856	list_add(&work->node, &worker->delayed_work_list);
 857	work->worker = worker;
 858	timer->expires = jiffies + delay;
 859	add_timer(timer);
 860}
 861
 862/**
 863 * kthread_queue_delayed_work - queue the associated kthread work
 864 *	after a delay.
 865 * @worker: target kthread_worker
 866 * @dwork: kthread_delayed_work to queue
 867 * @delay: number of jiffies to wait before queuing
 868 *
 869 * If the work has not been pending it starts a timer that will queue
 870 * the work after the given @delay. If @delay is zero, it queues the
 871 * work immediately.
 872 *
 873 * Return: %false if the @work has already been pending. It means that
 874 * either the timer was running or the work was queued. It returns %true
 875 * otherwise.
 876 */
 877bool kthread_queue_delayed_work(struct kthread_worker *worker,
 878				struct kthread_delayed_work *dwork,
 879				unsigned long delay)
 880{
 881	struct kthread_work *work = &dwork->work;
 882	unsigned long flags;
 883	bool ret = false;
 884
 885	spin_lock_irqsave(&worker->lock, flags);
 886
 887	if (!queuing_blocked(worker, work)) {
 888		__kthread_queue_delayed_work(worker, dwork, delay);
 889		ret = true;
 890	}
 891
 892	spin_unlock_irqrestore(&worker->lock, flags);
 893	return ret;
 894}
 895EXPORT_SYMBOL_GPL(kthread_queue_delayed_work);
 896
 897struct kthread_flush_work {
 898	struct kthread_work	work;
 899	struct completion	done;
 900};
 901
 902static void kthread_flush_work_fn(struct kthread_work *work)
 903{
 904	struct kthread_flush_work *fwork =
 905		container_of(work, struct kthread_flush_work, work);
 906	complete(&fwork->done);
 907}
 908
 909/**
 910 * kthread_flush_work - flush a kthread_work
 911 * @work: work to flush
 912 *
 913 * If @work is queued or executing, wait for it to finish execution.
 914 */
 915void kthread_flush_work(struct kthread_work *work)
 916{
 917	struct kthread_flush_work fwork = {
 918		KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
 919		COMPLETION_INITIALIZER_ONSTACK(fwork.done),
 920	};
 921	struct kthread_worker *worker;
 922	bool noop = false;
 923
 
 924	worker = work->worker;
 925	if (!worker)
 926		return;
 927
 928	spin_lock_irq(&worker->lock);
 929	/* Work must not be used with >1 worker, see kthread_queue_work(). */
 930	WARN_ON_ONCE(work->worker != worker);
 
 
 931
 932	if (!list_empty(&work->node))
 933		kthread_insert_work(worker, &fwork.work, work->node.next);
 934	else if (worker->current_work == work)
 935		kthread_insert_work(worker, &fwork.work,
 936				    worker->work_list.next);
 937	else
 938		noop = true;
 939
 940	spin_unlock_irq(&worker->lock);
 941
 942	if (!noop)
 943		wait_for_completion(&fwork.done);
 944}
 945EXPORT_SYMBOL_GPL(kthread_flush_work);
 946
 947/*
 948 * This function removes the work from the worker queue. Also it makes sure
 949 * that it won't get queued later via the delayed work's timer.
 950 *
 951 * The work might still be in use when this function finishes. See the
 952 * current_work proceed by the worker.
 953 *
 954 * Return: %true if @work was pending and successfully canceled,
 955 *	%false if @work was not pending
 956 */
 957static bool __kthread_cancel_work(struct kthread_work *work, bool is_dwork,
 958				  unsigned long *flags)
 959{
 960	/* Try to cancel the timer if exists. */
 961	if (is_dwork) {
 962		struct kthread_delayed_work *dwork =
 963			container_of(work, struct kthread_delayed_work, work);
 964		struct kthread_worker *worker = work->worker;
 965
 966		/*
 967		 * del_timer_sync() must be called to make sure that the timer
 968		 * callback is not running. The lock must be temporary released
 969		 * to avoid a deadlock with the callback. In the meantime,
 970		 * any queuing is blocked by setting the canceling counter.
 971		 */
 972		work->canceling++;
 973		spin_unlock_irqrestore(&worker->lock, *flags);
 974		del_timer_sync(&dwork->timer);
 975		spin_lock_irqsave(&worker->lock, *flags);
 976		work->canceling--;
 977	}
 978
 979	/*
 980	 * Try to remove the work from a worker list. It might either
 981	 * be from worker->work_list or from worker->delayed_work_list.
 982	 */
 983	if (!list_empty(&work->node)) {
 984		list_del_init(&work->node);
 985		return true;
 986	}
 987
 988	return false;
 989}
 990
 991/**
 992 * kthread_mod_delayed_work - modify delay of or queue a kthread delayed work
 993 * @worker: kthread worker to use
 994 * @dwork: kthread delayed work to queue
 995 * @delay: number of jiffies to wait before queuing
 996 *
 997 * If @dwork is idle, equivalent to kthread_queue_delayed_work(). Otherwise,
 998 * modify @dwork's timer so that it expires after @delay. If @delay is zero,
 999 * @work is guaranteed to be queued immediately.
1000 *
1001 * Return: %true if @dwork was pending and its timer was modified,
1002 * %false otherwise.
1003 *
1004 * A special case is when the work is being canceled in parallel.
1005 * It might be caused either by the real kthread_cancel_delayed_work_sync()
1006 * or yet another kthread_mod_delayed_work() call. We let the other command
1007 * win and return %false here. The caller is supposed to synchronize these
1008 * operations a reasonable way.
1009 *
1010 * This function is safe to call from any context including IRQ handler.
1011 * See __kthread_cancel_work() and kthread_delayed_work_timer_fn()
1012 * for details.
1013 */
1014bool kthread_mod_delayed_work(struct kthread_worker *worker,
1015			      struct kthread_delayed_work *dwork,
1016			      unsigned long delay)
1017{
1018	struct kthread_work *work = &dwork->work;
1019	unsigned long flags;
1020	int ret = false;
1021
1022	spin_lock_irqsave(&worker->lock, flags);
1023
1024	/* Do not bother with canceling when never queued. */
1025	if (!work->worker)
1026		goto fast_queue;
1027
1028	/* Work must not be used with >1 worker, see kthread_queue_work() */
1029	WARN_ON_ONCE(work->worker != worker);
1030
1031	/* Do not fight with another command that is canceling this work. */
1032	if (work->canceling)
1033		goto out;
1034
1035	ret = __kthread_cancel_work(work, true, &flags);
1036fast_queue:
1037	__kthread_queue_delayed_work(worker, dwork, delay);
1038out:
1039	spin_unlock_irqrestore(&worker->lock, flags);
1040	return ret;
1041}
1042EXPORT_SYMBOL_GPL(kthread_mod_delayed_work);
1043
1044static bool __kthread_cancel_work_sync(struct kthread_work *work, bool is_dwork)
1045{
1046	struct kthread_worker *worker = work->worker;
1047	unsigned long flags;
1048	int ret = false;
1049
1050	if (!worker)
1051		goto out;
1052
1053	spin_lock_irqsave(&worker->lock, flags);
1054	/* Work must not be used with >1 worker, see kthread_queue_work(). */
1055	WARN_ON_ONCE(work->worker != worker);
1056
1057	ret = __kthread_cancel_work(work, is_dwork, &flags);
1058
1059	if (worker->current_work != work)
1060		goto out_fast;
1061
1062	/*
1063	 * The work is in progress and we need to wait with the lock released.
1064	 * In the meantime, block any queuing by setting the canceling counter.
1065	 */
1066	work->canceling++;
1067	spin_unlock_irqrestore(&worker->lock, flags);
1068	kthread_flush_work(work);
1069	spin_lock_irqsave(&worker->lock, flags);
1070	work->canceling--;
1071
1072out_fast:
1073	spin_unlock_irqrestore(&worker->lock, flags);
1074out:
1075	return ret;
1076}
1077
1078/**
1079 * kthread_cancel_work_sync - cancel a kthread work and wait for it to finish
1080 * @work: the kthread work to cancel
1081 *
1082 * Cancel @work and wait for its execution to finish.  This function
1083 * can be used even if the work re-queues itself. On return from this
1084 * function, @work is guaranteed to be not pending or executing on any CPU.
1085 *
1086 * kthread_cancel_work_sync(&delayed_work->work) must not be used for
1087 * delayed_work's. Use kthread_cancel_delayed_work_sync() instead.
1088 *
1089 * The caller must ensure that the worker on which @work was last
1090 * queued can't be destroyed before this function returns.
1091 *
1092 * Return: %true if @work was pending, %false otherwise.
1093 */
1094bool kthread_cancel_work_sync(struct kthread_work *work)
1095{
1096	return __kthread_cancel_work_sync(work, false);
1097}
1098EXPORT_SYMBOL_GPL(kthread_cancel_work_sync);
1099
1100/**
1101 * kthread_cancel_delayed_work_sync - cancel a kthread delayed work and
1102 *	wait for it to finish.
1103 * @dwork: the kthread delayed work to cancel
1104 *
1105 * This is kthread_cancel_work_sync() for delayed works.
1106 *
1107 * Return: %true if @dwork was pending, %false otherwise.
1108 */
1109bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *dwork)
1110{
1111	return __kthread_cancel_work_sync(&dwork->work, true);
1112}
1113EXPORT_SYMBOL_GPL(kthread_cancel_delayed_work_sync);
1114
1115/**
1116 * kthread_flush_worker - flush all current works on a kthread_worker
1117 * @worker: worker to flush
1118 *
1119 * Wait until all currently executing or pending works on @worker are
1120 * finished.
1121 */
1122void kthread_flush_worker(struct kthread_worker *worker)
1123{
1124	struct kthread_flush_work fwork = {
1125		KTHREAD_WORK_INIT(fwork.work, kthread_flush_work_fn),
1126		COMPLETION_INITIALIZER_ONSTACK(fwork.done),
1127	};
1128
1129	kthread_queue_work(worker, &fwork.work);
1130	wait_for_completion(&fwork.done);
1131}
1132EXPORT_SYMBOL_GPL(kthread_flush_worker);
1133
1134/**
1135 * kthread_destroy_worker - destroy a kthread worker
1136 * @worker: worker to be destroyed
1137 *
1138 * Flush and destroy @worker.  The simple flush is enough because the kthread
1139 * worker API is used only in trivial scenarios.  There are no multi-step state
1140 * machines needed.
1141 */
1142void kthread_destroy_worker(struct kthread_worker *worker)
1143{
1144	struct task_struct *task;
1145
1146	task = worker->task;
1147	if (WARN_ON(!task))
1148		return;
1149
1150	kthread_flush_worker(worker);
1151	kthread_stop(task);
1152	WARN_ON(!list_empty(&worker->work_list));
1153	kfree(worker);
1154}
1155EXPORT_SYMBOL(kthread_destroy_worker);
1156
1157#ifdef CONFIG_BLK_CGROUP
1158/**
1159 * kthread_associate_blkcg - associate blkcg to current kthread
1160 * @css: the cgroup info
1161 *
1162 * Current thread must be a kthread. The thread is running jobs on behalf of
1163 * other threads. In some cases, we expect the jobs attach cgroup info of
1164 * original threads instead of that of current thread. This function stores
1165 * original thread's cgroup info in current kthread context for later
1166 * retrieval.
1167 */
1168void kthread_associate_blkcg(struct cgroup_subsys_state *css)
1169{
1170	struct kthread *kthread;
1171
1172	if (!(current->flags & PF_KTHREAD))
1173		return;
1174	kthread = to_kthread(current);
1175	if (!kthread)
1176		return;
1177
1178	if (kthread->blkcg_css) {
1179		css_put(kthread->blkcg_css);
1180		kthread->blkcg_css = NULL;
1181	}
1182	if (css) {
1183		css_get(css);
1184		kthread->blkcg_css = css;
1185	}
1186}
1187EXPORT_SYMBOL(kthread_associate_blkcg);
1188
1189/**
1190 * kthread_blkcg - get associated blkcg css of current kthread
1191 *
1192 * Current thread must be a kthread.
1193 */
1194struct cgroup_subsys_state *kthread_blkcg(void)
1195{
1196	struct kthread *kthread;
1197
1198	if (current->flags & PF_KTHREAD) {
1199		kthread = to_kthread(current);
1200		if (kthread)
1201			return kthread->blkcg_css;
1202	}
1203	return NULL;
1204}
1205EXPORT_SYMBOL(kthread_blkcg);
1206#endif