1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 *	linux/kernel/softirq.c
   4 *
   5 *	Copyright (C) 1992 Linus Torvalds
   6 *
   7 *	Rewritten. Old one was good in 2.2, but in 2.3 it was immoral. --ANK (990903)
   8 */
   9
  10#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  11
  12#include <linux/export.h>
  13#include <linux/kernel_stat.h>
  14#include <linux/interrupt.h>
  15#include <linux/init.h>
  16#include <linux/local_lock.h>
  17#include <linux/mm.h>
  18#include <linux/notifier.h>
  19#include <linux/percpu.h>
  20#include <linux/cpu.h>
  21#include <linux/freezer.h>
  22#include <linux/kthread.h>
  23#include <linux/rcupdate.h>
  24#include <linux/ftrace.h>
  25#include <linux/smp.h>
  26#include <linux/smpboot.h>
  27#include <linux/tick.h>
  28#include <linux/irq.h>
  29#include <linux/wait_bit.h>
  30#include <linux/workqueue.h>
  31
  32#include <asm/softirq_stack.h>
  33
  34#define CREATE_TRACE_POINTS
  35#include <trace/events/irq.h>
  36
  37/*
  38   - No shared variables, all the data are CPU local.
  39   - If a softirq needs serialization, let it serialize itself
  40     by its own spinlocks.
  41   - Even if softirq is serialized, only local cpu is marked for
  42     execution. Hence, we get something sort of weak cpu binding.
  43     Though it is still not clear, will it result in better locality
  44     or will not.
  45
  46   Examples:
  47   - NET RX softirq. It is multithreaded and does not require
  48     any global serialization.
  49   - NET TX softirq. It kicks software netdevice queues, hence
  50     it is logically serialized per device, but this serialization
  51     is invisible to common code.
  52   - Tasklets: serialized wrt itself.
  53 */
  54
  55#ifndef __ARCH_IRQ_STAT
  56DEFINE_PER_CPU_ALIGNED(irq_cpustat_t, irq_stat);
  57EXPORT_PER_CPU_SYMBOL(irq_stat);
  58#endif
  59
  60static struct softirq_action softirq_vec[NR_SOFTIRQS] __cacheline_aligned_in_smp;
  61
  62DEFINE_PER_CPU(struct task_struct *, ksoftirqd);
  63
  64const char * const softirq_to_name[NR_SOFTIRQS] = {
  65	"HI", "TIMER", "NET_TX", "NET_RX", "BLOCK", "IRQ_POLL",
  66	"TASKLET", "SCHED", "HRTIMER", "RCU"
  67};
  68
  69/*
  70 * we cannot loop indefinitely here to avoid userspace starvation,
  71 * but we also don't want to introduce a worst case 1/HZ latency
  72 * to the pending events, so lets the scheduler to balance
  73 * the softirq load for us.
  74 */
  75static void wakeup_softirqd(void)
  76{
  77	/* Interrupts are disabled: no need to stop preemption */
  78	struct task_struct *tsk = __this_cpu_read(ksoftirqd);
  79
  80	if (tsk)
  81		wake_up_process(tsk);
  82}
  83
  84#ifdef CONFIG_TRACE_IRQFLAGS
  85DEFINE_PER_CPU(int, hardirqs_enabled);
  86DEFINE_PER_CPU(int, hardirq_context);
  87EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
  88EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
  89#endif
  90
  91/*
  92 * SOFTIRQ_OFFSET usage:
  93 *
  94 * On !RT kernels 'count' is the preempt counter, on RT kernels this applies
  95 * to a per CPU counter and to task::softirqs_disabled_cnt.
  96 *
  97 * - count is changed by SOFTIRQ_OFFSET on entering or leaving softirq
  98 *   processing.
  99 *
 100 * - count is changed by SOFTIRQ_DISABLE_OFFSET (= 2 * SOFTIRQ_OFFSET)
 101 *   on local_bh_disable or local_bh_enable.
 102 *
 103 * This lets us distinguish between whether we are currently processing
 104 * softirq and whether we just have bh disabled.
 105 */
 106#ifdef CONFIG_PREEMPT_RT
 107
 108/*
 109 * RT accounts for BH disabled sections in task::softirqs_disabled_cnt and
 110 * also in per CPU softirq_ctrl::cnt. This is necessary to allow tasks in a
 111 * softirq disabled section to be preempted.
 112 *
 113 * The per task counter is used for softirq_count(), in_softirq() and
 114 * in_serving_softirqs() because these counts are only valid when the task
 115 * holding softirq_ctrl::lock is running.
 116 *
 117 * The per CPU counter prevents pointless wakeups of ksoftirqd in case that
 118 * the task which is in a softirq disabled section is preempted or blocks.
 119 */
 120struct softirq_ctrl {
 121	local_lock_t	lock;
 122	int		cnt;
 123};
 124
 125static DEFINE_PER_CPU(struct softirq_ctrl, softirq_ctrl) = {
 126	.lock	= INIT_LOCAL_LOCK(softirq_ctrl.lock),
 127};
 128
 129/**
 130 * local_bh_blocked() - Check for idle whether BH processing is blocked
 131 *
 132 * Returns false if the per CPU softirq::cnt is 0 otherwise true.
 133 *
 134 * This is invoked from the idle task to guard against false positive
 135 * softirq pending warnings, which would happen when the task which holds
 136 * softirq_ctrl::lock was the only running task on the CPU and blocks on
 137 * some other lock.
 138 */
 139bool local_bh_blocked(void)
 140{
 141	return __this_cpu_read(softirq_ctrl.cnt) != 0;
 142}
 143
 144void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
 145{
 146	unsigned long flags;
 147	int newcnt;
 148
 149	WARN_ON_ONCE(in_hardirq());
 150
 151	/* First entry of a task into a BH disabled section? */
 152	if (!current->softirq_disable_cnt) {
 153		if (preemptible()) {
 154			local_lock(&softirq_ctrl.lock);
 155			/* Required to meet the RCU bottomhalf requirements. */
 156			rcu_read_lock();
 157		} else {
 158			DEBUG_LOCKS_WARN_ON(this_cpu_read(softirq_ctrl.cnt));
 159		}
 160	}
 161
 162	/*
 163	 * Track the per CPU softirq disabled state. On RT this is per CPU
 164	 * state to allow preemption of bottom half disabled sections.
 165	 */
 166	newcnt = __this_cpu_add_return(softirq_ctrl.cnt, cnt);
 167	/*
 168	 * Reflect the result in the task state to prevent recursion on the
 169	 * local lock and to make softirq_count() & al work.
 170	 */
 171	current->softirq_disable_cnt = newcnt;
 172
 173	if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && newcnt == cnt) {
 174		raw_local_irq_save(flags);
 175		lockdep_softirqs_off(ip);
 176		raw_local_irq_restore(flags);
 177	}
 178}
 179EXPORT_SYMBOL(__local_bh_disable_ip);
 180
 181static void __local_bh_enable(unsigned int cnt, bool unlock)
 182{
 183	unsigned long flags;
 184	int newcnt;
 185
 186	DEBUG_LOCKS_WARN_ON(current->softirq_disable_cnt !=
 187			    this_cpu_read(softirq_ctrl.cnt));
 188
 189	if (IS_ENABLED(CONFIG_TRACE_IRQFLAGS) && softirq_count() == cnt) {
 190		raw_local_irq_save(flags);
 191		lockdep_softirqs_on(_RET_IP_);
 192		raw_local_irq_restore(flags);
 193	}
 194
 195	newcnt = __this_cpu_sub_return(softirq_ctrl.cnt, cnt);
 196	current->softirq_disable_cnt = newcnt;
 197
 198	if (!newcnt && unlock) {
 199		rcu_read_unlock();
 200		local_unlock(&softirq_ctrl.lock);
 201	}
 202}
 203
 204void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 205{
 206	bool preempt_on = preemptible();
 207	unsigned long flags;
 208	u32 pending;
 209	int curcnt;
 210
 211	WARN_ON_ONCE(in_hardirq());
 212	lockdep_assert_irqs_enabled();
 213
 214	local_irq_save(flags);
 215	curcnt = __this_cpu_read(softirq_ctrl.cnt);
 216
 217	/*
 218	 * If this is not reenabling soft interrupts, no point in trying to
 219	 * run pending ones.
 220	 */
 221	if (curcnt != cnt)
 222		goto out;
 223
 224	pending = local_softirq_pending();
 225	if (!pending)
 226		goto out;
 227
 228	/*
 229	 * If this was called from non preemptible context, wake up the
 230	 * softirq daemon.
 231	 */
 232	if (!preempt_on) {
 233		wakeup_softirqd();
 234		goto out;
 235	}
 236
 237	/*
 238	 * Adjust softirq count to SOFTIRQ_OFFSET which makes
 239	 * in_serving_softirq() become true.
 240	 */
 241	cnt = SOFTIRQ_OFFSET;
 242	__local_bh_enable(cnt, false);
 243	__do_softirq();
 244
 245out:
 246	__local_bh_enable(cnt, preempt_on);
 247	local_irq_restore(flags);
 248}
 249EXPORT_SYMBOL(__local_bh_enable_ip);
 250
 251/*
 252 * Invoked from ksoftirqd_run() outside of the interrupt disabled section
 253 * to acquire the per CPU local lock for reentrancy protection.
 254 */
 255static inline void ksoftirqd_run_begin(void)
 256{
 257	__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 258	local_irq_disable();
 259}
 260
 261/* Counterpart to ksoftirqd_run_begin() */
 262static inline void ksoftirqd_run_end(void)
 263{
 264	__local_bh_enable(SOFTIRQ_OFFSET, true);
 265	WARN_ON_ONCE(in_interrupt());
 266	local_irq_enable();
 267}
 268
 269static inline void softirq_handle_begin(void) { }
 270static inline void softirq_handle_end(void) { }
 271
 272static inline bool should_wake_ksoftirqd(void)
 273{
 274	return !this_cpu_read(softirq_ctrl.cnt);
 275}
 276
 277static inline void invoke_softirq(void)
 278{
 279	if (should_wake_ksoftirqd())
 280		wakeup_softirqd();
 281}
 282
 283#define SCHED_SOFTIRQ_MASK	BIT(SCHED_SOFTIRQ)
 284
 285/*
 286 * flush_smp_call_function_queue() can raise a soft interrupt in a function
 287 * call. On RT kernels this is undesired and the only known functionalities
 288 * are in the block layer which is disabled on RT, and in the scheduler for
 289 * idle load balancing. If soft interrupts get raised which haven't been
 290 * raised before the flush, warn if it is not a SCHED_SOFTIRQ so it can be
 291 * investigated.
 292 */
 293void do_softirq_post_smp_call_flush(unsigned int was_pending)
 294{
 295	unsigned int is_pending = local_softirq_pending();
 296
 297	if (unlikely(was_pending != is_pending)) {
 298		WARN_ON_ONCE(was_pending != (is_pending & ~SCHED_SOFTIRQ_MASK));
 299		invoke_softirq();
 300	}
 301}
 302
 303#else /* CONFIG_PREEMPT_RT */
 304
 305/*
 306 * This one is for softirq.c-internal use, where hardirqs are disabled
 307 * legitimately:
 308 */
 309#ifdef CONFIG_TRACE_IRQFLAGS
 310void __local_bh_disable_ip(unsigned long ip, unsigned int cnt)
 311{
 312	unsigned long flags;
 313
 314	WARN_ON_ONCE(in_hardirq());
 315
 316	raw_local_irq_save(flags);
 317	/*
 318	 * The preempt tracer hooks into preempt_count_add and will break
 319	 * lockdep because it calls back into lockdep after SOFTIRQ_OFFSET
 320	 * is set and before current->softirq_enabled is cleared.
 321	 * We must manually increment preempt_count here and manually
 322	 * call the trace_preempt_off later.
 323	 */
 324	__preempt_count_add(cnt);
 325	/*
 326	 * Were softirqs turned off above:
 327	 */
 328	if (softirq_count() == (cnt & SOFTIRQ_MASK))
 329		lockdep_softirqs_off(ip);
 330	raw_local_irq_restore(flags);
 331
 332	if (preempt_count() == cnt) {
 333#ifdef CONFIG_DEBUG_PREEMPT
 334		current->preempt_disable_ip = get_lock_parent_ip();
 335#endif
 336		trace_preempt_off(CALLER_ADDR0, get_lock_parent_ip());
 337	}
 338}
 339EXPORT_SYMBOL(__local_bh_disable_ip);
 340#endif /* CONFIG_TRACE_IRQFLAGS */
 341
 342static void __local_bh_enable(unsigned int cnt)
 343{
 344	lockdep_assert_irqs_disabled();
 345
 346	if (preempt_count() == cnt)
 347		trace_preempt_on(CALLER_ADDR0, get_lock_parent_ip());
 348
 349	if (softirq_count() == (cnt & SOFTIRQ_MASK))
 350		lockdep_softirqs_on(_RET_IP_);
 351
 352	__preempt_count_sub(cnt);
 353}
 354
 355/*
 356 * Special-case - softirqs can safely be enabled by __do_softirq(),
 357 * without processing still-pending softirqs:
 358 */
 359void _local_bh_enable(void)
 360{
 361	WARN_ON_ONCE(in_hardirq());
 362	__local_bh_enable(SOFTIRQ_DISABLE_OFFSET);
 363}
 364EXPORT_SYMBOL(_local_bh_enable);
 365
 366void __local_bh_enable_ip(unsigned long ip, unsigned int cnt)
 367{
 368	WARN_ON_ONCE(in_hardirq());
 369	lockdep_assert_irqs_enabled();
 370#ifdef CONFIG_TRACE_IRQFLAGS
 371	local_irq_disable();
 372#endif
 373	/*
 374	 * Are softirqs going to be turned on now:
 375	 */
 376	if (softirq_count() == SOFTIRQ_DISABLE_OFFSET)
 377		lockdep_softirqs_on(ip);
 378	/*
 379	 * Keep preemption disabled until we are done with
 380	 * softirq processing:
 381	 */
 382	__preempt_count_sub(cnt - 1);
 383
 384	if (unlikely(!in_interrupt() && local_softirq_pending())) {
 385		/*
 386		 * Run softirq if any pending. And do it in its own stack
 387		 * as we may be calling this deep in a task call stack already.
 388		 */
 389		do_softirq();
 390	}
 391
 392	preempt_count_dec();
 393#ifdef CONFIG_TRACE_IRQFLAGS
 394	local_irq_enable();
 395#endif
 396	preempt_check_resched();
 397}
 398EXPORT_SYMBOL(__local_bh_enable_ip);
 399
 400static inline void softirq_handle_begin(void)
 401{
 402	__local_bh_disable_ip(_RET_IP_, SOFTIRQ_OFFSET);
 403}
 404
 405static inline void softirq_handle_end(void)
 406{
 407	__local_bh_enable(SOFTIRQ_OFFSET);
 408	WARN_ON_ONCE(in_interrupt());
 409}
 410
 411static inline void ksoftirqd_run_begin(void)
 412{
 413	local_irq_disable();
 414}
 415
 416static inline void ksoftirqd_run_end(void)
 417{
 418	local_irq_enable();
 419}
 420
 421static inline bool should_wake_ksoftirqd(void)
 422{
 423	return true;
 424}
 425
 426static inline void invoke_softirq(void)
 427{
 428	if (!force_irqthreads() || !__this_cpu_read(ksoftirqd)) {
 429#ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK
 430		/*
 431		 * We can safely execute softirq on the current stack if
 432		 * it is the irq stack, because it should be near empty
 433		 * at this stage.
 434		 */
 435		__do_softirq();
 436#else
 437		/*
 438		 * Otherwise, irq_exit() is called on the task stack that can
 439		 * be potentially deep already. So call softirq in its own stack
 440		 * to prevent from any overrun.
 441		 */
 442		do_softirq_own_stack();
 443#endif
 444	} else {
 445		wakeup_softirqd();
 446	}
 447}
 448
 449asmlinkage __visible void do_softirq(void)
 450{
 451	__u32 pending;
 452	unsigned long flags;
 453
 454	if (in_interrupt())
 455		return;
 456
 457	local_irq_save(flags);
 458
 459	pending = local_softirq_pending();
 460
 461	if (pending)
 462		do_softirq_own_stack();
 463
 464	local_irq_restore(flags);
 465}
 466
 467#endif /* !CONFIG_PREEMPT_RT */
 468
 469/*
 470 * We restart softirq processing for at most MAX_SOFTIRQ_RESTART times,
 471 * but break the loop if need_resched() is set or after 2 ms.
 472 * The MAX_SOFTIRQ_TIME provides a nice upper bound in most cases, but in
 473 * certain cases, such as stop_machine(), jiffies may cease to
 474 * increment and so we need the MAX_SOFTIRQ_RESTART limit as
 475 * well to make sure we eventually return from this method.
 476 *
 477 * These limits have been established via experimentation.
 478 * The two things to balance is latency against fairness -
 479 * we want to handle softirqs as soon as possible, but they
 480 * should not be able to lock up the box.
 481 */
 482#define MAX_SOFTIRQ_TIME  msecs_to_jiffies(2)
 483#define MAX_SOFTIRQ_RESTART 10
 484
 485#ifdef CONFIG_TRACE_IRQFLAGS
 486/*
 487 * When we run softirqs from irq_exit() and thus on the hardirq stack we need
 488 * to keep the lockdep irq context tracking as tight as possible in order to
 489 * not miss-qualify lock contexts and miss possible deadlocks.
 490 */
 491
 492static inline bool lockdep_softirq_start(void)
 493{
 494	bool in_hardirq = false;
 495
 496	if (lockdep_hardirq_context()) {
 497		in_hardirq = true;
 498		lockdep_hardirq_exit();
 499	}
 500
 501	lockdep_softirq_enter();
 502
 503	return in_hardirq;
 504}
 505
 506static inline void lockdep_softirq_end(bool in_hardirq)
 507{
 508	lockdep_softirq_exit();
 509
 510	if (in_hardirq)
 511		lockdep_hardirq_enter();
 512}
 513#else
 514static inline bool lockdep_softirq_start(void) { return false; }
 515static inline void lockdep_softirq_end(bool in_hardirq) { }
 516#endif
 517
 518static void handle_softirqs(bool ksirqd)
 519{
 520	unsigned long end = jiffies + MAX_SOFTIRQ_TIME;
 521	unsigned long old_flags = current->flags;
 522	int max_restart = MAX_SOFTIRQ_RESTART;
 523	struct softirq_action *h;
 524	bool in_hardirq;
 525	__u32 pending;
 526	int softirq_bit;
 527
 528	/*
 529	 * Mask out PF_MEMALLOC as the current task context is borrowed for the
 530	 * softirq. A softirq handled, such as network RX, might set PF_MEMALLOC
 531	 * again if the socket is related to swapping.
 532	 */
 533	current->flags &= ~PF_MEMALLOC;
 534
 535	pending = local_softirq_pending();
 536
 537	softirq_handle_begin();
 538	in_hardirq = lockdep_softirq_start();
 539	account_softirq_enter(current);
 540
 541restart:
 542	/* Reset the pending bitmask before enabling irqs */
 543	set_softirq_pending(0);
 544
 545	local_irq_enable();
 546
 547	h = softirq_vec;
 548
 549	while ((softirq_bit = ffs(pending))) {
 550		unsigned int vec_nr;
 551		int prev_count;
 552
 553		h += softirq_bit - 1;
 554
 555		vec_nr = h - softirq_vec;
 556		prev_count = preempt_count();
 557
 558		kstat_incr_softirqs_this_cpu(vec_nr);
 559
 560		trace_softirq_entry(vec_nr);
 561		h->action();
 562		trace_softirq_exit(vec_nr);
 563		if (unlikely(prev_count != preempt_count())) {
 564			pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n",
 565			       vec_nr, softirq_to_name[vec_nr], h->action,
 566			       prev_count, preempt_count());
 567			preempt_count_set(prev_count);
 568		}
 569		h++;
 570		pending >>= softirq_bit;
 571	}
 572
 573	if (!IS_ENABLED(CONFIG_PREEMPT_RT) && ksirqd)
 574		rcu_softirq_qs();
 575
 576	local_irq_disable();
 577
 578	pending = local_softirq_pending();
 579	if (pending) {
 580		if (time_before(jiffies, end) && !need_resched() &&
 581		    --max_restart)
 582			goto restart;
 583
 584		wakeup_softirqd();
 585	}
 586
 587	account_softirq_exit(current);
 588	lockdep_softirq_end(in_hardirq);
 589	softirq_handle_end();
 590	current_restore_flags(old_flags, PF_MEMALLOC);
 591}
 592
 593asmlinkage __visible void __softirq_entry __do_softirq(void)
 594{
 595	handle_softirqs(false);
 596}
 597
 598/**
 599 * irq_enter_rcu - Enter an interrupt context with RCU watching
 600 */
 601void irq_enter_rcu(void)
 602{
 603	__irq_enter_raw();
 604
 605	if (tick_nohz_full_cpu(smp_processor_id()) ||
 606	    (is_idle_task(current) && (irq_count() == HARDIRQ_OFFSET)))
 607		tick_irq_enter();
 608
 609	account_hardirq_enter(current);
 610}
 611
 612/**
 613 * irq_enter - Enter an interrupt context including RCU update
 614 */
 615void irq_enter(void)
 616{
 617	ct_irq_enter();
 618	irq_enter_rcu();
 619}
 620
 621static inline void tick_irq_exit(void)
 622{
 623#ifdef CONFIG_NO_HZ_COMMON
 624	int cpu = smp_processor_id();
 625
 626	/* Make sure that timer wheel updates are propagated */
 627	if ((sched_core_idle_cpu(cpu) && !need_resched()) || tick_nohz_full_cpu(cpu)) {
 628		if (!in_hardirq())
 629			tick_nohz_irq_exit();
 630	}
 631#endif
 632}
 633
 634#ifdef CONFIG_IRQ_FORCED_THREADING
 635DEFINE_PER_CPU(struct task_struct *, ktimerd);
 636DEFINE_PER_CPU(unsigned long, pending_timer_softirq);
 637
 638static void wake_timersd(void)
 639{
 640	struct task_struct *tsk = __this_cpu_read(ktimerd);
 641
 642	if (tsk)
 643		wake_up_process(tsk);
 644}
 645
 646#else
 647
 648static inline void wake_timersd(void) { }
 649
 650#endif
 651
 652static inline void __irq_exit_rcu(void)
 653{
 654#ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED
 655	local_irq_disable();
 656#else
 657	lockdep_assert_irqs_disabled();
 658#endif
 659	account_hardirq_exit(current);
 660	preempt_count_sub(HARDIRQ_OFFSET);
 661	if (!in_interrupt() && local_softirq_pending())
 662		invoke_softirq();
 663
 664	if (IS_ENABLED(CONFIG_IRQ_FORCED_THREADING) && force_irqthreads() &&
 665	    local_timers_pending_force_th() && !(in_nmi() | in_hardirq()))
 666		wake_timersd();
 667
 668	tick_irq_exit();
 669}
 670
 671/**
 672 * irq_exit_rcu() - Exit an interrupt context without updating RCU
 673 *
 674 * Also processes softirqs if needed and possible.
 675 */
 676void irq_exit_rcu(void)
 677{
 678	__irq_exit_rcu();
 679	 /* must be last! */
 680	lockdep_hardirq_exit();
 681}
 682
 683/**
 684 * irq_exit - Exit an interrupt context, update RCU and lockdep
 685 *
 686 * Also processes softirqs if needed and possible.
 687 */
 688void irq_exit(void)
 689{
 690	__irq_exit_rcu();
 691	ct_irq_exit();
 692	 /* must be last! */
 693	lockdep_hardirq_exit();
 694}
 695
 696/*
 697 * This function must run with irqs disabled!
 698 */
 699inline void raise_softirq_irqoff(unsigned int nr)
 700{
 701	__raise_softirq_irqoff(nr);
 702
 703	/*
 704	 * If we're in an interrupt or softirq, we're done
 705	 * (this also catches softirq-disabled code). We will
 706	 * actually run the softirq once we return from
 707	 * the irq or softirq.
 708	 *
 709	 * Otherwise we wake up ksoftirqd to make sure we
 710	 * schedule the softirq soon.
 711	 */
 712	if (!in_interrupt() && should_wake_ksoftirqd())
 713		wakeup_softirqd();
 714}
 715
 716void raise_softirq(unsigned int nr)
 717{
 718	unsigned long flags;
 719
 720	local_irq_save(flags);
 721	raise_softirq_irqoff(nr);
 722	local_irq_restore(flags);
 723}
 724
 725void __raise_softirq_irqoff(unsigned int nr)
 726{
 727	lockdep_assert_irqs_disabled();
 728	trace_softirq_raise(nr);
 729	or_softirq_pending(1UL << nr);
 730}
 731
 732void open_softirq(int nr, void (*action)(void))
 733{
 734	softirq_vec[nr].action = action;
 735}
 736
 737/*
 738 * Tasklets
 739 */
 740struct tasklet_head {
 741	struct tasklet_struct *head;
 742	struct tasklet_struct **tail;
 743};
 744
 745static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec);
 746static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec);
 747
 748static void __tasklet_schedule_common(struct tasklet_struct *t,
 749				      struct tasklet_head __percpu *headp,
 750				      unsigned int softirq_nr)
 751{
 752	struct tasklet_head *head;
 753	unsigned long flags;
 754
 755	local_irq_save(flags);
 756	head = this_cpu_ptr(headp);
 757	t->next = NULL;
 758	*head->tail = t;
 759	head->tail = &(t->next);
 760	raise_softirq_irqoff(softirq_nr);
 761	local_irq_restore(flags);
 762}
 763
 764void __tasklet_schedule(struct tasklet_struct *t)
 765{
 766	__tasklet_schedule_common(t, &tasklet_vec,
 767				  TASKLET_SOFTIRQ);
 768}
 769EXPORT_SYMBOL(__tasklet_schedule);
 770
 771void __tasklet_hi_schedule(struct tasklet_struct *t)
 772{
 773	__tasklet_schedule_common(t, &tasklet_hi_vec,
 774				  HI_SOFTIRQ);
 775}
 776EXPORT_SYMBOL(__tasklet_hi_schedule);
 777
 778static bool tasklet_clear_sched(struct tasklet_struct *t)
 779{
 780	if (test_and_clear_wake_up_bit(TASKLET_STATE_SCHED, &t->state))
 781		return true;
 782
 783	WARN_ONCE(1, "tasklet SCHED state not set: %s %pS\n",
 784		  t->use_callback ? "callback" : "func",
 785		  t->use_callback ? (void *)t->callback : (void *)t->func);
 786
 787	return false;
 788}
 789
 790static void tasklet_action_common(struct tasklet_head *tl_head,
 791				  unsigned int softirq_nr)
 792{
 793	struct tasklet_struct *list;
 794
 795	local_irq_disable();
 796	list = tl_head->head;
 797	tl_head->head = NULL;
 798	tl_head->tail = &tl_head->head;
 799	local_irq_enable();
 800
 801	while (list) {
 802		struct tasklet_struct *t = list;
 803
 804		list = list->next;
 805
 806		if (tasklet_trylock(t)) {
 807			if (!atomic_read(&t->count)) {
 808				if (tasklet_clear_sched(t)) {
 809					if (t->use_callback) {
 810						trace_tasklet_entry(t, t->callback);
 811						t->callback(t);
 812						trace_tasklet_exit(t, t->callback);
 813					} else {
 814						trace_tasklet_entry(t, t->func);
 815						t->func(t->data);
 816						trace_tasklet_exit(t, t->func);
 817					}
 818				}
 819				tasklet_unlock(t);
 820				continue;
 821			}
 822			tasklet_unlock(t);
 823		}
 824
 825		local_irq_disable();
 826		t->next = NULL;
 827		*tl_head->tail = t;
 828		tl_head->tail = &t->next;
 829		__raise_softirq_irqoff(softirq_nr);
 830		local_irq_enable();
 831	}
 832}
 833
 834static __latent_entropy void tasklet_action(void)
 835{
 836	workqueue_softirq_action(false);
 837	tasklet_action_common(this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ);
 838}
 839
 840static __latent_entropy void tasklet_hi_action(void)
 841{
 842	workqueue_softirq_action(true);
 843	tasklet_action_common(this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ);
 844}
 845
 846void tasklet_setup(struct tasklet_struct *t,
 847		   void (*callback)(struct tasklet_struct *))
 848{
 849	t->next = NULL;
 850	t->state = 0;
 851	atomic_set(&t->count, 0);
 852	t->callback = callback;
 853	t->use_callback = true;
 854	t->data = 0;
 855}
 856EXPORT_SYMBOL(tasklet_setup);
 857
 858void tasklet_init(struct tasklet_struct *t,
 859		  void (*func)(unsigned long), unsigned long data)
 860{
 861	t->next = NULL;
 862	t->state = 0;
 863	atomic_set(&t->count, 0);
 864	t->func = func;
 865	t->use_callback = false;
 866	t->data = data;
 867}
 868EXPORT_SYMBOL(tasklet_init);
 869
 870#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
 871/*
 872 * Do not use in new code. Waiting for tasklets from atomic contexts is
 873 * error prone and should be avoided.
 874 */
 875void tasklet_unlock_spin_wait(struct tasklet_struct *t)
 876{
 877	while (test_bit(TASKLET_STATE_RUN, &(t)->state)) {
 878		if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
 879			/*
 880			 * Prevent a live lock when current preempted soft
 881			 * interrupt processing or prevents ksoftirqd from
 882			 * running. If the tasklet runs on a different CPU
 883			 * then this has no effect other than doing the BH
 884			 * disable/enable dance for nothing.
 885			 */
 886			local_bh_disable();
 887			local_bh_enable();
 888		} else {
 889			cpu_relax();
 890		}
 891	}
 892}
 893EXPORT_SYMBOL(tasklet_unlock_spin_wait);
 894#endif
 895
 896void tasklet_kill(struct tasklet_struct *t)
 897{
 898	if (in_interrupt())
 899		pr_notice("Attempt to kill tasklet from interrupt\n");
 900
 901	wait_on_bit_lock(&t->state, TASKLET_STATE_SCHED, TASK_UNINTERRUPTIBLE);
 902
 903	tasklet_unlock_wait(t);
 904	tasklet_clear_sched(t);
 905}
 906EXPORT_SYMBOL(tasklet_kill);
 907
 908#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT)
 909void tasklet_unlock(struct tasklet_struct *t)
 910{
 911	clear_and_wake_up_bit(TASKLET_STATE_RUN, &t->state);
 912}
 913EXPORT_SYMBOL_GPL(tasklet_unlock);
 914
 915void tasklet_unlock_wait(struct tasklet_struct *t)
 916{
 917	wait_on_bit(&t->state, TASKLET_STATE_RUN, TASK_UNINTERRUPTIBLE);
 918}
 919EXPORT_SYMBOL_GPL(tasklet_unlock_wait);
 920#endif
 921
 922void __init softirq_init(void)
 923{
 924	int cpu;
 925
 926	for_each_possible_cpu(cpu) {
 927		per_cpu(tasklet_vec, cpu).tail =
 928			&per_cpu(tasklet_vec, cpu).head;
 929		per_cpu(tasklet_hi_vec, cpu).tail =
 930			&per_cpu(tasklet_hi_vec, cpu).head;
 931	}
 932
 933	open_softirq(TASKLET_SOFTIRQ, tasklet_action);
 934	open_softirq(HI_SOFTIRQ, tasklet_hi_action);
 935}
 936
 937static int ksoftirqd_should_run(unsigned int cpu)
 938{
 939	return local_softirq_pending();
 940}
 941
 942static void run_ksoftirqd(unsigned int cpu)
 943{
 944	ksoftirqd_run_begin();
 945	if (local_softirq_pending()) {
 946		/*
 947		 * We can safely run softirq on inline stack, as we are not deep
 948		 * in the task stack here.
 949		 */
 950		handle_softirqs(true);
 951		ksoftirqd_run_end();
 952		cond_resched();
 953		return;
 954	}
 955	ksoftirqd_run_end();
 956}
 957
 958#ifdef CONFIG_HOTPLUG_CPU
 959static int takeover_tasklets(unsigned int cpu)
 960{
 961	workqueue_softirq_dead(cpu);
 962
 963	/* CPU is dead, so no lock needed. */
 964	local_irq_disable();
 965
 966	/* Find end, append list for that CPU. */
 967	if (&per_cpu(tasklet_vec, cpu).head != per_cpu(tasklet_vec, cpu).tail) {
 968		*__this_cpu_read(tasklet_vec.tail) = per_cpu(tasklet_vec, cpu).head;
 969		__this_cpu_write(tasklet_vec.tail, per_cpu(tasklet_vec, cpu).tail);
 970		per_cpu(tasklet_vec, cpu).head = NULL;
 971		per_cpu(tasklet_vec, cpu).tail = &per_cpu(tasklet_vec, cpu).head;
 972	}
 973	raise_softirq_irqoff(TASKLET_SOFTIRQ);
 974
 975	if (&per_cpu(tasklet_hi_vec, cpu).head != per_cpu(tasklet_hi_vec, cpu).tail) {
 976		*__this_cpu_read(tasklet_hi_vec.tail) = per_cpu(tasklet_hi_vec, cpu).head;
 977		__this_cpu_write(tasklet_hi_vec.tail, per_cpu(tasklet_hi_vec, cpu).tail);
 978		per_cpu(tasklet_hi_vec, cpu).head = NULL;
 979		per_cpu(tasklet_hi_vec, cpu).tail = &per_cpu(tasklet_hi_vec, cpu).head;
 980	}
 981	raise_softirq_irqoff(HI_SOFTIRQ);
 982
 983	local_irq_enable();
 984	return 0;
 985}
 986#else
 987#define takeover_tasklets	NULL
 988#endif /* CONFIG_HOTPLUG_CPU */
 989
 990static struct smp_hotplug_thread softirq_threads = {
 991	.store			= &ksoftirqd,
 992	.thread_should_run	= ksoftirqd_should_run,
 993	.thread_fn		= run_ksoftirqd,
 994	.thread_comm		= "ksoftirqd/%u",
 995};
 996
 997#ifdef CONFIG_IRQ_FORCED_THREADING
 998static void ktimerd_setup(unsigned int cpu)
 999{
1000	/* Above SCHED_NORMAL to handle timers before regular tasks. */
1001	sched_set_fifo_low(current);
1002}
1003
1004static int ktimerd_should_run(unsigned int cpu)
1005{
1006	return local_timers_pending_force_th();
1007}
1008
1009void raise_ktimers_thread(unsigned int nr)
1010{
1011	trace_softirq_raise(nr);
1012	__this_cpu_or(pending_timer_softirq, BIT(nr));
1013}
1014
1015static void run_ktimerd(unsigned int cpu)
1016{
1017	unsigned int timer_si;
1018
1019	ksoftirqd_run_begin();
1020
1021	timer_si = local_timers_pending_force_th();
1022	__this_cpu_write(pending_timer_softirq, 0);
1023	or_softirq_pending(timer_si);
1024
1025	__do_softirq();
1026
1027	ksoftirqd_run_end();
1028}
1029
1030static struct smp_hotplug_thread timer_thread = {
1031	.store			= &ktimerd,
1032	.setup			= ktimerd_setup,
1033	.thread_should_run	= ktimerd_should_run,
1034	.thread_fn		= run_ktimerd,
1035	.thread_comm		= "ktimers/%u",
1036};
1037#endif
1038
1039static __init int spawn_ksoftirqd(void)
1040{
1041	cpuhp_setup_state_nocalls(CPUHP_SOFTIRQ_DEAD, "softirq:dead", NULL,
1042				  takeover_tasklets);
1043	BUG_ON(smpboot_register_percpu_thread(&softirq_threads));
1044#ifdef CONFIG_IRQ_FORCED_THREADING
1045	if (force_irqthreads())
1046		BUG_ON(smpboot_register_percpu_thread(&timer_thread));
1047#endif
1048	return 0;
1049}
1050early_initcall(spawn_ksoftirqd);
1051
1052/*
1053 * [ These __weak aliases are kept in a separate compilation unit, so that
1054 *   GCC does not inline them incorrectly. ]
1055 */
1056
1057int __init __weak early_irq_init(void)
1058{
1059	return 0;
1060}
1061
1062int __init __weak arch_probe_nr_irqs(void)
1063{
1064	return NR_IRQS_LEGACY;
1065}
1066
1067int __init __weak arch_early_irq_init(void)
1068{
1069	return 0;
1070}
1071
1072unsigned int __weak arch_dynirq_lower_bound(unsigned int from)
1073{
1074	return from;
1075}