1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * SMP support for ppc.
   4 *
   5 * Written by Cort Dougan (cort@cs.nmt.edu) borrowing a great
   6 * deal of code from the sparc and intel versions.
   7 *
   8 * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
   9 *
  10 * PowerPC-64 Support added by Dave Engebretsen, Peter Bergner, and
  11 * Mike Corrigan {engebret|bergner|mikec}@us.ibm.com
  12 */
  13
  14#undef DEBUG
  15
  16#include <linux/kernel.h>
  17#include <linux/export.h>
  18#include <linux/sched/mm.h>
  19#include <linux/sched/task_stack.h>
  20#include <linux/sched/topology.h>
  21#include <linux/smp.h>
  22#include <linux/interrupt.h>
  23#include <linux/delay.h>
  24#include <linux/init.h>
  25#include <linux/spinlock.h>
  26#include <linux/cache.h>
  27#include <linux/err.h>
  28#include <linux/device.h>
  29#include <linux/cpu.h>
  30#include <linux/notifier.h>
  31#include <linux/topology.h>
  32#include <linux/profile.h>
  33#include <linux/processor.h>
  34#include <linux/random.h>
  35#include <linux/stackprotector.h>
  36#include <linux/pgtable.h>
  37#include <linux/clockchips.h>
 
  38
  39#include <asm/ptrace.h>
  40#include <linux/atomic.h>
  41#include <asm/irq.h>
  42#include <asm/hw_irq.h>
  43#include <asm/kvm_ppc.h>
  44#include <asm/dbell.h>
  45#include <asm/page.h>
  46#include <asm/prom.h>
  47#include <asm/smp.h>
  48#include <asm/time.h>
  49#include <asm/machdep.h>
 
  50#include <asm/cputhreads.h>
  51#include <asm/cputable.h>
  52#include <asm/mpic.h>
  53#include <asm/vdso_datapage.h>
  54#ifdef CONFIG_PPC64
  55#include <asm/paca.h>
  56#endif
  57#include <asm/vdso.h>
  58#include <asm/debug.h>
  59#include <asm/kexec.h>
  60#include <asm/asm-prototypes.h>
  61#include <asm/cpu_has_feature.h>
  62#include <asm/ftrace.h>
  63#include <asm/kup.h>
 
 
 
  64
  65#ifdef DEBUG
  66#include <asm/udbg.h>
  67#define DBG(fmt...) udbg_printf(fmt)
  68#else
  69#define DBG(fmt...)
  70#endif
  71
  72#ifdef CONFIG_HOTPLUG_CPU
  73/* State of each CPU during hotplug phases */
  74static DEFINE_PER_CPU(int, cpu_state) = { 0 };
  75#endif
  76
  77struct task_struct *secondary_current;
  78bool has_big_cores;
  79bool coregroup_enabled;
  80bool thread_group_shares_l2;
 
  81
  82DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
  83DEFINE_PER_CPU(cpumask_var_t, cpu_smallcore_map);
  84DEFINE_PER_CPU(cpumask_var_t, cpu_l2_cache_map);
  85DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
  86static DEFINE_PER_CPU(cpumask_var_t, cpu_coregroup_map);
  87
  88EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
  89EXPORT_PER_CPU_SYMBOL(cpu_l2_cache_map);
  90EXPORT_PER_CPU_SYMBOL(cpu_core_map);
  91EXPORT_SYMBOL_GPL(has_big_cores);
  92
  93enum {
  94#ifdef CONFIG_SCHED_SMT
  95	smt_idx,
  96#endif
  97	cache_idx,
  98	mc_idx,
  99	die_idx,
 100};
 101
 102#define MAX_THREAD_LIST_SIZE	8
 103#define THREAD_GROUP_SHARE_L1   1
 104#define THREAD_GROUP_SHARE_L2   2
 105struct thread_groups {
 106	unsigned int property;
 107	unsigned int nr_groups;
 108	unsigned int threads_per_group;
 109	unsigned int thread_list[MAX_THREAD_LIST_SIZE];
 110};
 111
 112/* Maximum number of properties that groups of threads within a core can share */
 113#define MAX_THREAD_GROUP_PROPERTIES 2
 114
 115struct thread_groups_list {
 116	unsigned int nr_properties;
 117	struct thread_groups property_tgs[MAX_THREAD_GROUP_PROPERTIES];
 118};
 119
 120static struct thread_groups_list tgl[NR_CPUS] __initdata;
 121/*
 122 * On big-cores system, thread_group_l1_cache_map for each CPU corresponds to
 123 * the set its siblings that share the L1-cache.
 124 */
 125static DEFINE_PER_CPU(cpumask_var_t, thread_group_l1_cache_map);
 126
 127/*
 128 * On some big-cores system, thread_group_l2_cache_map for each CPU
 129 * corresponds to the set its siblings within the core that share the
 130 * L2-cache.
 131 */
 132static DEFINE_PER_CPU(cpumask_var_t, thread_group_l2_cache_map);
 
 
 
 
 
 
 133
 134/* SMP operations for this machine */
 135struct smp_ops_t *smp_ops;
 136
 137/* Can't be static due to PowerMac hackery */
 138volatile unsigned int cpu_callin_map[NR_CPUS];
 139
 140int smt_enabled_at_boot = 1;
 141
 142/*
 143 * Returns 1 if the specified cpu should be brought up during boot.
 144 * Used to inhibit booting threads if they've been disabled or
 145 * limited on the command line
 146 */
 147int smp_generic_cpu_bootable(unsigned int nr)
 148{
 149	/* Special case - we inhibit secondary thread startup
 150	 * during boot if the user requests it.
 151	 */
 152	if (system_state < SYSTEM_RUNNING && cpu_has_feature(CPU_FTR_SMT)) {
 153		if (!smt_enabled_at_boot && cpu_thread_in_core(nr) != 0)
 154			return 0;
 155		if (smt_enabled_at_boot
 156		    && cpu_thread_in_core(nr) >= smt_enabled_at_boot)
 157			return 0;
 158	}
 159
 160	return 1;
 161}
 162
 163
 164#ifdef CONFIG_PPC64
 165int smp_generic_kick_cpu(int nr)
 166{
 167	if (nr < 0 || nr >= nr_cpu_ids)
 168		return -EINVAL;
 169
 170	/*
 171	 * The processor is currently spinning, waiting for the
 172	 * cpu_start field to become non-zero After we set cpu_start,
 173	 * the processor will continue on to secondary_start
 174	 */
 175	if (!paca_ptrs[nr]->cpu_start) {
 176		paca_ptrs[nr]->cpu_start = 1;
 177		smp_mb();
 178		return 0;
 179	}
 180
 181#ifdef CONFIG_HOTPLUG_CPU
 182	/*
 183	 * Ok it's not there, so it might be soft-unplugged, let's
 184	 * try to bring it back
 185	 */
 186	generic_set_cpu_up(nr);
 187	smp_wmb();
 188	smp_send_reschedule(nr);
 189#endif /* CONFIG_HOTPLUG_CPU */
 190
 191	return 0;
 192}
 193#endif /* CONFIG_PPC64 */
 194
 195static irqreturn_t call_function_action(int irq, void *data)
 196{
 197	generic_smp_call_function_interrupt();
 198	return IRQ_HANDLED;
 199}
 200
 201static irqreturn_t reschedule_action(int irq, void *data)
 202{
 203	scheduler_ipi();
 204	return IRQ_HANDLED;
 205}
 206
 207#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 208static irqreturn_t tick_broadcast_ipi_action(int irq, void *data)
 209{
 210	timer_broadcast_interrupt();
 211	return IRQ_HANDLED;
 212}
 213#endif
 214
 215#ifdef CONFIG_NMI_IPI
 216static irqreturn_t nmi_ipi_action(int irq, void *data)
 217{
 218	smp_handle_nmi_ipi(get_irq_regs());
 219	return IRQ_HANDLED;
 220}
 221#endif
 222
 223static irq_handler_t smp_ipi_action[] = {
 224	[PPC_MSG_CALL_FUNCTION] =  call_function_action,
 225	[PPC_MSG_RESCHEDULE] = reschedule_action,
 226#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 227	[PPC_MSG_TICK_BROADCAST] = tick_broadcast_ipi_action,
 228#endif
 229#ifdef CONFIG_NMI_IPI
 230	[PPC_MSG_NMI_IPI] = nmi_ipi_action,
 231#endif
 232};
 233
 234/*
 235 * The NMI IPI is a fallback and not truly non-maskable. It is simpler
 236 * than going through the call function infrastructure, and strongly
 237 * serialized, so it is more appropriate for debugging.
 238 */
 239const char *smp_ipi_name[] = {
 240	[PPC_MSG_CALL_FUNCTION] =  "ipi call function",
 241	[PPC_MSG_RESCHEDULE] = "ipi reschedule",
 242#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 243	[PPC_MSG_TICK_BROADCAST] = "ipi tick-broadcast",
 244#endif
 245#ifdef CONFIG_NMI_IPI
 246	[PPC_MSG_NMI_IPI] = "nmi ipi",
 247#endif
 248};
 249
 250/* optional function to request ipi, for controllers with >= 4 ipis */
 251int smp_request_message_ipi(int virq, int msg)
 252{
 253	int err;
 254
 255	if (msg < 0 || msg > PPC_MSG_NMI_IPI)
 256		return -EINVAL;
 257#ifndef CONFIG_NMI_IPI
 258	if (msg == PPC_MSG_NMI_IPI)
 259		return 1;
 260#endif
 261
 262	err = request_irq(virq, smp_ipi_action[msg],
 263			  IRQF_PERCPU | IRQF_NO_THREAD | IRQF_NO_SUSPEND,
 264			  smp_ipi_name[msg], NULL);
 265	WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n",
 266		virq, smp_ipi_name[msg], err);
 267
 268	return err;
 269}
 270
 271#ifdef CONFIG_PPC_SMP_MUXED_IPI
 272struct cpu_messages {
 273	long messages;			/* current messages */
 274};
 275static DEFINE_PER_CPU_SHARED_ALIGNED(struct cpu_messages, ipi_message);
 276
 277void smp_muxed_ipi_set_message(int cpu, int msg)
 278{
 279	struct cpu_messages *info = &per_cpu(ipi_message, cpu);
 280	char *message = (char *)&info->messages;
 281
 282	/*
 283	 * Order previous accesses before accesses in the IPI handler.
 284	 */
 285	smp_mb();
 286	message[msg] = 1;
 287}
 288
 289void smp_muxed_ipi_message_pass(int cpu, int msg)
 290{
 291	smp_muxed_ipi_set_message(cpu, msg);
 292
 293	/*
 294	 * cause_ipi functions are required to include a full barrier
 295	 * before doing whatever causes the IPI.
 296	 */
 297	smp_ops->cause_ipi(cpu);
 298}
 299
 300#ifdef __BIG_ENDIAN__
 301#define IPI_MESSAGE(A) (1uL << ((BITS_PER_LONG - 8) - 8 * (A)))
 302#else
 303#define IPI_MESSAGE(A) (1uL << (8 * (A)))
 304#endif
 305
 306irqreturn_t smp_ipi_demux(void)
 307{
 308	mb();	/* order any irq clear */
 309
 310	return smp_ipi_demux_relaxed();
 311}
 312
 313/* sync-free variant. Callers should ensure synchronization */
 314irqreturn_t smp_ipi_demux_relaxed(void)
 315{
 316	struct cpu_messages *info;
 317	unsigned long all;
 318
 319	info = this_cpu_ptr(&ipi_message);
 320	do {
 321		all = xchg(&info->messages, 0);
 322#if defined(CONFIG_KVM_XICS) && defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
 323		/*
 324		 * Must check for PPC_MSG_RM_HOST_ACTION messages
 325		 * before PPC_MSG_CALL_FUNCTION messages because when
 326		 * a VM is destroyed, we call kick_all_cpus_sync()
 327		 * to ensure that any pending PPC_MSG_RM_HOST_ACTION
 328		 * messages have completed before we free any VCPUs.
 329		 */
 330		if (all & IPI_MESSAGE(PPC_MSG_RM_HOST_ACTION))
 331			kvmppc_xics_ipi_action();
 332#endif
 333		if (all & IPI_MESSAGE(PPC_MSG_CALL_FUNCTION))
 334			generic_smp_call_function_interrupt();
 335		if (all & IPI_MESSAGE(PPC_MSG_RESCHEDULE))
 336			scheduler_ipi();
 337#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 338		if (all & IPI_MESSAGE(PPC_MSG_TICK_BROADCAST))
 339			timer_broadcast_interrupt();
 340#endif
 341#ifdef CONFIG_NMI_IPI
 342		if (all & IPI_MESSAGE(PPC_MSG_NMI_IPI))
 343			nmi_ipi_action(0, NULL);
 344#endif
 345	} while (info->messages);
 346
 347	return IRQ_HANDLED;
 348}
 349#endif /* CONFIG_PPC_SMP_MUXED_IPI */
 350
 351static inline void do_message_pass(int cpu, int msg)
 352{
 353	if (smp_ops->message_pass)
 354		smp_ops->message_pass(cpu, msg);
 355#ifdef CONFIG_PPC_SMP_MUXED_IPI
 356	else
 357		smp_muxed_ipi_message_pass(cpu, msg);
 358#endif
 359}
 360
 361void smp_send_reschedule(int cpu)
 362{
 363	if (likely(smp_ops))
 364		do_message_pass(cpu, PPC_MSG_RESCHEDULE);
 365}
 366EXPORT_SYMBOL_GPL(smp_send_reschedule);
 367
 368void arch_send_call_function_single_ipi(int cpu)
 369{
 370	do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
 371}
 372
 373void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 374{
 375	unsigned int cpu;
 376
 377	for_each_cpu(cpu, mask)
 378		do_message_pass(cpu, PPC_MSG_CALL_FUNCTION);
 379}
 380
 381#ifdef CONFIG_NMI_IPI
 382
 383/*
 384 * "NMI IPI" system.
 385 *
 386 * NMI IPIs may not be recoverable, so should not be used as ongoing part of
 387 * a running system. They can be used for crash, debug, halt/reboot, etc.
 388 *
 389 * The IPI call waits with interrupts disabled until all targets enter the
 390 * NMI handler, then returns. Subsequent IPIs can be issued before targets
 391 * have returned from their handlers, so there is no guarantee about
 392 * concurrency or re-entrancy.
 393 *
 394 * A new NMI can be issued before all targets exit the handler.
 395 *
 396 * The IPI call may time out without all targets entering the NMI handler.
 397 * In that case, there is some logic to recover (and ignore subsequent
 398 * NMI interrupts that may eventually be raised), but the platform interrupt
 399 * handler may not be able to distinguish this from other exception causes,
 400 * which may cause a crash.
 401 */
 402
 403static atomic_t __nmi_ipi_lock = ATOMIC_INIT(0);
 404static struct cpumask nmi_ipi_pending_mask;
 405static bool nmi_ipi_busy = false;
 406static void (*nmi_ipi_function)(struct pt_regs *) = NULL;
 407
 408static void nmi_ipi_lock_start(unsigned long *flags)
 409{
 410	raw_local_irq_save(*flags);
 411	hard_irq_disable();
 412	while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1) {
 413		raw_local_irq_restore(*flags);
 414		spin_until_cond(atomic_read(&__nmi_ipi_lock) == 0);
 415		raw_local_irq_save(*flags);
 416		hard_irq_disable();
 417	}
 418}
 419
 420static void nmi_ipi_lock(void)
 421{
 422	while (atomic_cmpxchg(&__nmi_ipi_lock, 0, 1) == 1)
 423		spin_until_cond(atomic_read(&__nmi_ipi_lock) == 0);
 424}
 425
 426static void nmi_ipi_unlock(void)
 427{
 428	smp_mb();
 429	WARN_ON(atomic_read(&__nmi_ipi_lock) != 1);
 430	atomic_set(&__nmi_ipi_lock, 0);
 431}
 432
 433static void nmi_ipi_unlock_end(unsigned long *flags)
 434{
 435	nmi_ipi_unlock();
 436	raw_local_irq_restore(*flags);
 437}
 438
 439/*
 440 * Platform NMI handler calls this to ack
 441 */
 442int smp_handle_nmi_ipi(struct pt_regs *regs)
 443{
 444	void (*fn)(struct pt_regs *) = NULL;
 445	unsigned long flags;
 446	int me = raw_smp_processor_id();
 447	int ret = 0;
 448
 449	/*
 450	 * Unexpected NMIs are possible here because the interrupt may not
 451	 * be able to distinguish NMI IPIs from other types of NMIs, or
 452	 * because the caller may have timed out.
 453	 */
 454	nmi_ipi_lock_start(&flags);
 455	if (cpumask_test_cpu(me, &nmi_ipi_pending_mask)) {
 456		cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
 457		fn = READ_ONCE(nmi_ipi_function);
 458		WARN_ON_ONCE(!fn);
 459		ret = 1;
 460	}
 461	nmi_ipi_unlock_end(&flags);
 462
 463	if (fn)
 464		fn(regs);
 465
 466	return ret;
 467}
 468
 469static void do_smp_send_nmi_ipi(int cpu, bool safe)
 470{
 471	if (!safe && smp_ops->cause_nmi_ipi && smp_ops->cause_nmi_ipi(cpu))
 472		return;
 473
 474	if (cpu >= 0) {
 475		do_message_pass(cpu, PPC_MSG_NMI_IPI);
 476	} else {
 477		int c;
 478
 479		for_each_online_cpu(c) {
 480			if (c == raw_smp_processor_id())
 481				continue;
 482			do_message_pass(c, PPC_MSG_NMI_IPI);
 483		}
 484	}
 485}
 486
 487/*
 488 * - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS.
 489 * - fn is the target callback function.
 490 * - delay_us > 0 is the delay before giving up waiting for targets to
 491 *   begin executing the handler, == 0 specifies indefinite delay.
 492 */
 493static int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *),
 494				u64 delay_us, bool safe)
 495{
 496	unsigned long flags;
 497	int me = raw_smp_processor_id();
 498	int ret = 1;
 499
 500	BUG_ON(cpu == me);
 501	BUG_ON(cpu < 0 && cpu != NMI_IPI_ALL_OTHERS);
 502
 503	if (unlikely(!smp_ops))
 504		return 0;
 505
 506	nmi_ipi_lock_start(&flags);
 507	while (nmi_ipi_busy) {
 508		nmi_ipi_unlock_end(&flags);
 509		spin_until_cond(!nmi_ipi_busy);
 510		nmi_ipi_lock_start(&flags);
 511	}
 512	nmi_ipi_busy = true;
 513	nmi_ipi_function = fn;
 514
 515	WARN_ON_ONCE(!cpumask_empty(&nmi_ipi_pending_mask));
 516
 517	if (cpu < 0) {
 518		/* ALL_OTHERS */
 519		cpumask_copy(&nmi_ipi_pending_mask, cpu_online_mask);
 520		cpumask_clear_cpu(me, &nmi_ipi_pending_mask);
 521	} else {
 522		cpumask_set_cpu(cpu, &nmi_ipi_pending_mask);
 523	}
 524
 525	nmi_ipi_unlock();
 526
 527	/* Interrupts remain hard disabled */
 528
 529	do_smp_send_nmi_ipi(cpu, safe);
 530
 531	nmi_ipi_lock();
 532	/* nmi_ipi_busy is set here, so unlock/lock is okay */
 533	while (!cpumask_empty(&nmi_ipi_pending_mask)) {
 534		nmi_ipi_unlock();
 535		udelay(1);
 536		nmi_ipi_lock();
 537		if (delay_us) {
 538			delay_us--;
 539			if (!delay_us)
 540				break;
 541		}
 542	}
 543
 544	if (!cpumask_empty(&nmi_ipi_pending_mask)) {
 545		/* Timeout waiting for CPUs to call smp_handle_nmi_ipi */
 546		ret = 0;
 547		cpumask_clear(&nmi_ipi_pending_mask);
 548	}
 549
 550	nmi_ipi_function = NULL;
 551	nmi_ipi_busy = false;
 552
 553	nmi_ipi_unlock_end(&flags);
 554
 555	return ret;
 556}
 557
 558int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)
 559{
 560	return __smp_send_nmi_ipi(cpu, fn, delay_us, false);
 561}
 562
 563int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us)
 564{
 565	return __smp_send_nmi_ipi(cpu, fn, delay_us, true);
 566}
 567#endif /* CONFIG_NMI_IPI */
 568
 569#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
 570void tick_broadcast(const struct cpumask *mask)
 571{
 572	unsigned int cpu;
 573
 574	for_each_cpu(cpu, mask)
 575		do_message_pass(cpu, PPC_MSG_TICK_BROADCAST);
 576}
 577#endif
 578
 579#ifdef CONFIG_DEBUGGER
 580static void debugger_ipi_callback(struct pt_regs *regs)
 581{
 582	debugger_ipi(regs);
 583}
 584
 585void smp_send_debugger_break(void)
 586{
 587	smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, debugger_ipi_callback, 1000000);
 588}
 589#endif
 590
 591#ifdef CONFIG_KEXEC_CORE
 592void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *))
 593{
 594	int cpu;
 595
 596	smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, crash_ipi_callback, 1000000);
 597	if (kdump_in_progress() && crash_wake_offline) {
 598		for_each_present_cpu(cpu) {
 599			if (cpu_online(cpu))
 600				continue;
 601			/*
 602			 * crash_ipi_callback will wait for
 603			 * all cpus, including offline CPUs.
 604			 * We don't care about nmi_ipi_function.
 605			 * Offline cpus will jump straight into
 606			 * crash_ipi_callback, we can skip the
 607			 * entire NMI dance and waiting for
 608			 * cpus to clear pending mask, etc.
 609			 */
 610			do_smp_send_nmi_ipi(cpu, false);
 611		}
 612	}
 613}
 614#endif
 615
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 616#ifdef CONFIG_NMI_IPI
 617static void nmi_stop_this_cpu(struct pt_regs *regs)
 618{
 619	/*
 620	 * IRQs are already hard disabled by the smp_handle_nmi_ipi.
 621	 */
 622	set_cpu_online(smp_processor_id(), false);
 623
 624	spin_begin();
 625	while (1)
 626		spin_cpu_relax();
 627}
 628
 629void smp_send_stop(void)
 630{
 631	smp_send_nmi_ipi(NMI_IPI_ALL_OTHERS, nmi_stop_this_cpu, 1000000);
 632}
 633
 634#else /* CONFIG_NMI_IPI */
 635
 636static void stop_this_cpu(void *dummy)
 637{
 638	hard_irq_disable();
 639
 640	/*
 641	 * Offlining CPUs in stop_this_cpu can result in scheduler warnings,
 642	 * (see commit de6e5d38417e), but printk_safe_flush_on_panic() wants
 643	 * to know other CPUs are offline before it breaks locks to flush
 644	 * printk buffers, in case we panic()ed while holding the lock.
 645	 */
 646	set_cpu_online(smp_processor_id(), false);
 647
 648	spin_begin();
 649	while (1)
 650		spin_cpu_relax();
 651}
 652
 653void smp_send_stop(void)
 654{
 655	static bool stopped = false;
 656
 657	/*
 658	 * Prevent waiting on csd lock from a previous smp_send_stop.
 659	 * This is racy, but in general callers try to do the right
 660	 * thing and only fire off one smp_send_stop (e.g., see
 661	 * kernel/panic.c)
 662	 */
 663	if (stopped)
 664		return;
 665
 666	stopped = true;
 667
 668	smp_call_function(stop_this_cpu, NULL, 0);
 669}
 670#endif /* CONFIG_NMI_IPI */
 671
 672struct task_struct *current_set[NR_CPUS];
 673
 674static void smp_store_cpu_info(int id)
 675{
 676	per_cpu(cpu_pvr, id) = mfspr(SPRN_PVR);
 677#ifdef CONFIG_PPC_FSL_BOOK3E
 678	per_cpu(next_tlbcam_idx, id)
 679		= (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) - 1;
 680#endif
 681}
 682
 683/*
 684 * Relationships between CPUs are maintained in a set of per-cpu cpumasks so
 685 * rather than just passing around the cpumask we pass around a function that
 686 * returns the that cpumask for the given CPU.
 687 */
 688static void set_cpus_related(int i, int j, struct cpumask *(*get_cpumask)(int))
 689{
 690	cpumask_set_cpu(i, get_cpumask(j));
 691	cpumask_set_cpu(j, get_cpumask(i));
 692}
 693
 694#ifdef CONFIG_HOTPLUG_CPU
 695static void set_cpus_unrelated(int i, int j,
 696		struct cpumask *(*get_cpumask)(int))
 697{
 698	cpumask_clear_cpu(i, get_cpumask(j));
 699	cpumask_clear_cpu(j, get_cpumask(i));
 700}
 701#endif
 702
 703/*
 704 * Extends set_cpus_related. Instead of setting one CPU at a time in
 705 * dstmask, set srcmask at oneshot. dstmask should be super set of srcmask.
 706 */
 707static void or_cpumasks_related(int i, int j, struct cpumask *(*srcmask)(int),
 708				struct cpumask *(*dstmask)(int))
 709{
 710	struct cpumask *mask;
 711	int k;
 712
 713	mask = srcmask(j);
 714	for_each_cpu(k, srcmask(i))
 715		cpumask_or(dstmask(k), dstmask(k), mask);
 716
 717	if (i == j)
 718		return;
 719
 720	mask = srcmask(i);
 721	for_each_cpu(k, srcmask(j))
 722		cpumask_or(dstmask(k), dstmask(k), mask);
 723}
 724
 725/*
 726 * parse_thread_groups: Parses the "ibm,thread-groups" device tree
 727 *                      property for the CPU device node @dn and stores
 728 *                      the parsed output in the thread_groups_list
 729 *                      structure @tglp.
 730 *
 731 * @dn: The device node of the CPU device.
 732 * @tglp: Pointer to a thread group list structure into which the parsed
 733 *      output of "ibm,thread-groups" is stored.
 734 *
 735 * ibm,thread-groups[0..N-1] array defines which group of threads in
 736 * the CPU-device node can be grouped together based on the property.
 737 *
 738 * This array can represent thread groupings for multiple properties.
 739 *
 740 * ibm,thread-groups[i + 0] tells us the property based on which the
 741 * threads are being grouped together. If this value is 1, it implies
 742 * that the threads in the same group share L1, translation cache. If
 743 * the value is 2, it implies that the threads in the same group share
 744 * the same L2 cache.
 745 *
 746 * ibm,thread-groups[i+1] tells us how many such thread groups exist for the
 747 * property ibm,thread-groups[i]
 748 *
 749 * ibm,thread-groups[i+2] tells us the number of threads in each such
 750 * group.
 751 * Suppose k = (ibm,thread-groups[i+1] * ibm,thread-groups[i+2]), then,
 752 *
 753 * ibm,thread-groups[i+3..i+k+2] (is the list of threads identified by
 754 * "ibm,ppc-interrupt-server#s" arranged as per their membership in
 755 * the grouping.
 756 *
 757 * Example:
 758 * If "ibm,thread-groups" = [1,2,4,8,10,12,14,9,11,13,15,2,2,4,8,10,12,14,9,11,13,15]
 759 * This can be decomposed up into two consecutive arrays:
 760 * a) [1,2,4,8,10,12,14,9,11,13,15]
 761 * b) [2,2,4,8,10,12,14,9,11,13,15]
 762 *
 763 * where in,
 764 *
 765 * a) provides information of Property "1" being shared by "2" groups,
 766 *  each with "4" threads each. The "ibm,ppc-interrupt-server#s" of
 767 *  the first group is {8,10,12,14} and the
 768 *  "ibm,ppc-interrupt-server#s" of the second group is
 769 *  {9,11,13,15}. Property "1" is indicative of the thread in the
 770 *  group sharing L1 cache, translation cache and Instruction Data
 771 *  flow.
 772 *
 773 * b) provides information of Property "2" being shared by "2" groups,
 774 *  each group with "4" threads. The "ibm,ppc-interrupt-server#s" of
 775 *  the first group is {8,10,12,14} and the
 776 *  "ibm,ppc-interrupt-server#s" of the second group is
 777 *  {9,11,13,15}. Property "2" indicates that the threads in each
 778 *  group share the L2-cache.
 779 *
 780 * Returns 0 on success, -EINVAL if the property does not exist,
 781 * -ENODATA if property does not have a value, and -EOVERFLOW if the
 782 * property data isn't large enough.
 783 */
 784static int parse_thread_groups(struct device_node *dn,
 785			       struct thread_groups_list *tglp)
 786{
 787	unsigned int property_idx = 0;
 788	u32 *thread_group_array;
 789	size_t total_threads;
 790	int ret = 0, count;
 791	u32 *thread_list;
 792	int i = 0;
 793
 794	count = of_property_count_u32_elems(dn, "ibm,thread-groups");
 795	thread_group_array = kcalloc(count, sizeof(u32), GFP_KERNEL);
 796	ret = of_property_read_u32_array(dn, "ibm,thread-groups",
 797					 thread_group_array, count);
 798	if (ret)
 799		goto out_free;
 800
 801	while (i < count && property_idx < MAX_THREAD_GROUP_PROPERTIES) {
 802		int j;
 803		struct thread_groups *tg = &tglp->property_tgs[property_idx++];
 804
 805		tg->property = thread_group_array[i];
 806		tg->nr_groups = thread_group_array[i + 1];
 807		tg->threads_per_group = thread_group_array[i + 2];
 808		total_threads = tg->nr_groups * tg->threads_per_group;
 809
 810		thread_list = &thread_group_array[i + 3];
 811
 812		for (j = 0; j < total_threads; j++)
 813			tg->thread_list[j] = thread_list[j];
 814		i = i + 3 + total_threads;
 815	}
 816
 817	tglp->nr_properties = property_idx;
 818
 819out_free:
 820	kfree(thread_group_array);
 821	return ret;
 822}
 823
 824/*
 825 * get_cpu_thread_group_start : Searches the thread group in tg->thread_list
 826 *                              that @cpu belongs to.
 827 *
 828 * @cpu : The logical CPU whose thread group is being searched.
 829 * @tg : The thread-group structure of the CPU node which @cpu belongs
 830 *       to.
 831 *
 832 * Returns the index to tg->thread_list that points to the the start
 833 * of the thread_group that @cpu belongs to.
 834 *
 835 * Returns -1 if cpu doesn't belong to any of the groups pointed to by
 836 * tg->thread_list.
 837 */
 838static int get_cpu_thread_group_start(int cpu, struct thread_groups *tg)
 839{
 840	int hw_cpu_id = get_hard_smp_processor_id(cpu);
 841	int i, j;
 842
 843	for (i = 0; i < tg->nr_groups; i++) {
 844		int group_start = i * tg->threads_per_group;
 845
 846		for (j = 0; j < tg->threads_per_group; j++) {
 847			int idx = group_start + j;
 848
 849			if (tg->thread_list[idx] == hw_cpu_id)
 850				return group_start;
 851		}
 852	}
 853
 854	return -1;
 855}
 856
 857static struct thread_groups *__init get_thread_groups(int cpu,
 858						      int group_property,
 859						      int *err)
 860{
 861	struct device_node *dn = of_get_cpu_node(cpu, NULL);
 862	struct thread_groups_list *cpu_tgl = &tgl[cpu];
 863	struct thread_groups *tg = NULL;
 864	int i;
 865	*err = 0;
 866
 867	if (!dn) {
 868		*err = -ENODATA;
 869		return NULL;
 870	}
 871
 872	if (!cpu_tgl->nr_properties) {
 873		*err = parse_thread_groups(dn, cpu_tgl);
 874		if (*err)
 875			goto out;
 876	}
 877
 878	for (i = 0; i < cpu_tgl->nr_properties; i++) {
 879		if (cpu_tgl->property_tgs[i].property == group_property) {
 880			tg = &cpu_tgl->property_tgs[i];
 881			break;
 882		}
 883	}
 884
 885	if (!tg)
 886		*err = -EINVAL;
 887out:
 888	of_node_put(dn);
 889	return tg;
 890}
 891
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 892static int __init init_thread_group_cache_map(int cpu, int cache_property)
 893
 894{
 895	int first_thread = cpu_first_thread_sibling(cpu);
 896	int i, cpu_group_start = -1, err = 0;
 897	struct thread_groups *tg = NULL;
 898	cpumask_var_t *mask = NULL;
 899
 900	if (cache_property != THREAD_GROUP_SHARE_L1 &&
 901	    cache_property != THREAD_GROUP_SHARE_L2)
 902		return -EINVAL;
 903
 904	tg = get_thread_groups(cpu, cache_property, &err);
 
 905	if (!tg)
 906		return err;
 907
 908	cpu_group_start = get_cpu_thread_group_start(cpu, tg);
 909
 910	if (unlikely(cpu_group_start == -1)) {
 911		WARN_ON_ONCE(1);
 912		return -ENODATA;
 913	}
 914
 915	if (cache_property == THREAD_GROUP_SHARE_L1)
 916		mask = &per_cpu(thread_group_l1_cache_map, cpu);
 917	else if (cache_property == THREAD_GROUP_SHARE_L2)
 
 
 918		mask = &per_cpu(thread_group_l2_cache_map, cpu);
 919
 920	zalloc_cpumask_var_node(mask, GFP_KERNEL, cpu_to_node(cpu));
 921
 922	for (i = first_thread; i < first_thread + threads_per_core; i++) {
 923		int i_group_start = get_cpu_thread_group_start(i, tg);
 924
 925		if (unlikely(i_group_start == -1)) {
 926			WARN_ON_ONCE(1);
 927			return -ENODATA;
 928		}
 929
 930		if (i_group_start == cpu_group_start)
 931			cpumask_set_cpu(i, *mask);
 932	}
 933
 
 934	return 0;
 935}
 936
 937static bool shared_caches;
 938
 939#ifdef CONFIG_SCHED_SMT
 940/* cpumask of CPUs with asymmetric SMT dependency */
 941static int powerpc_smt_flags(void)
 942{
 943	int flags = SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
 944
 945	if (cpu_has_feature(CPU_FTR_ASYM_SMT)) {
 946		printk_once(KERN_INFO "Enabling Asymmetric SMT scheduling\n");
 947		flags |= SD_ASYM_PACKING;
 948	}
 949	return flags;
 950}
 951#endif
 952
 953/*
 
 
 
 
 
 
 
 954 * P9 has a slightly odd architecture where pairs of cores share an L2 cache.
 955 * This topology makes it *much* cheaper to migrate tasks between adjacent cores
 956 * since the migrated task remains cache hot. We want to take advantage of this
 957 * at the scheduler level so an extra topology level is required.
 958 */
 959static int powerpc_shared_cache_flags(void)
 960{
 
 
 
 961	return SD_SHARE_PKG_RESOURCES;
 962}
 963
 
 
 
 
 
 
 
 
 964/*
 965 * We can't just pass cpu_l2_cache_mask() directly because
 966 * returns a non-const pointer and the compiler barfs on that.
 967 */
 968static const struct cpumask *shared_cache_mask(int cpu)
 969{
 970	return per_cpu(cpu_l2_cache_map, cpu);
 971}
 972
 973#ifdef CONFIG_SCHED_SMT
 974static const struct cpumask *smallcore_smt_mask(int cpu)
 975{
 976	return cpu_smallcore_mask(cpu);
 977}
 978#endif
 979
 980static struct cpumask *cpu_coregroup_mask(int cpu)
 981{
 982	return per_cpu(cpu_coregroup_map, cpu);
 983}
 984
 985static bool has_coregroup_support(void)
 986{
 
 
 
 
 987	return coregroup_enabled;
 988}
 989
 990static const struct cpumask *cpu_mc_mask(int cpu)
 991{
 992	return cpu_coregroup_mask(cpu);
 993}
 994
 995static struct sched_domain_topology_level powerpc_topology[] = {
 996#ifdef CONFIG_SCHED_SMT
 997	{ cpu_smt_mask, powerpc_smt_flags, SD_INIT_NAME(SMT) },
 998#endif
 999	{ shared_cache_mask, powerpc_shared_cache_flags, SD_INIT_NAME(CACHE) },
1000	{ cpu_mc_mask, SD_INIT_NAME(MC) },
1001	{ cpu_cpu_mask, SD_INIT_NAME(DIE) },
1002	{ NULL, },
1003};
1004
1005static int __init init_big_cores(void)
1006{
1007	int cpu;
1008
1009	for_each_possible_cpu(cpu) {
1010		int err = init_thread_group_cache_map(cpu, THREAD_GROUP_SHARE_L1);
1011
1012		if (err)
1013			return err;
1014
1015		zalloc_cpumask_var_node(&per_cpu(cpu_smallcore_map, cpu),
1016					GFP_KERNEL,
1017					cpu_to_node(cpu));
1018	}
1019
1020	has_big_cores = true;
1021
1022	for_each_possible_cpu(cpu) {
1023		int err = init_thread_group_cache_map(cpu, THREAD_GROUP_SHARE_L2);
1024
1025		if (err)
1026			return err;
1027	}
1028
1029	thread_group_shares_l2 = true;
1030	pr_debug("L2 cache only shared by the threads in the small core\n");
 
 
1031	return 0;
1032}
1033
1034void __init smp_prepare_cpus(unsigned int max_cpus)
1035{
1036	unsigned int cpu;
1037
1038	DBG("smp_prepare_cpus\n");
1039
1040	/* 
1041	 * setup_cpu may need to be called on the boot cpu. We havent
1042	 * spun any cpus up but lets be paranoid.
1043	 */
1044	BUG_ON(boot_cpuid != smp_processor_id());
1045
1046	/* Fixup boot cpu */
1047	smp_store_cpu_info(boot_cpuid);
1048	cpu_callin_map[boot_cpuid] = 1;
1049
1050	for_each_possible_cpu(cpu) {
1051		zalloc_cpumask_var_node(&per_cpu(cpu_sibling_map, cpu),
1052					GFP_KERNEL, cpu_to_node(cpu));
1053		zalloc_cpumask_var_node(&per_cpu(cpu_l2_cache_map, cpu),
1054					GFP_KERNEL, cpu_to_node(cpu));
1055		zalloc_cpumask_var_node(&per_cpu(cpu_core_map, cpu),
1056					GFP_KERNEL, cpu_to_node(cpu));
1057		if (has_coregroup_support())
1058			zalloc_cpumask_var_node(&per_cpu(cpu_coregroup_map, cpu),
1059						GFP_KERNEL, cpu_to_node(cpu));
1060
1061#ifdef CONFIG_NUMA
1062		/*
1063		 * numa_node_id() works after this.
1064		 */
1065		if (cpu_present(cpu)) {
1066			set_cpu_numa_node(cpu, numa_cpu_lookup_table[cpu]);
1067			set_cpu_numa_mem(cpu,
1068				local_memory_node(numa_cpu_lookup_table[cpu]));
1069		}
1070#endif
1071	}
1072
1073	/* Init the cpumasks so the boot CPU is related to itself */
1074	cpumask_set_cpu(boot_cpuid, cpu_sibling_mask(boot_cpuid));
1075	cpumask_set_cpu(boot_cpuid, cpu_l2_cache_mask(boot_cpuid));
1076	cpumask_set_cpu(boot_cpuid, cpu_core_mask(boot_cpuid));
1077
1078	if (has_coregroup_support())
1079		cpumask_set_cpu(boot_cpuid, cpu_coregroup_mask(boot_cpuid));
1080
1081	init_big_cores();
1082	if (has_big_cores) {
1083		cpumask_set_cpu(boot_cpuid,
1084				cpu_smallcore_mask(boot_cpuid));
1085	}
1086
1087	if (cpu_to_chip_id(boot_cpuid) != -1) {
1088		int idx = DIV_ROUND_UP(num_possible_cpus(), threads_per_core);
1089
1090		/*
1091		 * All threads of a core will all belong to the same core,
1092		 * chip_id_lookup_table will have one entry per core.
1093		 * Assumption: if boot_cpuid doesn't have a chip-id, then no
1094		 * other CPUs, will also not have chip-id.
1095		 */
1096		chip_id_lookup_table = kcalloc(idx, sizeof(int), GFP_KERNEL);
1097		if (chip_id_lookup_table)
1098			memset(chip_id_lookup_table, -1, sizeof(int) * idx);
1099	}
1100
1101	if (smp_ops && smp_ops->probe)
1102		smp_ops->probe();
 
 
 
 
 
 
1103}
1104
1105void smp_prepare_boot_cpu(void)
1106{
1107	BUG_ON(smp_processor_id() != boot_cpuid);
1108#ifdef CONFIG_PPC64
1109	paca_ptrs[boot_cpuid]->__current = current;
1110#endif
1111	set_numa_node(numa_cpu_lookup_table[boot_cpuid]);
1112	current_set[boot_cpuid] = current;
1113}
1114
1115#ifdef CONFIG_HOTPLUG_CPU
1116
1117int generic_cpu_disable(void)
1118{
1119	unsigned int cpu = smp_processor_id();
1120
1121	if (cpu == boot_cpuid)
1122		return -EBUSY;
1123
1124	set_cpu_online(cpu, false);
1125#ifdef CONFIG_PPC64
1126	vdso_data->processorCount--;
1127#endif
1128	/* Update affinity of all IRQs previously aimed at this CPU */
1129	irq_migrate_all_off_this_cpu();
1130
1131	/*
1132	 * Depending on the details of the interrupt controller, it's possible
1133	 * that one of the interrupts we just migrated away from this CPU is
1134	 * actually already pending on this CPU. If we leave it in that state
1135	 * the interrupt will never be EOI'ed, and will never fire again. So
1136	 * temporarily enable interrupts here, to allow any pending interrupt to
1137	 * be received (and EOI'ed), before we take this CPU offline.
1138	 */
1139	local_irq_enable();
1140	mdelay(1);
1141	local_irq_disable();
1142
1143	return 0;
1144}
1145
1146void generic_cpu_die(unsigned int cpu)
1147{
1148	int i;
1149
1150	for (i = 0; i < 100; i++) {
1151		smp_rmb();
1152		if (is_cpu_dead(cpu))
1153			return;
1154		msleep(100);
1155	}
1156	printk(KERN_ERR "CPU%d didn't die...\n", cpu);
1157}
1158
1159void generic_set_cpu_dead(unsigned int cpu)
1160{
1161	per_cpu(cpu_state, cpu) = CPU_DEAD;
1162}
1163
1164/*
1165 * The cpu_state should be set to CPU_UP_PREPARE in kick_cpu(), otherwise
1166 * the cpu_state is always CPU_DEAD after calling generic_set_cpu_dead(),
1167 * which makes the delay in generic_cpu_die() not happen.
1168 */
1169void generic_set_cpu_up(unsigned int cpu)
1170{
1171	per_cpu(cpu_state, cpu) = CPU_UP_PREPARE;
1172}
1173
1174int generic_check_cpu_restart(unsigned int cpu)
1175{
1176	return per_cpu(cpu_state, cpu) == CPU_UP_PREPARE;
1177}
1178
1179int is_cpu_dead(unsigned int cpu)
1180{
1181	return per_cpu(cpu_state, cpu) == CPU_DEAD;
1182}
1183
1184static bool secondaries_inhibited(void)
1185{
1186	return kvm_hv_mode_active();
1187}
1188
1189#else /* HOTPLUG_CPU */
1190
1191#define secondaries_inhibited()		0
1192
1193#endif
1194
1195static void cpu_idle_thread_init(unsigned int cpu, struct task_struct *idle)
1196{
1197#ifdef CONFIG_PPC64
1198	paca_ptrs[cpu]->__current = idle;
1199	paca_ptrs[cpu]->kstack = (unsigned long)task_stack_page(idle) +
1200				 THREAD_SIZE - STACK_FRAME_OVERHEAD;
1201#endif
1202	idle->cpu = cpu;
1203	secondary_current = current_set[cpu] = idle;
1204}
1205
1206int __cpu_up(unsigned int cpu, struct task_struct *tidle)
1207{
1208	int rc, c;
 
 
 
 
 
1209
1210	/*
1211	 * Don't allow secondary threads to come online if inhibited
1212	 */
1213	if (threads_per_core > 1 && secondaries_inhibited() &&
1214	    cpu_thread_in_subcore(cpu))
1215		return -EBUSY;
1216
1217	if (smp_ops == NULL ||
1218	    (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)))
1219		return -EINVAL;
1220
1221	cpu_idle_thread_init(cpu, tidle);
1222
1223	/*
1224	 * The platform might need to allocate resources prior to bringing
1225	 * up the CPU
1226	 */
1227	if (smp_ops->prepare_cpu) {
1228		rc = smp_ops->prepare_cpu(cpu);
1229		if (rc)
1230			return rc;
1231	}
1232
1233	/* Make sure callin-map entry is 0 (can be leftover a CPU
1234	 * hotplug
1235	 */
1236	cpu_callin_map[cpu] = 0;
1237
1238	/* The information for processor bringup must
1239	 * be written out to main store before we release
1240	 * the processor.
1241	 */
1242	smp_mb();
1243
1244	/* wake up cpus */
1245	DBG("smp: kicking cpu %d\n", cpu);
1246	rc = smp_ops->kick_cpu(cpu);
1247	if (rc) {
1248		pr_err("smp: failed starting cpu %d (rc %d)\n", cpu, rc);
1249		return rc;
1250	}
1251
1252	/*
1253	 * wait to see if the cpu made a callin (is actually up).
1254	 * use this value that I found through experimentation.
1255	 * -- Cort
1256	 */
1257	if (system_state < SYSTEM_RUNNING)
1258		for (c = 50000; c && !cpu_callin_map[cpu]; c--)
1259			udelay(100);
1260#ifdef CONFIG_HOTPLUG_CPU
1261	else
1262		/*
1263		 * CPUs can take much longer to come up in the
1264		 * hotplug case.  Wait five seconds.
1265		 */
1266		for (c = 5000; c && !cpu_callin_map[cpu]; c--)
1267			msleep(1);
1268#endif
 
1269
1270	if (!cpu_callin_map[cpu]) {
1271		printk(KERN_ERR "Processor %u is stuck.\n", cpu);
1272		return -ENOENT;
1273	}
1274
1275	DBG("Processor %u found.\n", cpu);
1276
1277	if (smp_ops->give_timebase)
1278		smp_ops->give_timebase();
1279
1280	/* Wait until cpu puts itself in the online & active maps */
1281	spin_until_cond(cpu_online(cpu));
1282
1283	return 0;
1284}
1285
1286/* Return the value of the reg property corresponding to the given
1287 * logical cpu.
1288 */
1289int cpu_to_core_id(int cpu)
1290{
1291	struct device_node *np;
1292	const __be32 *reg;
1293	int id = -1;
1294
1295	np = of_get_cpu_node(cpu, NULL);
1296	if (!np)
1297		goto out;
1298
1299	reg = of_get_property(np, "reg", NULL);
1300	if (!reg)
1301		goto out;
1302
1303	id = be32_to_cpup(reg);
1304out:
1305	of_node_put(np);
1306	return id;
1307}
1308EXPORT_SYMBOL_GPL(cpu_to_core_id);
1309
1310/* Helper routines for cpu to core mapping */
1311int cpu_core_index_of_thread(int cpu)
1312{
1313	return cpu >> threads_shift;
1314}
1315EXPORT_SYMBOL_GPL(cpu_core_index_of_thread);
1316
1317int cpu_first_thread_of_core(int core)
1318{
1319	return core << threads_shift;
1320}
1321EXPORT_SYMBOL_GPL(cpu_first_thread_of_core);
1322
1323/* Must be called when no change can occur to cpu_present_mask,
1324 * i.e. during cpu online or offline.
1325 */
1326static struct device_node *cpu_to_l2cache(int cpu)
1327{
1328	struct device_node *np;
1329	struct device_node *cache;
1330
1331	if (!cpu_present(cpu))
1332		return NULL;
1333
1334	np = of_get_cpu_node(cpu, NULL);
1335	if (np == NULL)
1336		return NULL;
1337
1338	cache = of_find_next_cache_node(np);
1339
1340	of_node_put(np);
1341
1342	return cache;
1343}
1344
1345static bool update_mask_by_l2(int cpu, cpumask_var_t *mask)
1346{
1347	struct cpumask *(*submask_fn)(int) = cpu_sibling_mask;
1348	struct device_node *l2_cache, *np;
1349	int i;
1350
1351	if (has_big_cores)
1352		submask_fn = cpu_smallcore_mask;
1353
1354	/*
1355	 * If the threads in a thread-group share L2 cache, then the
1356	 * L2-mask can be obtained from thread_group_l2_cache_map.
1357	 */
1358	if (thread_group_shares_l2) {
1359		cpumask_set_cpu(cpu, cpu_l2_cache_mask(cpu));
1360
1361		for_each_cpu(i, per_cpu(thread_group_l2_cache_map, cpu)) {
1362			if (cpu_online(i))
1363				set_cpus_related(i, cpu, cpu_l2_cache_mask);
1364		}
1365
1366		/* Verify that L1-cache siblings are a subset of L2 cache-siblings */
1367		if (!cpumask_equal(submask_fn(cpu), cpu_l2_cache_mask(cpu)) &&
1368		    !cpumask_subset(submask_fn(cpu), cpu_l2_cache_mask(cpu))) {
1369			pr_warn_once("CPU %d : Inconsistent L1 and L2 cache siblings\n",
1370				     cpu);
1371		}
1372
1373		return true;
1374	}
1375
1376	l2_cache = cpu_to_l2cache(cpu);
1377	if (!l2_cache || !*mask) {
1378		/* Assume only core siblings share cache with this CPU */
1379		for_each_cpu(i, submask_fn(cpu))
1380			set_cpus_related(cpu, i, cpu_l2_cache_mask);
1381
1382		return false;
1383	}
1384
1385	cpumask_and(*mask, cpu_online_mask, cpu_cpu_mask(cpu));
1386
1387	/* Update l2-cache mask with all the CPUs that are part of submask */
1388	or_cpumasks_related(cpu, cpu, submask_fn, cpu_l2_cache_mask);
1389
1390	/* Skip all CPUs already part of current CPU l2-cache mask */
1391	cpumask_andnot(*mask, *mask, cpu_l2_cache_mask(cpu));
1392
1393	for_each_cpu(i, *mask) {
1394		/*
1395		 * when updating the marks the current CPU has not been marked
1396		 * online, but we need to update the cache masks
1397		 */
1398		np = cpu_to_l2cache(i);
1399
1400		/* Skip all CPUs already part of current CPU l2-cache */
1401		if (np == l2_cache) {
1402			or_cpumasks_related(cpu, i, submask_fn, cpu_l2_cache_mask);
1403			cpumask_andnot(*mask, *mask, submask_fn(i));
1404		} else {
1405			cpumask_andnot(*mask, *mask, cpu_l2_cache_mask(i));
1406		}
1407
1408		of_node_put(np);
1409	}
1410	of_node_put(l2_cache);
1411
1412	return true;
1413}
1414
1415#ifdef CONFIG_HOTPLUG_CPU
1416static void remove_cpu_from_masks(int cpu)
1417{
1418	struct cpumask *(*mask_fn)(int) = cpu_sibling_mask;
1419	int i;
1420
 
 
1421	if (shared_caches)
1422		mask_fn = cpu_l2_cache_mask;
1423
1424	for_each_cpu(i, mask_fn(cpu)) {
1425		set_cpus_unrelated(cpu, i, cpu_l2_cache_mask);
1426		set_cpus_unrelated(cpu, i, cpu_sibling_mask);
1427		if (has_big_cores)
1428			set_cpus_unrelated(cpu, i, cpu_smallcore_mask);
1429	}
1430
1431	for_each_cpu(i, cpu_core_mask(cpu))
1432		set_cpus_unrelated(cpu, i, cpu_core_mask);
1433
1434	if (has_coregroup_support()) {
1435		for_each_cpu(i, cpu_coregroup_mask(cpu))
1436			set_cpus_unrelated(cpu, i, cpu_coregroup_mask);
1437	}
1438}
1439#endif
1440
1441static inline void add_cpu_to_smallcore_masks(int cpu)
1442{
1443	int i;
1444
1445	if (!has_big_cores)
1446		return;
1447
1448	cpumask_set_cpu(cpu, cpu_smallcore_mask(cpu));
1449
1450	for_each_cpu(i, per_cpu(thread_group_l1_cache_map, cpu)) {
1451		if (cpu_online(i))
1452			set_cpus_related(i, cpu, cpu_smallcore_mask);
1453	}
1454}
1455
1456static void update_coregroup_mask(int cpu, cpumask_var_t *mask)
1457{
1458	struct cpumask *(*submask_fn)(int) = cpu_sibling_mask;
1459	int coregroup_id = cpu_to_coregroup_id(cpu);
1460	int i;
1461
1462	if (shared_caches)
1463		submask_fn = cpu_l2_cache_mask;
1464
1465	if (!*mask) {
1466		/* Assume only siblings are part of this CPU's coregroup */
1467		for_each_cpu(i, submask_fn(cpu))
1468			set_cpus_related(cpu, i, cpu_coregroup_mask);
1469
1470		return;
1471	}
1472
1473	cpumask_and(*mask, cpu_online_mask, cpu_cpu_mask(cpu));
1474
1475	/* Update coregroup mask with all the CPUs that are part of submask */
1476	or_cpumasks_related(cpu, cpu, submask_fn, cpu_coregroup_mask);
1477
1478	/* Skip all CPUs already part of coregroup mask */
1479	cpumask_andnot(*mask, *mask, cpu_coregroup_mask(cpu));
1480
1481	for_each_cpu(i, *mask) {
1482		/* Skip all CPUs not part of this coregroup */
1483		if (coregroup_id == cpu_to_coregroup_id(i)) {
1484			or_cpumasks_related(cpu, i, submask_fn, cpu_coregroup_mask);
1485			cpumask_andnot(*mask, *mask, submask_fn(i));
1486		} else {
1487			cpumask_andnot(*mask, *mask, cpu_coregroup_mask(i));
1488		}
1489	}
1490}
1491
1492static void add_cpu_to_masks(int cpu)
1493{
1494	struct cpumask *(*submask_fn)(int) = cpu_sibling_mask;
1495	int first_thread = cpu_first_thread_sibling(cpu);
1496	cpumask_var_t mask;
1497	int chip_id = -1;
1498	bool ret;
1499	int i;
1500
1501	/*
1502	 * This CPU will not be in the online mask yet so we need to manually
1503	 * add it to it's own thread sibling mask.
1504	 */
 
1505	cpumask_set_cpu(cpu, cpu_sibling_mask(cpu));
1506	cpumask_set_cpu(cpu, cpu_core_mask(cpu));
1507
1508	for (i = first_thread; i < first_thread + threads_per_core; i++)
1509		if (cpu_online(i))
1510			set_cpus_related(i, cpu, cpu_sibling_mask);
1511
1512	add_cpu_to_smallcore_masks(cpu);
1513
1514	/* In CPU-hotplug path, hence use GFP_ATOMIC */
1515	ret = alloc_cpumask_var_node(&mask, GFP_ATOMIC, cpu_to_node(cpu));
1516	update_mask_by_l2(cpu, &mask);
1517
1518	if (has_coregroup_support())
1519		update_coregroup_mask(cpu, &mask);
1520
1521	if (chip_id_lookup_table && ret)
1522		chip_id = cpu_to_chip_id(cpu);
1523
1524	if (shared_caches)
1525		submask_fn = cpu_l2_cache_mask;
1526
1527	/* Update core_mask with all the CPUs that are part of submask */
1528	or_cpumasks_related(cpu, cpu, submask_fn, cpu_core_mask);
1529
1530	/* Skip all CPUs already part of current CPU core mask */
1531	cpumask_andnot(mask, cpu_online_mask, cpu_core_mask(cpu));
1532
1533	/* If chip_id is -1; limit the cpu_core_mask to within DIE*/
1534	if (chip_id == -1)
1535		cpumask_and(mask, mask, cpu_cpu_mask(cpu));
1536
1537	for_each_cpu(i, mask) {
1538		if (chip_id == cpu_to_chip_id(i)) {
1539			or_cpumasks_related(cpu, i, submask_fn, cpu_core_mask);
1540			cpumask_andnot(mask, mask, submask_fn(i));
1541		} else {
1542			cpumask_andnot(mask, mask, cpu_core_mask(i));
1543		}
1544	}
1545
1546	free_cpumask_var(mask);
1547}
1548
1549/* Activate a secondary processor. */
 
1550void start_secondary(void *unused)
1551{
1552	unsigned int cpu = raw_smp_processor_id();
1553
1554	/* PPC64 calls setup_kup() in early_setup_secondary() */
1555	if (IS_ENABLED(CONFIG_PPC32))
1556		setup_kup();
1557
1558	mmgrab(&init_mm);
1559	current->active_mm = &init_mm;
 
 
 
1560
1561	smp_store_cpu_info(cpu);
1562	set_dec(tb_ticks_per_jiffy);
1563	rcu_cpu_starting(cpu);
1564	cpu_callin_map[cpu] = 1;
1565
1566	if (smp_ops->setup_cpu)
1567		smp_ops->setup_cpu(cpu);
1568	if (smp_ops->take_timebase)
1569		smp_ops->take_timebase();
1570
1571	secondary_cpu_time_init();
1572
1573#ifdef CONFIG_PPC64
1574	if (system_state == SYSTEM_RUNNING)
1575		vdso_data->processorCount++;
1576
1577	vdso_getcpu_init();
1578#endif
1579	set_numa_node(numa_cpu_lookup_table[cpu]);
1580	set_numa_mem(local_memory_node(numa_cpu_lookup_table[cpu]));
1581
1582	/* Update topology CPU masks */
1583	add_cpu_to_masks(cpu);
1584
1585	/*
1586	 * Check for any shared caches. Note that this must be done on a
1587	 * per-core basis because one core in the pair might be disabled.
1588	 */
1589	if (!shared_caches) {
1590		struct cpumask *(*sibling_mask)(int) = cpu_sibling_mask;
1591		struct cpumask *mask = cpu_l2_cache_mask(cpu);
1592
1593		if (has_big_cores)
1594			sibling_mask = cpu_smallcore_mask;
1595
1596		if (cpumask_weight(mask) > cpumask_weight(sibling_mask(cpu)))
1597			shared_caches = true;
1598	}
1599
1600	smp_wmb();
1601	notify_cpu_starting(cpu);
1602	set_cpu_online(cpu, true);
1603
1604	boot_init_stack_canary();
1605
1606	local_irq_enable();
1607
1608	/* We can enable ftrace for secondary cpus now */
1609	this_cpu_enable_ftrace();
1610
1611	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
1612
1613	BUG();
1614}
1615
1616int setup_profiling_timer(unsigned int multiplier)
1617{
1618	return 0;
1619}
1620
1621static void fixup_topology(void)
1622{
1623	int i;
 
 
 
1624
1625#ifdef CONFIG_SCHED_SMT
1626	if (has_big_cores) {
1627		pr_info("Big cores detected but using small core scheduling\n");
1628		powerpc_topology[smt_idx].mask = smallcore_smt_mask;
 
 
 
 
 
 
1629	}
1630#endif
 
 
 
 
 
 
 
 
 
 
 
 
 
1631
1632	if (!has_coregroup_support())
1633		powerpc_topology[mc_idx].mask = powerpc_topology[cache_idx].mask;
1634
1635	/*
1636	 * Try to consolidate topology levels here instead of
1637	 * allowing scheduler to degenerate.
1638	 * - Dont consolidate if masks are different.
1639	 * - Dont consolidate if sd_flags exists and are different.
1640	 */
1641	for (i = 1; i <= die_idx; i++) {
1642		if (powerpc_topology[i].mask != powerpc_topology[i - 1].mask)
1643			continue;
1644
1645		if (powerpc_topology[i].sd_flags && powerpc_topology[i - 1].sd_flags &&
1646				powerpc_topology[i].sd_flags != powerpc_topology[i - 1].sd_flags)
1647			continue;
1648
1649		if (!powerpc_topology[i - 1].sd_flags)
1650			powerpc_topology[i - 1].sd_flags = powerpc_topology[i].sd_flags;
1651
1652		powerpc_topology[i].mask = powerpc_topology[i + 1].mask;
1653		powerpc_topology[i].sd_flags = powerpc_topology[i + 1].sd_flags;
1654#ifdef CONFIG_SCHED_DEBUG
1655		powerpc_topology[i].name = powerpc_topology[i + 1].name;
1656#endif
1657	}
1658}
1659
1660void __init smp_cpus_done(unsigned int max_cpus)
1661{
1662	/*
1663	 * We are running pinned to the boot CPU, see rest_init().
1664	 */
1665	if (smp_ops && smp_ops->setup_cpu)
1666		smp_ops->setup_cpu(boot_cpuid);
1667
1668	if (smp_ops && smp_ops->bringup_done)
1669		smp_ops->bringup_done();
1670
1671	dump_numa_cpu_topology();
 
 
1672
1673	fixup_topology();
1674	set_sched_topology(powerpc_topology);
 
 
 
 
 
 
 
 
 
1675}
1676
1677#ifdef CONFIG_HOTPLUG_CPU
1678int __cpu_disable(void)
1679{
1680	int cpu = smp_processor_id();
1681	int err;
1682
1683	if (!smp_ops->cpu_disable)
1684		return -ENOSYS;
1685
1686	this_cpu_disable_ftrace();
1687
1688	err = smp_ops->cpu_disable();
1689	if (err)
1690		return err;
1691
1692	/* Update sibling maps */
1693	remove_cpu_from_masks(cpu);
1694
1695	return 0;
1696}
1697
1698void __cpu_die(unsigned int cpu)
1699{
 
 
 
 
 
 
 
 
1700	if (smp_ops->cpu_die)
1701		smp_ops->cpu_die(cpu);
1702}
1703
1704void arch_cpu_idle_dead(void)
1705{
1706	/*
1707	 * Disable on the down path. This will be re-enabled by
1708	 * start_secondary() via start_secondary_resume() below
1709	 */
1710	this_cpu_disable_ftrace();
1711
1712	if (smp_ops->cpu_offline_self)
1713		smp_ops->cpu_offline_self();
1714
1715	/* If we return, we re-enter start_secondary */
1716	start_secondary_resume();
1717}
1718
1719#endif