1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * intel_idle.c - native hardware idle loop for modern Intel processors
   4 *
   5 * Copyright (c) 2013 - 2020, Intel Corporation.
   6 * Len Brown <len.brown@intel.com>
   7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com>
   8 */
   9
  10/*
  11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
  12 * in lieu of the legacy ACPI processor_idle driver.  The intent is to
  13 * make Linux more efficient on these processors, as intel_idle knows
  14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
  15 */
  16
  17/*
  18 * Design Assumptions
  19 *
  20 * All CPUs have same idle states as boot CPU
  21 *
  22 * Chipset BM_STS (bus master status) bit is a NOP
  23 *	for preventing entry into deep C-states
  24 *
  25 * CPU will flush caches as needed when entering a C-state via MWAIT
  26 *	(in contrast to entering ACPI C3, in which case the WBINVD
  27 *	instruction needs to be executed to flush the caches)
  28 */
  29
  30/*
  31 * Known limitations
  32 *
  33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
  34 * to avoid complications with the lapic timer workaround.
  35 * Have not seen issues with suspend, but may need same workaround here.
  36 *
  37 */
  38
  39/* un-comment DEBUG to enable pr_debug() statements */
  40/* #define DEBUG */
  41
  42#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  43
  44#include <linux/acpi.h>
  45#include <linux/kernel.h>
  46#include <linux/cpuidle.h>
  47#include <linux/tick.h>
  48#include <trace/events/power.h>
  49#include <linux/sched.h>
  50#include <linux/sched/smt.h>
  51#include <linux/notifier.h>
  52#include <linux/cpu.h>
  53#include <linux/moduleparam.h>
  54#include <asm/cpu_device_id.h>
  55#include <asm/intel-family.h>
  56#include <asm/mwait.h>
  57#include <asm/spec-ctrl.h>
  58#include <asm/fpu/api.h>
  59
  60#define INTEL_IDLE_VERSION "0.5.1"
  61
  62static struct cpuidle_driver intel_idle_driver = {
  63	.name = "intel_idle",
  64	.owner = THIS_MODULE,
  65};
  66/* intel_idle.max_cstate=0 disables driver */
  67static int max_cstate = CPUIDLE_STATE_MAX - 1;
  68static unsigned int disabled_states_mask __read_mostly;
  69static unsigned int preferred_states_mask __read_mostly;
  70static bool force_irq_on __read_mostly;
  71static bool ibrs_off __read_mostly;
  72
  73static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
  74
  75static unsigned long auto_demotion_disable_flags;
  76
  77static enum {
  78	C1E_PROMOTION_PRESERVE,
  79	C1E_PROMOTION_ENABLE,
  80	C1E_PROMOTION_DISABLE
  81} c1e_promotion = C1E_PROMOTION_PRESERVE;
  82
  83struct idle_cpu {
  84	struct cpuidle_state *state_table;
  85
  86	/*
  87	 * Hardware C-state auto-demotion may not always be optimal.
  88	 * Indicate which enable bits to clear here.
  89	 */
  90	unsigned long auto_demotion_disable_flags;
  91	bool byt_auto_demotion_disable_flag;
  92	bool disable_promotion_to_c1e;
  93	bool use_acpi;
  94};
  95
  96static const struct idle_cpu *icpu __initdata;
  97static struct cpuidle_state *cpuidle_state_table __initdata;
  98
  99static unsigned int mwait_substates __initdata;
 100
 101/*
 102 * Enable interrupts before entering the C-state. On some platforms and for
 103 * some C-states, this may measurably decrease interrupt latency.
 104 */
 105#define CPUIDLE_FLAG_IRQ_ENABLE		BIT(14)
 106
 107/*
 108 * Enable this state by default even if the ACPI _CST does not list it.
 109 */
 110#define CPUIDLE_FLAG_ALWAYS_ENABLE	BIT(15)
 111
 112/*
 113 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
 114 * above.
 115 */
 116#define CPUIDLE_FLAG_IBRS		BIT(16)
 117
 118/*
 119 * Initialize large xstate for the C6-state entrance.
 120 */
 121#define CPUIDLE_FLAG_INIT_XSTATE	BIT(17)
 122
 123/*
 124 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
 125 * the C-state (top nibble) and sub-state (bottom nibble)
 126 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
 127 *
 128 * We store the hint at the top of our "flags" for each state.
 129 */
 130#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
 131#define MWAIT2flg(eax) ((eax & 0xFF) << 24)
 132
 133static __always_inline int __intel_idle(struct cpuidle_device *dev,
 134					struct cpuidle_driver *drv,
 135					int index, bool irqoff)
 136{
 137	struct cpuidle_state *state = &drv->states[index];
 138	unsigned long eax = flg2MWAIT(state->flags);
 139	unsigned long ecx = 1*irqoff; /* break on interrupt flag */
 140
 141	mwait_idle_with_hints(eax, ecx);
 142
 143	return index;
 144}
 145
 146/**
 147 * intel_idle - Ask the processor to enter the given idle state.
 148 * @dev: cpuidle device of the target CPU.
 149 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 150 * @index: Target idle state index.
 151 *
 152 * Use the MWAIT instruction to notify the processor that the CPU represented by
 153 * @dev is idle and it can try to enter the idle state corresponding to @index.
 154 *
 155 * If the local APIC timer is not known to be reliable in the target idle state,
 156 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
 157 *
 
 
 
 158 * Must be called under local_irq_disable().
 159 */
 160static __cpuidle int intel_idle(struct cpuidle_device *dev,
 161				struct cpuidle_driver *drv, int index)
 162{
 163	return __intel_idle(dev, drv, index, true);
 164}
 165
 166static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
 167				    struct cpuidle_driver *drv, int index)
 168{
 169	return __intel_idle(dev, drv, index, false);
 170}
 171
 172static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
 173				     struct cpuidle_driver *drv, int index)
 174{
 175	bool smt_active = sched_smt_active();
 176	u64 spec_ctrl = spec_ctrl_current();
 177	int ret;
 178
 179	if (smt_active)
 180		__update_spec_ctrl(0);
 181
 182	ret = __intel_idle(dev, drv, index, true);
 183
 184	if (smt_active)
 185		__update_spec_ctrl(spec_ctrl);
 186
 187	return ret;
 188}
 189
 190static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
 191				       struct cpuidle_driver *drv, int index)
 192{
 193	fpu_idle_fpregs();
 194	return __intel_idle(dev, drv, index, true);
 195}
 196
 197/**
 198 * intel_idle_s2idle - Ask the processor to enter the given idle state.
 199 * @dev: cpuidle device of the target CPU.
 200 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 201 * @index: Target idle state index.
 202 *
 203 * Use the MWAIT instruction to notify the processor that the CPU represented by
 204 * @dev is idle and it can try to enter the idle state corresponding to @index.
 205 *
 206 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
 207 * scheduler tick and suspended scheduler clock on the target CPU.
 208 */
 209static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
 210				       struct cpuidle_driver *drv, int index)
 211{
 
 212	unsigned long ecx = 1; /* break on interrupt flag */
 213	struct cpuidle_state *state = &drv->states[index];
 214	unsigned long eax = flg2MWAIT(state->flags);
 215
 216	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
 217		fpu_idle_fpregs();
 218
 219	mwait_idle_with_hints(eax, ecx);
 220
 221	return 0;
 222}
 223
 224/*
 225 * States are indexed by the cstate number,
 226 * which is also the index into the MWAIT hint array.
 227 * Thus C0 is a dummy.
 228 */
 229static struct cpuidle_state nehalem_cstates[] __initdata = {
 230	{
 231		.name = "C1",
 232		.desc = "MWAIT 0x00",
 233		.flags = MWAIT2flg(0x00),
 234		.exit_latency = 3,
 235		.target_residency = 6,
 236		.enter = &intel_idle,
 237		.enter_s2idle = intel_idle_s2idle, },
 238	{
 239		.name = "C1E",
 240		.desc = "MWAIT 0x01",
 241		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 242		.exit_latency = 10,
 243		.target_residency = 20,
 244		.enter = &intel_idle,
 245		.enter_s2idle = intel_idle_s2idle, },
 246	{
 247		.name = "C3",
 248		.desc = "MWAIT 0x10",
 249		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 250		.exit_latency = 20,
 251		.target_residency = 80,
 252		.enter = &intel_idle,
 253		.enter_s2idle = intel_idle_s2idle, },
 254	{
 255		.name = "C6",
 256		.desc = "MWAIT 0x20",
 257		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 258		.exit_latency = 200,
 259		.target_residency = 800,
 260		.enter = &intel_idle,
 261		.enter_s2idle = intel_idle_s2idle, },
 262	{
 263		.enter = NULL }
 264};
 265
 266static struct cpuidle_state snb_cstates[] __initdata = {
 267	{
 268		.name = "C1",
 269		.desc = "MWAIT 0x00",
 270		.flags = MWAIT2flg(0x00),
 271		.exit_latency = 2,
 272		.target_residency = 2,
 273		.enter = &intel_idle,
 274		.enter_s2idle = intel_idle_s2idle, },
 275	{
 276		.name = "C1E",
 277		.desc = "MWAIT 0x01",
 278		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 279		.exit_latency = 10,
 280		.target_residency = 20,
 281		.enter = &intel_idle,
 282		.enter_s2idle = intel_idle_s2idle, },
 283	{
 284		.name = "C3",
 285		.desc = "MWAIT 0x10",
 286		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 287		.exit_latency = 80,
 288		.target_residency = 211,
 289		.enter = &intel_idle,
 290		.enter_s2idle = intel_idle_s2idle, },
 291	{
 292		.name = "C6",
 293		.desc = "MWAIT 0x20",
 294		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 295		.exit_latency = 104,
 296		.target_residency = 345,
 297		.enter = &intel_idle,
 298		.enter_s2idle = intel_idle_s2idle, },
 299	{
 300		.name = "C7",
 301		.desc = "MWAIT 0x30",
 302		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
 303		.exit_latency = 109,
 304		.target_residency = 345,
 305		.enter = &intel_idle,
 306		.enter_s2idle = intel_idle_s2idle, },
 307	{
 308		.enter = NULL }
 309};
 310
 311static struct cpuidle_state byt_cstates[] __initdata = {
 312	{
 313		.name = "C1",
 314		.desc = "MWAIT 0x00",
 315		.flags = MWAIT2flg(0x00),
 316		.exit_latency = 1,
 317		.target_residency = 1,
 318		.enter = &intel_idle,
 319		.enter_s2idle = intel_idle_s2idle, },
 320	{
 321		.name = "C6N",
 322		.desc = "MWAIT 0x58",
 323		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
 324		.exit_latency = 300,
 325		.target_residency = 275,
 326		.enter = &intel_idle,
 327		.enter_s2idle = intel_idle_s2idle, },
 328	{
 329		.name = "C6S",
 330		.desc = "MWAIT 0x52",
 331		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
 332		.exit_latency = 500,
 333		.target_residency = 560,
 334		.enter = &intel_idle,
 335		.enter_s2idle = intel_idle_s2idle, },
 336	{
 337		.name = "C7",
 338		.desc = "MWAIT 0x60",
 339		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 340		.exit_latency = 1200,
 341		.target_residency = 4000,
 342		.enter = &intel_idle,
 343		.enter_s2idle = intel_idle_s2idle, },
 344	{
 345		.name = "C7S",
 346		.desc = "MWAIT 0x64",
 347		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
 348		.exit_latency = 10000,
 349		.target_residency = 20000,
 350		.enter = &intel_idle,
 351		.enter_s2idle = intel_idle_s2idle, },
 352	{
 353		.enter = NULL }
 354};
 355
 356static struct cpuidle_state cht_cstates[] __initdata = {
 357	{
 358		.name = "C1",
 359		.desc = "MWAIT 0x00",
 360		.flags = MWAIT2flg(0x00),
 361		.exit_latency = 1,
 362		.target_residency = 1,
 363		.enter = &intel_idle,
 364		.enter_s2idle = intel_idle_s2idle, },
 365	{
 366		.name = "C6N",
 367		.desc = "MWAIT 0x58",
 368		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
 369		.exit_latency = 80,
 370		.target_residency = 275,
 371		.enter = &intel_idle,
 372		.enter_s2idle = intel_idle_s2idle, },
 373	{
 374		.name = "C6S",
 375		.desc = "MWAIT 0x52",
 376		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
 377		.exit_latency = 200,
 378		.target_residency = 560,
 379		.enter = &intel_idle,
 380		.enter_s2idle = intel_idle_s2idle, },
 381	{
 382		.name = "C7",
 383		.desc = "MWAIT 0x60",
 384		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 385		.exit_latency = 1200,
 386		.target_residency = 4000,
 387		.enter = &intel_idle,
 388		.enter_s2idle = intel_idle_s2idle, },
 389	{
 390		.name = "C7S",
 391		.desc = "MWAIT 0x64",
 392		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
 393		.exit_latency = 10000,
 394		.target_residency = 20000,
 395		.enter = &intel_idle,
 396		.enter_s2idle = intel_idle_s2idle, },
 397	{
 398		.enter = NULL }
 399};
 400
 401static struct cpuidle_state ivb_cstates[] __initdata = {
 402	{
 403		.name = "C1",
 404		.desc = "MWAIT 0x00",
 405		.flags = MWAIT2flg(0x00),
 406		.exit_latency = 1,
 407		.target_residency = 1,
 408		.enter = &intel_idle,
 409		.enter_s2idle = intel_idle_s2idle, },
 410	{
 411		.name = "C1E",
 412		.desc = "MWAIT 0x01",
 413		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 414		.exit_latency = 10,
 415		.target_residency = 20,
 416		.enter = &intel_idle,
 417		.enter_s2idle = intel_idle_s2idle, },
 418	{
 419		.name = "C3",
 420		.desc = "MWAIT 0x10",
 421		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 422		.exit_latency = 59,
 423		.target_residency = 156,
 424		.enter = &intel_idle,
 425		.enter_s2idle = intel_idle_s2idle, },
 426	{
 427		.name = "C6",
 428		.desc = "MWAIT 0x20",
 429		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 430		.exit_latency = 80,
 431		.target_residency = 300,
 432		.enter = &intel_idle,
 433		.enter_s2idle = intel_idle_s2idle, },
 434	{
 435		.name = "C7",
 436		.desc = "MWAIT 0x30",
 437		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
 438		.exit_latency = 87,
 439		.target_residency = 300,
 440		.enter = &intel_idle,
 441		.enter_s2idle = intel_idle_s2idle, },
 442	{
 443		.enter = NULL }
 444};
 445
 446static struct cpuidle_state ivt_cstates[] __initdata = {
 447	{
 448		.name = "C1",
 449		.desc = "MWAIT 0x00",
 450		.flags = MWAIT2flg(0x00),
 451		.exit_latency = 1,
 452		.target_residency = 1,
 453		.enter = &intel_idle,
 454		.enter_s2idle = intel_idle_s2idle, },
 455	{
 456		.name = "C1E",
 457		.desc = "MWAIT 0x01",
 458		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 459		.exit_latency = 10,
 460		.target_residency = 80,
 461		.enter = &intel_idle,
 462		.enter_s2idle = intel_idle_s2idle, },
 463	{
 464		.name = "C3",
 465		.desc = "MWAIT 0x10",
 466		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 467		.exit_latency = 59,
 468		.target_residency = 156,
 469		.enter = &intel_idle,
 470		.enter_s2idle = intel_idle_s2idle, },
 471	{
 472		.name = "C6",
 473		.desc = "MWAIT 0x20",
 474		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 475		.exit_latency = 82,
 476		.target_residency = 300,
 477		.enter = &intel_idle,
 478		.enter_s2idle = intel_idle_s2idle, },
 479	{
 480		.enter = NULL }
 481};
 482
 483static struct cpuidle_state ivt_cstates_4s[] __initdata = {
 484	{
 485		.name = "C1",
 486		.desc = "MWAIT 0x00",
 487		.flags = MWAIT2flg(0x00),
 488		.exit_latency = 1,
 489		.target_residency = 1,
 490		.enter = &intel_idle,
 491		.enter_s2idle = intel_idle_s2idle, },
 492	{
 493		.name = "C1E",
 494		.desc = "MWAIT 0x01",
 495		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 496		.exit_latency = 10,
 497		.target_residency = 250,
 498		.enter = &intel_idle,
 499		.enter_s2idle = intel_idle_s2idle, },
 500	{
 501		.name = "C3",
 502		.desc = "MWAIT 0x10",
 503		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 504		.exit_latency = 59,
 505		.target_residency = 300,
 506		.enter = &intel_idle,
 507		.enter_s2idle = intel_idle_s2idle, },
 508	{
 509		.name = "C6",
 510		.desc = "MWAIT 0x20",
 511		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 512		.exit_latency = 84,
 513		.target_residency = 400,
 514		.enter = &intel_idle,
 515		.enter_s2idle = intel_idle_s2idle, },
 516	{
 517		.enter = NULL }
 518};
 519
 520static struct cpuidle_state ivt_cstates_8s[] __initdata = {
 521	{
 522		.name = "C1",
 523		.desc = "MWAIT 0x00",
 524		.flags = MWAIT2flg(0x00),
 525		.exit_latency = 1,
 526		.target_residency = 1,
 527		.enter = &intel_idle,
 528		.enter_s2idle = intel_idle_s2idle, },
 529	{
 530		.name = "C1E",
 531		.desc = "MWAIT 0x01",
 532		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 533		.exit_latency = 10,
 534		.target_residency = 500,
 535		.enter = &intel_idle,
 536		.enter_s2idle = intel_idle_s2idle, },
 537	{
 538		.name = "C3",
 539		.desc = "MWAIT 0x10",
 540		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 541		.exit_latency = 59,
 542		.target_residency = 600,
 543		.enter = &intel_idle,
 544		.enter_s2idle = intel_idle_s2idle, },
 545	{
 546		.name = "C6",
 547		.desc = "MWAIT 0x20",
 548		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 549		.exit_latency = 88,
 550		.target_residency = 700,
 551		.enter = &intel_idle,
 552		.enter_s2idle = intel_idle_s2idle, },
 553	{
 554		.enter = NULL }
 555};
 556
 557static struct cpuidle_state hsw_cstates[] __initdata = {
 558	{
 559		.name = "C1",
 560		.desc = "MWAIT 0x00",
 561		.flags = MWAIT2flg(0x00),
 562		.exit_latency = 2,
 563		.target_residency = 2,
 564		.enter = &intel_idle,
 565		.enter_s2idle = intel_idle_s2idle, },
 566	{
 567		.name = "C1E",
 568		.desc = "MWAIT 0x01",
 569		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 570		.exit_latency = 10,
 571		.target_residency = 20,
 572		.enter = &intel_idle,
 573		.enter_s2idle = intel_idle_s2idle, },
 574	{
 575		.name = "C3",
 576		.desc = "MWAIT 0x10",
 577		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 578		.exit_latency = 33,
 579		.target_residency = 100,
 580		.enter = &intel_idle,
 581		.enter_s2idle = intel_idle_s2idle, },
 582	{
 583		.name = "C6",
 584		.desc = "MWAIT 0x20",
 585		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 586		.exit_latency = 133,
 587		.target_residency = 400,
 588		.enter = &intel_idle,
 589		.enter_s2idle = intel_idle_s2idle, },
 590	{
 591		.name = "C7s",
 592		.desc = "MWAIT 0x32",
 593		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
 594		.exit_latency = 166,
 595		.target_residency = 500,
 596		.enter = &intel_idle,
 597		.enter_s2idle = intel_idle_s2idle, },
 598	{
 599		.name = "C8",
 600		.desc = "MWAIT 0x40",
 601		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 602		.exit_latency = 300,
 603		.target_residency = 900,
 604		.enter = &intel_idle,
 605		.enter_s2idle = intel_idle_s2idle, },
 606	{
 607		.name = "C9",
 608		.desc = "MWAIT 0x50",
 609		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
 610		.exit_latency = 600,
 611		.target_residency = 1800,
 612		.enter = &intel_idle,
 613		.enter_s2idle = intel_idle_s2idle, },
 614	{
 615		.name = "C10",
 616		.desc = "MWAIT 0x60",
 617		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 618		.exit_latency = 2600,
 619		.target_residency = 7700,
 620		.enter = &intel_idle,
 621		.enter_s2idle = intel_idle_s2idle, },
 622	{
 623		.enter = NULL }
 624};
 625static struct cpuidle_state bdw_cstates[] __initdata = {
 626	{
 627		.name = "C1",
 628		.desc = "MWAIT 0x00",
 629		.flags = MWAIT2flg(0x00),
 630		.exit_latency = 2,
 631		.target_residency = 2,
 632		.enter = &intel_idle,
 633		.enter_s2idle = intel_idle_s2idle, },
 634	{
 635		.name = "C1E",
 636		.desc = "MWAIT 0x01",
 637		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 638		.exit_latency = 10,
 639		.target_residency = 20,
 640		.enter = &intel_idle,
 641		.enter_s2idle = intel_idle_s2idle, },
 642	{
 643		.name = "C3",
 644		.desc = "MWAIT 0x10",
 645		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 646		.exit_latency = 40,
 647		.target_residency = 100,
 648		.enter = &intel_idle,
 649		.enter_s2idle = intel_idle_s2idle, },
 650	{
 651		.name = "C6",
 652		.desc = "MWAIT 0x20",
 653		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 654		.exit_latency = 133,
 655		.target_residency = 400,
 656		.enter = &intel_idle,
 657		.enter_s2idle = intel_idle_s2idle, },
 658	{
 659		.name = "C7s",
 660		.desc = "MWAIT 0x32",
 661		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
 662		.exit_latency = 166,
 663		.target_residency = 500,
 664		.enter = &intel_idle,
 665		.enter_s2idle = intel_idle_s2idle, },
 666	{
 667		.name = "C8",
 668		.desc = "MWAIT 0x40",
 669		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 670		.exit_latency = 300,
 671		.target_residency = 900,
 672		.enter = &intel_idle,
 673		.enter_s2idle = intel_idle_s2idle, },
 674	{
 675		.name = "C9",
 676		.desc = "MWAIT 0x50",
 677		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
 678		.exit_latency = 600,
 679		.target_residency = 1800,
 680		.enter = &intel_idle,
 681		.enter_s2idle = intel_idle_s2idle, },
 682	{
 683		.name = "C10",
 684		.desc = "MWAIT 0x60",
 685		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 686		.exit_latency = 2600,
 687		.target_residency = 7700,
 688		.enter = &intel_idle,
 689		.enter_s2idle = intel_idle_s2idle, },
 690	{
 691		.enter = NULL }
 692};
 693
 694static struct cpuidle_state skl_cstates[] __initdata = {
 695	{
 696		.name = "C1",
 697		.desc = "MWAIT 0x00",
 698		.flags = MWAIT2flg(0x00),
 699		.exit_latency = 2,
 700		.target_residency = 2,
 701		.enter = &intel_idle,
 702		.enter_s2idle = intel_idle_s2idle, },
 703	{
 704		.name = "C1E",
 705		.desc = "MWAIT 0x01",
 706		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 707		.exit_latency = 10,
 708		.target_residency = 20,
 709		.enter = &intel_idle,
 710		.enter_s2idle = intel_idle_s2idle, },
 711	{
 712		.name = "C3",
 713		.desc = "MWAIT 0x10",
 714		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 715		.exit_latency = 70,
 716		.target_residency = 100,
 717		.enter = &intel_idle,
 718		.enter_s2idle = intel_idle_s2idle, },
 719	{
 720		.name = "C6",
 721		.desc = "MWAIT 0x20",
 722		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 723		.exit_latency = 85,
 724		.target_residency = 200,
 725		.enter = &intel_idle,
 726		.enter_s2idle = intel_idle_s2idle, },
 727	{
 728		.name = "C7s",
 729		.desc = "MWAIT 0x33",
 730		.flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 731		.exit_latency = 124,
 732		.target_residency = 800,
 733		.enter = &intel_idle,
 734		.enter_s2idle = intel_idle_s2idle, },
 735	{
 736		.name = "C8",
 737		.desc = "MWAIT 0x40",
 738		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 739		.exit_latency = 200,
 740		.target_residency = 800,
 741		.enter = &intel_idle,
 742		.enter_s2idle = intel_idle_s2idle, },
 743	{
 744		.name = "C9",
 745		.desc = "MWAIT 0x50",
 746		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 747		.exit_latency = 480,
 748		.target_residency = 5000,
 749		.enter = &intel_idle,
 750		.enter_s2idle = intel_idle_s2idle, },
 751	{
 752		.name = "C10",
 753		.desc = "MWAIT 0x60",
 754		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 755		.exit_latency = 890,
 756		.target_residency = 5000,
 757		.enter = &intel_idle,
 758		.enter_s2idle = intel_idle_s2idle, },
 759	{
 760		.enter = NULL }
 761};
 762
 763static struct cpuidle_state skx_cstates[] __initdata = {
 764	{
 765		.name = "C1",
 766		.desc = "MWAIT 0x00",
 767		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
 768		.exit_latency = 2,
 769		.target_residency = 2,
 770		.enter = &intel_idle,
 771		.enter_s2idle = intel_idle_s2idle, },
 772	{
 773		.name = "C1E",
 774		.desc = "MWAIT 0x01",
 775		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 776		.exit_latency = 10,
 777		.target_residency = 20,
 778		.enter = &intel_idle,
 779		.enter_s2idle = intel_idle_s2idle, },
 780	{
 781		.name = "C6",
 782		.desc = "MWAIT 0x20",
 783		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 784		.exit_latency = 133,
 785		.target_residency = 600,
 786		.enter = &intel_idle,
 787		.enter_s2idle = intel_idle_s2idle, },
 788	{
 789		.enter = NULL }
 790};
 791
 792static struct cpuidle_state icx_cstates[] __initdata = {
 793	{
 794		.name = "C1",
 795		.desc = "MWAIT 0x00",
 796		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
 797		.exit_latency = 1,
 798		.target_residency = 1,
 799		.enter = &intel_idle,
 800		.enter_s2idle = intel_idle_s2idle, },
 801	{
 802		.name = "C1E",
 803		.desc = "MWAIT 0x01",
 804		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 805		.exit_latency = 4,
 806		.target_residency = 4,
 807		.enter = &intel_idle,
 808		.enter_s2idle = intel_idle_s2idle, },
 809	{
 810		.name = "C6",
 811		.desc = "MWAIT 0x20",
 812		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 813		.exit_latency = 170,
 814		.target_residency = 600,
 815		.enter = &intel_idle,
 816		.enter_s2idle = intel_idle_s2idle, },
 817	{
 818		.enter = NULL }
 819};
 820
 821/*
 822 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
 823 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
 824 * But in this case there is effectively no C1, because C1 requests are
 825 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
 826 * and C1E requests end up with C1, so there is effectively no C1E.
 827 *
 828 * By default we enable C1E and disable C1 by marking it with
 829 * 'CPUIDLE_FLAG_UNUSABLE'.
 830 */
 831static struct cpuidle_state adl_cstates[] __initdata = {
 832	{
 833		.name = "C1",
 834		.desc = "MWAIT 0x00",
 835		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 836		.exit_latency = 1,
 837		.target_residency = 1,
 838		.enter = &intel_idle,
 839		.enter_s2idle = intel_idle_s2idle, },
 840	{
 841		.name = "C1E",
 842		.desc = "MWAIT 0x01",
 843		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 844		.exit_latency = 2,
 845		.target_residency = 4,
 846		.enter = &intel_idle,
 847		.enter_s2idle = intel_idle_s2idle, },
 848	{
 849		.name = "C6",
 850		.desc = "MWAIT 0x20",
 851		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 852		.exit_latency = 220,
 853		.target_residency = 600,
 854		.enter = &intel_idle,
 855		.enter_s2idle = intel_idle_s2idle, },
 856	{
 857		.name = "C8",
 858		.desc = "MWAIT 0x40",
 859		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 860		.exit_latency = 280,
 861		.target_residency = 800,
 862		.enter = &intel_idle,
 863		.enter_s2idle = intel_idle_s2idle, },
 864	{
 865		.name = "C10",
 866		.desc = "MWAIT 0x60",
 867		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 868		.exit_latency = 680,
 869		.target_residency = 2000,
 870		.enter = &intel_idle,
 871		.enter_s2idle = intel_idle_s2idle, },
 872	{
 873		.enter = NULL }
 874};
 875
 876static struct cpuidle_state adl_l_cstates[] __initdata = {
 877	{
 878		.name = "C1",
 879		.desc = "MWAIT 0x00",
 880		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 881		.exit_latency = 1,
 882		.target_residency = 1,
 883		.enter = &intel_idle,
 884		.enter_s2idle = intel_idle_s2idle, },
 885	{
 886		.name = "C1E",
 887		.desc = "MWAIT 0x01",
 888		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 889		.exit_latency = 2,
 890		.target_residency = 4,
 891		.enter = &intel_idle,
 892		.enter_s2idle = intel_idle_s2idle, },
 893	{
 894		.name = "C6",
 895		.desc = "MWAIT 0x20",
 896		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 897		.exit_latency = 170,
 898		.target_residency = 500,
 899		.enter = &intel_idle,
 900		.enter_s2idle = intel_idle_s2idle, },
 901	{
 902		.name = "C8",
 903		.desc = "MWAIT 0x40",
 904		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 905		.exit_latency = 200,
 906		.target_residency = 600,
 907		.enter = &intel_idle,
 908		.enter_s2idle = intel_idle_s2idle, },
 909	{
 910		.name = "C10",
 911		.desc = "MWAIT 0x60",
 912		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 913		.exit_latency = 230,
 914		.target_residency = 700,
 915		.enter = &intel_idle,
 916		.enter_s2idle = intel_idle_s2idle, },
 917	{
 918		.enter = NULL }
 919};
 920
 921static struct cpuidle_state mtl_l_cstates[] __initdata = {
 922	{
 923		.name = "C1E",
 924		.desc = "MWAIT 0x01",
 925		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 926		.exit_latency = 1,
 927		.target_residency = 1,
 928		.enter = &intel_idle,
 929		.enter_s2idle = intel_idle_s2idle, },
 930	{
 931		.name = "C6",
 932		.desc = "MWAIT 0x20",
 933		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 934		.exit_latency = 140,
 935		.target_residency = 420,
 936		.enter = &intel_idle,
 937		.enter_s2idle = intel_idle_s2idle, },
 938	{
 939		.name = "C10",
 940		.desc = "MWAIT 0x60",
 941		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 942		.exit_latency = 310,
 943		.target_residency = 930,
 944		.enter = &intel_idle,
 945		.enter_s2idle = intel_idle_s2idle, },
 946	{
 947		.enter = NULL }
 948};
 949
 950static struct cpuidle_state gmt_cstates[] __initdata = {
 951	{
 952		.name = "C1",
 953		.desc = "MWAIT 0x00",
 954		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 955		.exit_latency = 1,
 956		.target_residency = 1,
 957		.enter = &intel_idle,
 958		.enter_s2idle = intel_idle_s2idle, },
 959	{
 960		.name = "C1E",
 961		.desc = "MWAIT 0x01",
 962		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 963		.exit_latency = 2,
 964		.target_residency = 4,
 965		.enter = &intel_idle,
 966		.enter_s2idle = intel_idle_s2idle, },
 967	{
 968		.name = "C6",
 969		.desc = "MWAIT 0x20",
 970		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 971		.exit_latency = 195,
 972		.target_residency = 585,
 973		.enter = &intel_idle,
 974		.enter_s2idle = intel_idle_s2idle, },
 975	{
 976		.name = "C8",
 977		.desc = "MWAIT 0x40",
 978		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 979		.exit_latency = 260,
 980		.target_residency = 1040,
 981		.enter = &intel_idle,
 982		.enter_s2idle = intel_idle_s2idle, },
 983	{
 984		.name = "C10",
 985		.desc = "MWAIT 0x60",
 986		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 987		.exit_latency = 660,
 988		.target_residency = 1980,
 989		.enter = &intel_idle,
 990		.enter_s2idle = intel_idle_s2idle, },
 991	{
 992		.enter = NULL }
 993};
 994
 995static struct cpuidle_state spr_cstates[] __initdata = {
 996	{
 997		.name = "C1",
 998		.desc = "MWAIT 0x00",
 999		.flags = MWAIT2flg(0x00),
1000		.exit_latency = 1,
1001		.target_residency = 1,
1002		.enter = &intel_idle,
1003		.enter_s2idle = intel_idle_s2idle, },
1004	{
1005		.name = "C1E",
1006		.desc = "MWAIT 0x01",
1007		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1008		.exit_latency = 2,
1009		.target_residency = 4,
1010		.enter = &intel_idle,
1011		.enter_s2idle = intel_idle_s2idle, },
1012	{
1013		.name = "C6",
1014		.desc = "MWAIT 0x20",
1015		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1016					   CPUIDLE_FLAG_INIT_XSTATE,
1017		.exit_latency = 290,
1018		.target_residency = 800,
1019		.enter = &intel_idle,
1020		.enter_s2idle = intel_idle_s2idle, },
1021	{
1022		.enter = NULL }
1023};
1024
1025static struct cpuidle_state atom_cstates[] __initdata = {
1026	{
1027		.name = "C1E",
1028		.desc = "MWAIT 0x00",
1029		.flags = MWAIT2flg(0x00),
1030		.exit_latency = 10,
1031		.target_residency = 20,
1032		.enter = &intel_idle,
1033		.enter_s2idle = intel_idle_s2idle, },
1034	{
1035		.name = "C2",
1036		.desc = "MWAIT 0x10",
1037		.flags = MWAIT2flg(0x10),
1038		.exit_latency = 20,
1039		.target_residency = 80,
1040		.enter = &intel_idle,
1041		.enter_s2idle = intel_idle_s2idle, },
1042	{
1043		.name = "C4",
1044		.desc = "MWAIT 0x30",
1045		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1046		.exit_latency = 100,
1047		.target_residency = 400,
1048		.enter = &intel_idle,
1049		.enter_s2idle = intel_idle_s2idle, },
1050	{
1051		.name = "C6",
1052		.desc = "MWAIT 0x52",
1053		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1054		.exit_latency = 140,
1055		.target_residency = 560,
1056		.enter = &intel_idle,
1057		.enter_s2idle = intel_idle_s2idle, },
1058	{
1059		.enter = NULL }
1060};
1061static struct cpuidle_state tangier_cstates[] __initdata = {
1062	{
1063		.name = "C1",
1064		.desc = "MWAIT 0x00",
1065		.flags = MWAIT2flg(0x00),
1066		.exit_latency = 1,
1067		.target_residency = 4,
1068		.enter = &intel_idle,
1069		.enter_s2idle = intel_idle_s2idle, },
1070	{
1071		.name = "C4",
1072		.desc = "MWAIT 0x30",
1073		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1074		.exit_latency = 100,
1075		.target_residency = 400,
1076		.enter = &intel_idle,
1077		.enter_s2idle = intel_idle_s2idle, },
1078	{
1079		.name = "C6",
1080		.desc = "MWAIT 0x52",
1081		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1082		.exit_latency = 140,
1083		.target_residency = 560,
1084		.enter = &intel_idle,
1085		.enter_s2idle = intel_idle_s2idle, },
1086	{
1087		.name = "C7",
1088		.desc = "MWAIT 0x60",
1089		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1090		.exit_latency = 1200,
1091		.target_residency = 4000,
1092		.enter = &intel_idle,
1093		.enter_s2idle = intel_idle_s2idle, },
1094	{
1095		.name = "C9",
1096		.desc = "MWAIT 0x64",
1097		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1098		.exit_latency = 10000,
1099		.target_residency = 20000,
1100		.enter = &intel_idle,
1101		.enter_s2idle = intel_idle_s2idle, },
1102	{
1103		.enter = NULL }
1104};
1105static struct cpuidle_state avn_cstates[] __initdata = {
1106	{
1107		.name = "C1",
1108		.desc = "MWAIT 0x00",
1109		.flags = MWAIT2flg(0x00),
1110		.exit_latency = 2,
1111		.target_residency = 2,
1112		.enter = &intel_idle,
1113		.enter_s2idle = intel_idle_s2idle, },
1114	{
1115		.name = "C6",
1116		.desc = "MWAIT 0x51",
1117		.flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1118		.exit_latency = 15,
1119		.target_residency = 45,
1120		.enter = &intel_idle,
1121		.enter_s2idle = intel_idle_s2idle, },
1122	{
1123		.enter = NULL }
1124};
1125static struct cpuidle_state knl_cstates[] __initdata = {
1126	{
1127		.name = "C1",
1128		.desc = "MWAIT 0x00",
1129		.flags = MWAIT2flg(0x00),
1130		.exit_latency = 1,
1131		.target_residency = 2,
1132		.enter = &intel_idle,
1133		.enter_s2idle = intel_idle_s2idle },
1134	{
1135		.name = "C6",
1136		.desc = "MWAIT 0x10",
1137		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1138		.exit_latency = 120,
1139		.target_residency = 500,
1140		.enter = &intel_idle,
1141		.enter_s2idle = intel_idle_s2idle },
1142	{
1143		.enter = NULL }
1144};
1145
1146static struct cpuidle_state bxt_cstates[] __initdata = {
1147	{
1148		.name = "C1",
1149		.desc = "MWAIT 0x00",
1150		.flags = MWAIT2flg(0x00),
1151		.exit_latency = 2,
1152		.target_residency = 2,
1153		.enter = &intel_idle,
1154		.enter_s2idle = intel_idle_s2idle, },
1155	{
1156		.name = "C1E",
1157		.desc = "MWAIT 0x01",
1158		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1159		.exit_latency = 10,
1160		.target_residency = 20,
1161		.enter = &intel_idle,
1162		.enter_s2idle = intel_idle_s2idle, },
1163	{
1164		.name = "C6",
1165		.desc = "MWAIT 0x20",
1166		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1167		.exit_latency = 133,
1168		.target_residency = 133,
1169		.enter = &intel_idle,
1170		.enter_s2idle = intel_idle_s2idle, },
1171	{
1172		.name = "C7s",
1173		.desc = "MWAIT 0x31",
1174		.flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1175		.exit_latency = 155,
1176		.target_residency = 155,
1177		.enter = &intel_idle,
1178		.enter_s2idle = intel_idle_s2idle, },
1179	{
1180		.name = "C8",
1181		.desc = "MWAIT 0x40",
1182		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1183		.exit_latency = 1000,
1184		.target_residency = 1000,
1185		.enter = &intel_idle,
1186		.enter_s2idle = intel_idle_s2idle, },
1187	{
1188		.name = "C9",
1189		.desc = "MWAIT 0x50",
1190		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1191		.exit_latency = 2000,
1192		.target_residency = 2000,
1193		.enter = &intel_idle,
1194		.enter_s2idle = intel_idle_s2idle, },
1195	{
1196		.name = "C10",
1197		.desc = "MWAIT 0x60",
1198		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1199		.exit_latency = 10000,
1200		.target_residency = 10000,
1201		.enter = &intel_idle,
1202		.enter_s2idle = intel_idle_s2idle, },
1203	{
1204		.enter = NULL }
1205};
1206
1207static struct cpuidle_state dnv_cstates[] __initdata = {
1208	{
1209		.name = "C1",
1210		.desc = "MWAIT 0x00",
1211		.flags = MWAIT2flg(0x00),
1212		.exit_latency = 2,
1213		.target_residency = 2,
1214		.enter = &intel_idle,
1215		.enter_s2idle = intel_idle_s2idle, },
1216	{
1217		.name = "C1E",
1218		.desc = "MWAIT 0x01",
1219		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1220		.exit_latency = 10,
1221		.target_residency = 20,
1222		.enter = &intel_idle,
1223		.enter_s2idle = intel_idle_s2idle, },
1224	{
1225		.name = "C6",
1226		.desc = "MWAIT 0x20",
1227		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1228		.exit_latency = 50,
1229		.target_residency = 500,
1230		.enter = &intel_idle,
1231		.enter_s2idle = intel_idle_s2idle, },
1232	{
1233		.enter = NULL }
1234};
1235
1236/*
1237 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1238 * C6, and this is indicated in the CPUID mwait leaf.
1239 */
1240static struct cpuidle_state snr_cstates[] __initdata = {
1241	{
1242		.name = "C1",
1243		.desc = "MWAIT 0x00",
1244		.flags = MWAIT2flg(0x00),
1245		.exit_latency = 2,
1246		.target_residency = 2,
1247		.enter = &intel_idle,
1248		.enter_s2idle = intel_idle_s2idle, },
1249	{
1250		.name = "C1E",
1251		.desc = "MWAIT 0x01",
1252		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1253		.exit_latency = 15,
1254		.target_residency = 25,
1255		.enter = &intel_idle,
1256		.enter_s2idle = intel_idle_s2idle, },
1257	{
1258		.name = "C6",
1259		.desc = "MWAIT 0x20",
1260		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1261		.exit_latency = 130,
1262		.target_residency = 500,
1263		.enter = &intel_idle,
1264		.enter_s2idle = intel_idle_s2idle, },
1265	{
1266		.enter = NULL }
1267};
1268
1269static struct cpuidle_state grr_cstates[] __initdata = {
1270	{
1271		.name = "C1",
1272		.desc = "MWAIT 0x00",
1273		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1274		.exit_latency = 1,
1275		.target_residency = 1,
1276		.enter = &intel_idle,
1277		.enter_s2idle = intel_idle_s2idle, },
1278	{
1279		.name = "C1E",
1280		.desc = "MWAIT 0x01",
1281		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1282		.exit_latency = 2,
1283		.target_residency = 10,
1284		.enter = &intel_idle,
1285		.enter_s2idle = intel_idle_s2idle, },
1286	{
1287		.name = "C6S",
1288		.desc = "MWAIT 0x22",
1289		.flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1290		.exit_latency = 140,
1291		.target_residency = 500,
1292		.enter = &intel_idle,
1293		.enter_s2idle = intel_idle_s2idle, },
1294	{
1295		.enter = NULL }
1296};
1297
1298static struct cpuidle_state srf_cstates[] __initdata = {
1299	{
1300		.name = "C1",
1301		.desc = "MWAIT 0x00",
1302		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1303		.exit_latency = 1,
1304		.target_residency = 1,
1305		.enter = &intel_idle,
1306		.enter_s2idle = intel_idle_s2idle, },
1307	{
1308		.name = "C1E",
1309		.desc = "MWAIT 0x01",
1310		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1311		.exit_latency = 2,
1312		.target_residency = 10,
1313		.enter = &intel_idle,
1314		.enter_s2idle = intel_idle_s2idle, },
1315	{
1316		.name = "C6S",
1317		.desc = "MWAIT 0x22",
1318		.flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1319		.exit_latency = 270,
1320		.target_residency = 700,
1321		.enter = &intel_idle,
1322		.enter_s2idle = intel_idle_s2idle, },
1323	{
1324		.name = "C6SP",
1325		.desc = "MWAIT 0x23",
1326		.flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED,
1327		.exit_latency = 310,
1328		.target_residency = 900,
1329		.enter = &intel_idle,
1330		.enter_s2idle = intel_idle_s2idle, },
1331	{
1332		.enter = NULL }
1333};
1334
1335static const struct idle_cpu idle_cpu_nehalem __initconst = {
1336	.state_table = nehalem_cstates,
1337	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1338	.disable_promotion_to_c1e = true,
1339};
1340
1341static const struct idle_cpu idle_cpu_nhx __initconst = {
1342	.state_table = nehalem_cstates,
1343	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1344	.disable_promotion_to_c1e = true,
1345	.use_acpi = true,
1346};
1347
1348static const struct idle_cpu idle_cpu_atom __initconst = {
1349	.state_table = atom_cstates,
1350};
1351
1352static const struct idle_cpu idle_cpu_tangier __initconst = {
1353	.state_table = tangier_cstates,
1354};
1355
1356static const struct idle_cpu idle_cpu_lincroft __initconst = {
1357	.state_table = atom_cstates,
1358	.auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1359};
1360
1361static const struct idle_cpu idle_cpu_snb __initconst = {
1362	.state_table = snb_cstates,
1363	.disable_promotion_to_c1e = true,
1364};
1365
1366static const struct idle_cpu idle_cpu_snx __initconst = {
1367	.state_table = snb_cstates,
1368	.disable_promotion_to_c1e = true,
1369	.use_acpi = true,
1370};
1371
1372static const struct idle_cpu idle_cpu_byt __initconst = {
1373	.state_table = byt_cstates,
1374	.disable_promotion_to_c1e = true,
1375	.byt_auto_demotion_disable_flag = true,
1376};
1377
1378static const struct idle_cpu idle_cpu_cht __initconst = {
1379	.state_table = cht_cstates,
1380	.disable_promotion_to_c1e = true,
1381	.byt_auto_demotion_disable_flag = true,
1382};
1383
1384static const struct idle_cpu idle_cpu_ivb __initconst = {
1385	.state_table = ivb_cstates,
1386	.disable_promotion_to_c1e = true,
1387};
1388
1389static const struct idle_cpu idle_cpu_ivt __initconst = {
1390	.state_table = ivt_cstates,
1391	.disable_promotion_to_c1e = true,
1392	.use_acpi = true,
1393};
1394
1395static const struct idle_cpu idle_cpu_hsw __initconst = {
1396	.state_table = hsw_cstates,
1397	.disable_promotion_to_c1e = true,
1398};
1399
1400static const struct idle_cpu idle_cpu_hsx __initconst = {
1401	.state_table = hsw_cstates,
1402	.disable_promotion_to_c1e = true,
1403	.use_acpi = true,
1404};
1405
1406static const struct idle_cpu idle_cpu_bdw __initconst = {
1407	.state_table = bdw_cstates,
1408	.disable_promotion_to_c1e = true,
1409};
1410
1411static const struct idle_cpu idle_cpu_bdx __initconst = {
1412	.state_table = bdw_cstates,
1413	.disable_promotion_to_c1e = true,
1414	.use_acpi = true,
1415};
1416
1417static const struct idle_cpu idle_cpu_skl __initconst = {
1418	.state_table = skl_cstates,
1419	.disable_promotion_to_c1e = true,
1420};
1421
1422static const struct idle_cpu idle_cpu_skx __initconst = {
1423	.state_table = skx_cstates,
1424	.disable_promotion_to_c1e = true,
1425	.use_acpi = true,
1426};
1427
1428static const struct idle_cpu idle_cpu_icx __initconst = {
1429	.state_table = icx_cstates,
1430	.disable_promotion_to_c1e = true,
1431	.use_acpi = true,
1432};
1433
1434static const struct idle_cpu idle_cpu_adl __initconst = {
1435	.state_table = adl_cstates,
1436};
1437
1438static const struct idle_cpu idle_cpu_adl_l __initconst = {
1439	.state_table = adl_l_cstates,
1440};
1441
1442static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1443	.state_table = mtl_l_cstates,
1444};
1445
1446static const struct idle_cpu idle_cpu_gmt __initconst = {
1447	.state_table = gmt_cstates,
1448};
1449
1450static const struct idle_cpu idle_cpu_spr __initconst = {
1451	.state_table = spr_cstates,
1452	.disable_promotion_to_c1e = true,
1453	.use_acpi = true,
1454};
1455
1456static const struct idle_cpu idle_cpu_avn __initconst = {
1457	.state_table = avn_cstates,
1458	.disable_promotion_to_c1e = true,
1459	.use_acpi = true,
1460};
1461
1462static const struct idle_cpu idle_cpu_knl __initconst = {
1463	.state_table = knl_cstates,
1464	.use_acpi = true,
1465};
1466
1467static const struct idle_cpu idle_cpu_bxt __initconst = {
1468	.state_table = bxt_cstates,
1469	.disable_promotion_to_c1e = true,
1470};
1471
1472static const struct idle_cpu idle_cpu_dnv __initconst = {
1473	.state_table = dnv_cstates,
1474	.disable_promotion_to_c1e = true,
1475	.use_acpi = true,
1476};
1477
1478static const struct idle_cpu idle_cpu_snr __initconst = {
1479	.state_table = snr_cstates,
1480	.disable_promotion_to_c1e = true,
1481	.use_acpi = true,
1482};
1483
1484static const struct idle_cpu idle_cpu_grr __initconst = {
1485	.state_table = grr_cstates,
1486	.disable_promotion_to_c1e = true,
1487	.use_acpi = true,
1488};
1489
1490static const struct idle_cpu idle_cpu_srf __initconst = {
1491	.state_table = srf_cstates,
1492	.disable_promotion_to_c1e = true,
1493	.use_acpi = true,
1494};
1495
1496static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1497	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP,		&idle_cpu_nhx),
1498	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM,		&idle_cpu_nehalem),
1499	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G,		&idle_cpu_nehalem),
1500	X86_MATCH_INTEL_FAM6_MODEL(WESTMERE,		&idle_cpu_nehalem),
1501	X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP,		&idle_cpu_nhx),
1502	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX,		&idle_cpu_nhx),
1503	X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL,	&idle_cpu_atom),
1504	X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID,	&idle_cpu_lincroft),
1505	X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX,		&idle_cpu_nhx),
1506	X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE,		&idle_cpu_snb),
1507	X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X,	&idle_cpu_snx),
1508	X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL,	&idle_cpu_atom),
1509	X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT,	&idle_cpu_byt),
1510	X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID,	&idle_cpu_tangier),
1511	X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT,	&idle_cpu_cht),
1512	X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE,		&idle_cpu_ivb),
1513	X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X,		&idle_cpu_ivt),
1514	X86_MATCH_INTEL_FAM6_MODEL(HASWELL,		&idle_cpu_hsw),
1515	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X,		&idle_cpu_hsx),
1516	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L,		&idle_cpu_hsw),
1517	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G,		&idle_cpu_hsw),
1518	X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D,	&idle_cpu_avn),
1519	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL,		&idle_cpu_bdw),
1520	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G,		&idle_cpu_bdw),
1521	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X,		&idle_cpu_bdx),
1522	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D,		&idle_cpu_bdx),
1523	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L,		&idle_cpu_skl),
1524	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE,		&idle_cpu_skl),
1525	X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L,		&idle_cpu_skl),
1526	X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE,		&idle_cpu_skl),
1527	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X,		&idle_cpu_skx),
1528	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X,		&idle_cpu_icx),
1529	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D,		&idle_cpu_icx),
1530	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE,		&idle_cpu_adl),
1531	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L,		&idle_cpu_adl_l),
1532	X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L,	&idle_cpu_mtl_l),
1533	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT,	&idle_cpu_gmt),
1534	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X,	&idle_cpu_spr),
1535	X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X,	&idle_cpu_spr),
1536	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL,	&idle_cpu_knl),
1537	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM,	&idle_cpu_knl),
1538	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT,	&idle_cpu_bxt),
1539	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS,	&idle_cpu_bxt),
1540	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D,	&idle_cpu_dnv),
1541	X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D,	&idle_cpu_snr),
1542	X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT,	&idle_cpu_grr),
1543	X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT_X,	&idle_cpu_srf),
1544	{}
1545};
1546
1547static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1548	X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
1549	{}
1550};
1551
1552static bool __init intel_idle_max_cstate_reached(int cstate)
1553{
1554	if (cstate + 1 > max_cstate) {
1555		pr_info("max_cstate %d reached\n", max_cstate);
1556		return true;
1557	}
1558	return false;
1559}
1560
1561static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1562{
1563	unsigned long eax = flg2MWAIT(state->flags);
1564
1565	if (boot_cpu_has(X86_FEATURE_ARAT))
1566		return false;
1567
1568	/*
1569	 * Switch over to one-shot tick broadcast if the target C-state
1570	 * is deeper than C1.
1571	 */
1572	return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1573}
1574
1575#ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1576#include <acpi/processor.h>
1577
1578static bool no_acpi __read_mostly;
1579module_param(no_acpi, bool, 0444);
1580MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1581
1582static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1583module_param_named(use_acpi, force_use_acpi, bool, 0444);
1584MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1585
1586static struct acpi_processor_power acpi_state_table __initdata;
1587
1588/**
1589 * intel_idle_cst_usable - Check if the _CST information can be used.
1590 *
1591 * Check if all of the C-states listed by _CST in the max_cstate range are
1592 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1593 */
1594static bool __init intel_idle_cst_usable(void)
1595{
1596	int cstate, limit;
1597
1598	limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1599		      acpi_state_table.count);
1600
1601	for (cstate = 1; cstate < limit; cstate++) {
1602		struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1603
1604		if (cx->entry_method != ACPI_CSTATE_FFH)
1605			return false;
1606	}
1607
1608	return true;
1609}
1610
1611static bool __init intel_idle_acpi_cst_extract(void)
1612{
1613	unsigned int cpu;
1614
1615	if (no_acpi) {
1616		pr_debug("Not allowed to use ACPI _CST\n");
1617		return false;
1618	}
1619
1620	for_each_possible_cpu(cpu) {
1621		struct acpi_processor *pr = per_cpu(processors, cpu);
1622
1623		if (!pr)
1624			continue;
1625
1626		if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1627			continue;
1628
1629		acpi_state_table.count++;
1630
1631		if (!intel_idle_cst_usable())
1632			continue;
1633
1634		if (!acpi_processor_claim_cst_control())
1635			break;
1636
1637		return true;
1638	}
1639
1640	acpi_state_table.count = 0;
1641	pr_debug("ACPI _CST not found or not usable\n");
1642	return false;
1643}
1644
1645static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1646{
1647	int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1648
1649	/*
1650	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1651	 * the interesting states are ACPI_CSTATE_FFH.
1652	 */
1653	for (cstate = 1; cstate < limit; cstate++) {
1654		struct acpi_processor_cx *cx;
1655		struct cpuidle_state *state;
1656
1657		if (intel_idle_max_cstate_reached(cstate - 1))
1658			break;
1659
1660		cx = &acpi_state_table.states[cstate];
1661
1662		state = &drv->states[drv->state_count++];
1663
1664		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1665		strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1666		state->exit_latency = cx->latency;
1667		/*
1668		 * For C1-type C-states use the same number for both the exit
1669		 * latency and target residency, because that is the case for
1670		 * C1 in the majority of the static C-states tables above.
1671		 * For the other types of C-states, however, set the target
1672		 * residency to 3 times the exit latency which should lead to
1673		 * a reasonable balance between energy-efficiency and
1674		 * performance in the majority of interesting cases.
1675		 */
1676		state->target_residency = cx->latency;
1677		if (cx->type > ACPI_STATE_C1)
1678			state->target_residency *= 3;
1679
1680		state->flags = MWAIT2flg(cx->address);
1681		if (cx->type > ACPI_STATE_C2)
1682			state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1683
1684		if (disabled_states_mask & BIT(cstate))
1685			state->flags |= CPUIDLE_FLAG_OFF;
1686
1687		if (intel_idle_state_needs_timer_stop(state))
1688			state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1689
1690		state->enter = intel_idle;
1691		state->enter_s2idle = intel_idle_s2idle;
1692	}
1693}
1694
1695static bool __init intel_idle_off_by_default(u32 mwait_hint)
1696{
1697	int cstate, limit;
1698
1699	/*
1700	 * If there are no _CST C-states, do not disable any C-states by
1701	 * default.
1702	 */
1703	if (!acpi_state_table.count)
1704		return false;
1705
1706	limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1707	/*
1708	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1709	 * the interesting states are ACPI_CSTATE_FFH.
1710	 */
1711	for (cstate = 1; cstate < limit; cstate++) {
1712		if (acpi_state_table.states[cstate].address == mwait_hint)
1713			return false;
1714	}
1715	return true;
1716}
1717#else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1718#define force_use_acpi	(false)
1719
1720static inline bool intel_idle_acpi_cst_extract(void) { return false; }
1721static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
1722static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; }
1723#endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1724
1725/**
1726 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1727 *
1728 * Tune IVT multi-socket targets.
1729 * Assumption: num_sockets == (max_package_num + 1).
1730 */
1731static void __init ivt_idle_state_table_update(void)
1732{
1733	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1734	int cpu, package_num, num_sockets = 1;
1735
1736	for_each_online_cpu(cpu) {
1737		package_num = topology_physical_package_id(cpu);
1738		if (package_num + 1 > num_sockets) {
1739			num_sockets = package_num + 1;
1740
1741			if (num_sockets > 4) {
1742				cpuidle_state_table = ivt_cstates_8s;
1743				return;
1744			}
1745		}
1746	}
1747
1748	if (num_sockets > 2)
1749		cpuidle_state_table = ivt_cstates_4s;
1750
1751	/* else, 1 and 2 socket systems use default ivt_cstates */
1752}
1753
1754/**
1755 * irtl_2_usec - IRTL to microseconds conversion.
1756 * @irtl: IRTL MSR value.
1757 *
1758 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1759 */
1760static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1761{
1762	static const unsigned int irtl_ns_units[] __initconst = {
1763		1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1764	};
1765	unsigned long long ns;
1766
1767	if (!irtl)
1768		return 0;
1769
1770	ns = irtl_ns_units[(irtl >> 10) & 0x7];
1771
1772	return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
1773}
1774
1775/**
1776 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1777 *
1778 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1779 * definitive maximum latency and use the same value for target_residency.
1780 */
1781static void __init bxt_idle_state_table_update(void)
1782{
1783	unsigned long long msr;
1784	unsigned int usec;
1785
1786	rdmsrl(MSR_PKGC6_IRTL, msr);
1787	usec = irtl_2_usec(msr);
1788	if (usec) {
1789		bxt_cstates[2].exit_latency = usec;
1790		bxt_cstates[2].target_residency = usec;
1791	}
1792
1793	rdmsrl(MSR_PKGC7_IRTL, msr);
1794	usec = irtl_2_usec(msr);
1795	if (usec) {
1796		bxt_cstates[3].exit_latency = usec;
1797		bxt_cstates[3].target_residency = usec;
1798	}
1799
1800	rdmsrl(MSR_PKGC8_IRTL, msr);
1801	usec = irtl_2_usec(msr);
1802	if (usec) {
1803		bxt_cstates[4].exit_latency = usec;
1804		bxt_cstates[4].target_residency = usec;
1805	}
1806
1807	rdmsrl(MSR_PKGC9_IRTL, msr);
1808	usec = irtl_2_usec(msr);
1809	if (usec) {
1810		bxt_cstates[5].exit_latency = usec;
1811		bxt_cstates[5].target_residency = usec;
1812	}
1813
1814	rdmsrl(MSR_PKGC10_IRTL, msr);
1815	usec = irtl_2_usec(msr);
1816	if (usec) {
1817		bxt_cstates[6].exit_latency = usec;
1818		bxt_cstates[6].target_residency = usec;
1819	}
1820
1821}
1822
1823/**
1824 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1825 *
1826 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1827 */
1828static void __init sklh_idle_state_table_update(void)
1829{
1830	unsigned long long msr;
1831	unsigned int eax, ebx, ecx, edx;
1832
1833
1834	/* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1835	if (max_cstate <= 7)
1836		return;
1837
1838	/* if PC10 not present in CPUID.MWAIT.EDX */
1839	if ((mwait_substates & (0xF << 28)) == 0)
1840		return;
1841
1842	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1843
1844	/* PC10 is not enabled in PKG C-state limit */
1845	if ((msr & 0xF) != 8)
1846		return;
1847
1848	ecx = 0;
1849	cpuid(7, &eax, &ebx, &ecx, &edx);
1850
1851	/* if SGX is present */
1852	if (ebx & (1 << 2)) {
1853
1854		rdmsrl(MSR_IA32_FEAT_CTL, msr);
1855
1856		/* if SGX is enabled */
1857		if (msr & (1 << 18))
1858			return;
1859	}
1860
1861	skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C8-SKL */
1862	skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C9-SKL */
1863}
1864
1865/**
1866 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1867 * idle states table.
1868 */
1869static void __init skx_idle_state_table_update(void)
1870{
1871	unsigned long long msr;
1872
1873	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1874
1875	/*
1876	 * 000b: C0/C1 (no package C-state support)
1877	 * 001b: C2
1878	 * 010b: C6 (non-retention)
1879	 * 011b: C6 (retention)
1880	 * 111b: No Package C state limits.
1881	 */
1882	if ((msr & 0x7) < 2) {
1883		/*
1884		 * Uses the CC6 + PC0 latency and 3 times of
1885		 * latency for target_residency if the PC6
1886		 * is disabled in BIOS. This is consistent
1887		 * with how intel_idle driver uses _CST
1888		 * to set the target_residency.
1889		 */
1890		skx_cstates[2].exit_latency = 92;
1891		skx_cstates[2].target_residency = 276;
1892	}
1893}
1894
1895/**
1896 * adl_idle_state_table_update - Adjust AlderLake idle states table.
1897 */
1898static void __init adl_idle_state_table_update(void)
1899{
1900	/* Check if user prefers C1 over C1E. */
1901	if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
1902		cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
1903		cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
1904
1905		/* Disable C1E by clearing the "C1E promotion" bit. */
1906		c1e_promotion = C1E_PROMOTION_DISABLE;
1907		return;
1908	}
1909
1910	/* Make sure C1E is enabled by default */
1911	c1e_promotion = C1E_PROMOTION_ENABLE;
1912}
1913
1914/**
1915 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
1916 */
1917static void __init spr_idle_state_table_update(void)
1918{
1919	unsigned long long msr;
1920
1921	/*
1922	 * By default, the C6 state assumes the worst-case scenario of package
1923	 * C6. However, if PC6 is disabled, we update the numbers to match
1924	 * core C6.
1925	 */
1926	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1927
1928	/* Limit value 2 and above allow for PC6. */
1929	if ((msr & 0x7) < 2) {
1930		spr_cstates[2].exit_latency = 190;
1931		spr_cstates[2].target_residency = 600;
1932	}
1933}
1934
1935static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
1936{
1937	unsigned int mwait_cstate = MWAIT_HINT2CSTATE(mwait_hint) + 1;
1938	unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
1939					MWAIT_SUBSTATE_MASK;
1940
1941	/* Ignore the C-state if there are NO sub-states in CPUID for it. */
1942	if (num_substates == 0)
1943		return false;
1944
1945	if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1946		mark_tsc_unstable("TSC halts in idle states deeper than C2");
1947
1948	return true;
1949}
1950
1951static void state_update_enter_method(struct cpuidle_state *state, int cstate)
1952{
1953	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
1954		/*
1955		 * Combining with XSTATE with IBRS or IRQ_ENABLE flags
1956		 * is not currently supported but this driver.
1957		 */
1958		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
1959		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
1960		state->enter = intel_idle_xstate;
1961		return;
1962	}
1963
1964	if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
1965			((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
1966		/*
1967		 * IBRS mitigation requires that C-states are entered
1968		 * with interrupts disabled.
1969		 */
1970		if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
1971			state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
1972		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
1973		state->enter = intel_idle_ibrs;
1974		return;
1975	}
1976
1977	if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
1978		state->enter = intel_idle_irq;
1979		return;
1980	}
1981
1982	if (force_irq_on) {
1983		pr_info("forced intel_idle_irq for state %d\n", cstate);
1984		state->enter = intel_idle_irq;
1985	}
1986}
1987
1988static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
1989{
1990	int cstate;
1991
1992	switch (boot_cpu_data.x86_model) {
1993	case INTEL_FAM6_IVYBRIDGE_X:
1994		ivt_idle_state_table_update();
1995		break;
1996	case INTEL_FAM6_ATOM_GOLDMONT:
1997	case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1998		bxt_idle_state_table_update();
1999		break;
2000	case INTEL_FAM6_SKYLAKE:
2001		sklh_idle_state_table_update();
2002		break;
2003	case INTEL_FAM6_SKYLAKE_X:
2004		skx_idle_state_table_update();
2005		break;
2006	case INTEL_FAM6_SAPPHIRERAPIDS_X:
2007	case INTEL_FAM6_EMERALDRAPIDS_X:
2008		spr_idle_state_table_update();
2009		break;
2010	case INTEL_FAM6_ALDERLAKE:
2011	case INTEL_FAM6_ALDERLAKE_L:
2012	case INTEL_FAM6_ATOM_GRACEMONT:
2013		adl_idle_state_table_update();
2014		break;
2015	}
2016
2017	for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2018		struct cpuidle_state *state;
2019		unsigned int mwait_hint;
2020
2021		if (intel_idle_max_cstate_reached(cstate))
2022			break;
2023
2024		if (!cpuidle_state_table[cstate].enter &&
2025		    !cpuidle_state_table[cstate].enter_s2idle)
2026			break;
2027
2028		/* If marked as unusable, skip this state. */
2029		if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2030			pr_debug("state %s is disabled\n",
2031				 cpuidle_state_table[cstate].name);
2032			continue;
2033		}
2034
2035		mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2036		if (!intel_idle_verify_cstate(mwait_hint))
2037			continue;
2038
2039		/* Structure copy. */
2040		drv->states[drv->state_count] = cpuidle_state_table[cstate];
2041		state = &drv->states[drv->state_count];
2042
2043		state_update_enter_method(state, cstate);
2044
2045
2046		if ((disabled_states_mask & BIT(drv->state_count)) ||
2047		    ((icpu->use_acpi || force_use_acpi) &&
2048		     intel_idle_off_by_default(mwait_hint) &&
2049		     !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2050			state->flags |= CPUIDLE_FLAG_OFF;
2051
2052		if (intel_idle_state_needs_timer_stop(state))
2053			state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2054
2055		drv->state_count++;
2056	}
2057
2058	if (icpu->byt_auto_demotion_disable_flag) {
2059		wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2060		wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
2061	}
2062}
2063
2064/**
2065 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2066 * @drv: cpuidle driver structure to initialize.
2067 */
2068static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2069{
2070	cpuidle_poll_state_init(drv);
2071
2072	if (disabled_states_mask & BIT(0))
2073		drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2074
2075	drv->state_count = 1;
2076
2077	if (icpu)
2078		intel_idle_init_cstates_icpu(drv);
2079	else
2080		intel_idle_init_cstates_acpi(drv);
2081}
2082
2083static void auto_demotion_disable(void)
2084{
2085	unsigned long long msr_bits;
2086
2087	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2088	msr_bits &= ~auto_demotion_disable_flags;
2089	wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2090}
2091
2092static void c1e_promotion_enable(void)
2093{
2094	unsigned long long msr_bits;
2095
2096	rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2097	msr_bits |= 0x2;
2098	wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2099}
2100
2101static void c1e_promotion_disable(void)
2102{
2103	unsigned long long msr_bits;
2104
2105	rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2106	msr_bits &= ~0x2;
2107	wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2108}
2109
2110/**
2111 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2112 * @cpu: CPU to initialize.
2113 *
2114 * Register a cpuidle device object for @cpu and update its MSRs in accordance
2115 * with the processor model flags.
2116 */
2117static int intel_idle_cpu_init(unsigned int cpu)
2118{
2119	struct cpuidle_device *dev;
2120
2121	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2122	dev->cpu = cpu;
2123
2124	if (cpuidle_register_device(dev)) {
2125		pr_debug("cpuidle_register_device %d failed!\n", cpu);
2126		return -EIO;
2127	}
2128
2129	if (auto_demotion_disable_flags)
2130		auto_demotion_disable();
2131
2132	if (c1e_promotion == C1E_PROMOTION_ENABLE)
2133		c1e_promotion_enable();
2134	else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2135		c1e_promotion_disable();
2136
2137	return 0;
2138}
2139
2140static int intel_idle_cpu_online(unsigned int cpu)
2141{
2142	struct cpuidle_device *dev;
2143
2144	if (!boot_cpu_has(X86_FEATURE_ARAT))
2145		tick_broadcast_enable();
2146
2147	/*
2148	 * Some systems can hotplug a cpu at runtime after
2149	 * the kernel has booted, we have to initialize the
2150	 * driver in this case
2151	 */
2152	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2153	if (!dev->registered)
2154		return intel_idle_cpu_init(cpu);
2155
2156	return 0;
2157}
2158
2159/**
2160 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2161 */
2162static void __init intel_idle_cpuidle_devices_uninit(void)
2163{
2164	int i;
2165
2166	for_each_online_cpu(i)
2167		cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2168}
2169
2170static int __init intel_idle_init(void)
2171{
2172	const struct x86_cpu_id *id;
2173	unsigned int eax, ebx, ecx;
2174	int retval;
2175
2176	/* Do not load intel_idle at all for now if idle= is passed */
2177	if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2178		return -ENODEV;
2179
2180	if (max_cstate == 0) {
2181		pr_debug("disabled\n");
2182		return -EPERM;
2183	}
2184
2185	id = x86_match_cpu(intel_idle_ids);
2186	if (id) {
2187		if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2188			pr_debug("Please enable MWAIT in BIOS SETUP\n");
2189			return -ENODEV;
2190		}
2191	} else {
2192		id = x86_match_cpu(intel_mwait_ids);
2193		if (!id)
2194			return -ENODEV;
2195	}
2196
2197	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
2198		return -ENODEV;
2199
2200	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
2201
2202	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2203	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2204	    !mwait_substates)
2205			return -ENODEV;
2206
2207	pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2208
2209	icpu = (const struct idle_cpu *)id->driver_data;
2210	if (icpu) {
2211		cpuidle_state_table = icpu->state_table;
2212		auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2213		if (icpu->disable_promotion_to_c1e)
2214			c1e_promotion = C1E_PROMOTION_DISABLE;
2215		if (icpu->use_acpi || force_use_acpi)
2216			intel_idle_acpi_cst_extract();
2217	} else if (!intel_idle_acpi_cst_extract()) {
2218		return -ENODEV;
2219	}
2220
2221	pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2222		 boot_cpu_data.x86_model);
2223
2224	intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2225	if (!intel_idle_cpuidle_devices)
2226		return -ENOMEM;
2227
2228	intel_idle_cpuidle_driver_init(&intel_idle_driver);
2229
2230	retval = cpuidle_register_driver(&intel_idle_driver);
2231	if (retval) {
2232		struct cpuidle_driver *drv = cpuidle_get_driver();
2233		printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2234		       drv ? drv->name : "none");
2235		goto init_driver_fail;
2236	}
2237
2238	retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2239				   intel_idle_cpu_online, NULL);
2240	if (retval < 0)
2241		goto hp_setup_fail;
2242
2243	pr_debug("Local APIC timer is reliable in %s\n",
2244		 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2245
2246	return 0;
2247
2248hp_setup_fail:
2249	intel_idle_cpuidle_devices_uninit();
2250	cpuidle_unregister_driver(&intel_idle_driver);
2251init_driver_fail:
2252	free_percpu(intel_idle_cpuidle_devices);
2253	return retval;
2254
2255}
2256device_initcall(intel_idle_init);
2257
2258/*
2259 * We are not really modular, but we used to support that.  Meaning we also
2260 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2261 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2262 * is the easiest way (currently) to continue doing that.
2263 */
2264module_param(max_cstate, int, 0444);
2265/*
2266 * The positions of the bits that are set in this number are the indices of the
2267 * idle states to be disabled by default (as reflected by the names of the
2268 * corresponding idle state directories in sysfs, "state0", "state1" ...
2269 * "state<i>" ..., where <i> is the index of the given state).
2270 */
2271module_param_named(states_off, disabled_states_mask, uint, 0444);
2272MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2273/*
2274 * Some platforms come with mutually exclusive C-states, so that if one is
2275 * enabled, the other C-states must not be used. Example: C1 and C1E on
2276 * Sapphire Rapids platform. This parameter allows for selecting the
2277 * preferred C-states among the groups of mutually exclusive C-states - the
2278 * selected C-states will be registered, the other C-states from the mutually
2279 * exclusive group won't be registered. If the platform has no mutually
2280 * exclusive C-states, this parameter has no effect.
2281 */
2282module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2283MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2284/*
2285 * Debugging option that forces the driver to enter all C-states with
2286 * interrupts enabled. Does not apply to C-states with
2287 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2288 */
2289module_param(force_irq_on, bool, 0444);
2290/*
2291 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2292 * CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2293 */
2294module_param(ibrs_off, bool, 0444);
2295MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");