1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * intel_idle.c - native hardware idle loop for modern Intel processors
   4 *
   5 * Copyright (c) 2013 - 2020, Intel Corporation.
   6 * Len Brown <len.brown@intel.com>
   7 * Rafael J. Wysocki <rafael.j.wysocki@intel.com>
 
 
 
 
 
 
 
 
 
 
 
 
   8 */
   9
  10/*
  11 * intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
  12 * in lieu of the legacy ACPI processor_idle driver.  The intent is to
  13 * make Linux more efficient on these processors, as intel_idle knows
  14 * more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
  15 */
  16
  17/*
  18 * Design Assumptions
  19 *
  20 * All CPUs have same idle states as boot CPU
  21 *
  22 * Chipset BM_STS (bus master status) bit is a NOP
  23 *	for preventing entry into deep C-states
  24 *
  25 * CPU will flush caches as needed when entering a C-state via MWAIT
  26 *	(in contrast to entering ACPI C3, in which case the WBINVD
  27 *	instruction needs to be executed to flush the caches)
  28 */
  29
  30/*
  31 * Known limitations
  32 *
 
 
 
 
 
  33 * ACPI has a .suspend hack to turn off deep c-statees during suspend
  34 * to avoid complications with the lapic timer workaround.
  35 * Have not seen issues with suspend, but may need same workaround here.
  36 *
 
 
  37 */
  38
  39/* un-comment DEBUG to enable pr_debug() statements */
  40/* #define DEBUG */
  41
  42#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  43
  44#include <linux/acpi.h>
  45#include <linux/kernel.h>
  46#include <linux/cpuidle.h>
  47#include <linux/tick.h>
  48#include <trace/events/power.h>
  49#include <linux/sched.h>
  50#include <linux/sched/smt.h>
  51#include <linux/notifier.h>
  52#include <linux/cpu.h>
  53#include <linux/moduleparam.h>
  54#include <asm/cpu_device_id.h>
  55#include <asm/intel-family.h>
  56#include <asm/mwait.h>
  57#include <asm/spec-ctrl.h>
  58#include <asm/tsc.h>
  59#include <asm/fpu/api.h>
  60
  61#define INTEL_IDLE_VERSION "0.5.1"
 
  62
  63static struct cpuidle_driver intel_idle_driver = {
  64	.name = "intel_idle",
  65	.owner = THIS_MODULE,
  66};
  67/* intel_idle.max_cstate=0 disables driver */
  68static int max_cstate = CPUIDLE_STATE_MAX - 1;
  69static unsigned int disabled_states_mask __read_mostly;
  70static unsigned int preferred_states_mask __read_mostly;
  71static bool force_irq_on __read_mostly;
  72static bool ibrs_off __read_mostly;
  73
  74static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
  75
  76static unsigned long auto_demotion_disable_flags;
  77
  78static enum {
  79	C1E_PROMOTION_PRESERVE,
  80	C1E_PROMOTION_ENABLE,
  81	C1E_PROMOTION_DISABLE
  82} c1e_promotion = C1E_PROMOTION_PRESERVE;
  83
  84struct idle_cpu {
  85	struct cpuidle_state *state_table;
  86
  87	/*
  88	 * Hardware C-state auto-demotion may not always be optimal.
  89	 * Indicate which enable bits to clear here.
  90	 */
  91	unsigned long auto_demotion_disable_flags;
  92	bool byt_auto_demotion_disable_flag;
  93	bool disable_promotion_to_c1e;
  94	bool use_acpi;
  95};
  96
  97static const struct idle_cpu *icpu __initdata;
  98static struct cpuidle_state *cpuidle_state_table __initdata;
  99
 100static unsigned int mwait_substates __initdata;
 101
 102/*
 103 * Enable interrupts before entering the C-state. On some platforms and for
 104 * some C-states, this may measurably decrease interrupt latency.
 105 */
 106#define CPUIDLE_FLAG_IRQ_ENABLE		BIT(14)
 107
 108/*
 109 * Enable this state by default even if the ACPI _CST does not list it.
 110 */
 111#define CPUIDLE_FLAG_ALWAYS_ENABLE	BIT(15)
 112
 113/*
 114 * Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
 115 * above.
 116 */
 117#define CPUIDLE_FLAG_IBRS		BIT(16)
 118
 119/*
 120 * Initialize large xstate for the C6-state entrance.
 121 */
 122#define CPUIDLE_FLAG_INIT_XSTATE	BIT(17)
 123
 124/*
 125 * Ignore the sub-state when matching mwait hints between the ACPI _CST and
 126 * custom tables.
 
 
 127 */
 128#define CPUIDLE_FLAG_PARTIAL_HINT_MATCH	BIT(18)
 129
 130/*
 131 * MWAIT takes an 8-bit "hint" in EAX "suggesting"
 132 * the C-state (top nibble) and sub-state (bottom nibble)
 133 * 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
 134 *
 135 * We store the hint at the top of our "flags" for each state.
 136 */
 137#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
 138#define MWAIT2flg(eax) ((eax & 0xFF) << 24)
 139
 140static __always_inline int __intel_idle(struct cpuidle_device *dev,
 141					struct cpuidle_driver *drv,
 142					int index, bool irqoff)
 143{
 144	struct cpuidle_state *state = &drv->states[index];
 145	unsigned long eax = flg2MWAIT(state->flags);
 146	unsigned long ecx = 1*irqoff; /* break on interrupt flag */
 147
 148	mwait_idle_with_hints(eax, ecx);
 149
 150	return index;
 151}
 152
 153/**
 154 * intel_idle - Ask the processor to enter the given idle state.
 155 * @dev: cpuidle device of the target CPU.
 156 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 157 * @index: Target idle state index.
 158 *
 159 * Use the MWAIT instruction to notify the processor that the CPU represented by
 160 * @dev is idle and it can try to enter the idle state corresponding to @index.
 161 *
 162 * If the local APIC timer is not known to be reliable in the target idle state,
 163 * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
 164 *
 165 * Must be called under local_irq_disable().
 166 */
 167static __cpuidle int intel_idle(struct cpuidle_device *dev,
 168				struct cpuidle_driver *drv, int index)
 169{
 170	return __intel_idle(dev, drv, index, true);
 171}
 172
 173static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
 174				    struct cpuidle_driver *drv, int index)
 175{
 176	return __intel_idle(dev, drv, index, false);
 177}
 178
 179static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
 180				     struct cpuidle_driver *drv, int index)
 181{
 182	bool smt_active = sched_smt_active();
 183	u64 spec_ctrl = spec_ctrl_current();
 184	int ret;
 185
 186	if (smt_active)
 187		__update_spec_ctrl(0);
 188
 189	ret = __intel_idle(dev, drv, index, true);
 190
 191	if (smt_active)
 192		__update_spec_ctrl(spec_ctrl);
 193
 194	return ret;
 195}
 196
 197static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
 198				       struct cpuidle_driver *drv, int index)
 199{
 200	fpu_idle_fpregs();
 201	return __intel_idle(dev, drv, index, true);
 202}
 203
 204/**
 205 * intel_idle_s2idle - Ask the processor to enter the given idle state.
 206 * @dev: cpuidle device of the target CPU.
 207 * @drv: cpuidle driver (assumed to point to intel_idle_driver).
 208 * @index: Target idle state index.
 209 *
 210 * Use the MWAIT instruction to notify the processor that the CPU represented by
 211 * @dev is idle and it can try to enter the idle state corresponding to @index.
 212 *
 213 * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
 214 * scheduler tick and suspended scheduler clock on the target CPU.
 215 */
 216static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
 217				       struct cpuidle_driver *drv, int index)
 218{
 219	unsigned long ecx = 1; /* break on interrupt flag */
 220	struct cpuidle_state *state = &drv->states[index];
 221	unsigned long eax = flg2MWAIT(state->flags);
 222
 223	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
 224		fpu_idle_fpregs();
 225
 226	mwait_idle_with_hints(eax, ecx);
 227
 228	return 0;
 229}
 230
 231/*
 232 * States are indexed by the cstate number,
 233 * which is also the index into the MWAIT hint array.
 234 * Thus C0 is a dummy.
 235 */
 236static struct cpuidle_state nehalem_cstates[] __initdata = {
 237	{
 238		.name = "C1",
 239		.desc = "MWAIT 0x00",
 240		.flags = MWAIT2flg(0x00),
 241		.exit_latency = 3,
 242		.target_residency = 6,
 243		.enter = &intel_idle,
 244		.enter_s2idle = intel_idle_s2idle, },
 245	{
 246		.name = "C1E",
 247		.desc = "MWAIT 0x01",
 248		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 249		.exit_latency = 10,
 250		.target_residency = 20,
 251		.enter = &intel_idle,
 252		.enter_s2idle = intel_idle_s2idle, },
 253	{
 254		.name = "C3",
 255		.desc = "MWAIT 0x10",
 256		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 257		.exit_latency = 20,
 258		.target_residency = 80,
 259		.enter = &intel_idle,
 260		.enter_s2idle = intel_idle_s2idle, },
 261	{
 262		.name = "C6",
 263		.desc = "MWAIT 0x20",
 264		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 265		.exit_latency = 200,
 266		.target_residency = 800,
 267		.enter = &intel_idle,
 268		.enter_s2idle = intel_idle_s2idle, },
 269	{
 270		.enter = NULL }
 271};
 272
 273static struct cpuidle_state snb_cstates[] __initdata = {
 274	{
 275		.name = "C1",
 276		.desc = "MWAIT 0x00",
 277		.flags = MWAIT2flg(0x00),
 278		.exit_latency = 2,
 279		.target_residency = 2,
 280		.enter = &intel_idle,
 281		.enter_s2idle = intel_idle_s2idle, },
 282	{
 283		.name = "C1E",
 284		.desc = "MWAIT 0x01",
 285		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 286		.exit_latency = 10,
 287		.target_residency = 20,
 288		.enter = &intel_idle,
 289		.enter_s2idle = intel_idle_s2idle, },
 290	{
 291		.name = "C3",
 292		.desc = "MWAIT 0x10",
 293		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 294		.exit_latency = 80,
 295		.target_residency = 211,
 296		.enter = &intel_idle,
 297		.enter_s2idle = intel_idle_s2idle, },
 298	{
 299		.name = "C6",
 300		.desc = "MWAIT 0x20",
 301		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 302		.exit_latency = 104,
 303		.target_residency = 345,
 304		.enter = &intel_idle,
 305		.enter_s2idle = intel_idle_s2idle, },
 306	{
 307		.name = "C7",
 308		.desc = "MWAIT 0x30",
 309		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
 310		.exit_latency = 109,
 311		.target_residency = 345,
 312		.enter = &intel_idle,
 313		.enter_s2idle = intel_idle_s2idle, },
 314	{
 315		.enter = NULL }
 316};
 317
 318static struct cpuidle_state byt_cstates[] __initdata = {
 319	{
 320		.name = "C1",
 321		.desc = "MWAIT 0x00",
 322		.flags = MWAIT2flg(0x00),
 323		.exit_latency = 1,
 324		.target_residency = 1,
 325		.enter = &intel_idle,
 326		.enter_s2idle = intel_idle_s2idle, },
 327	{
 328		.name = "C6N",
 329		.desc = "MWAIT 0x58",
 330		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
 331		.exit_latency = 300,
 332		.target_residency = 275,
 333		.enter = &intel_idle,
 334		.enter_s2idle = intel_idle_s2idle, },
 335	{
 336		.name = "C6S",
 337		.desc = "MWAIT 0x52",
 338		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
 339		.exit_latency = 500,
 340		.target_residency = 560,
 341		.enter = &intel_idle,
 342		.enter_s2idle = intel_idle_s2idle, },
 343	{
 344		.name = "C7",
 345		.desc = "MWAIT 0x60",
 346		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 347		.exit_latency = 1200,
 348		.target_residency = 4000,
 349		.enter = &intel_idle,
 350		.enter_s2idle = intel_idle_s2idle, },
 351	{
 352		.name = "C7S",
 353		.desc = "MWAIT 0x64",
 354		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
 355		.exit_latency = 10000,
 356		.target_residency = 20000,
 357		.enter = &intel_idle,
 358		.enter_s2idle = intel_idle_s2idle, },
 359	{
 360		.enter = NULL }
 361};
 362
 363static struct cpuidle_state cht_cstates[] __initdata = {
 364	{
 365		.name = "C1",
 366		.desc = "MWAIT 0x00",
 367		.flags = MWAIT2flg(0x00),
 368		.exit_latency = 1,
 369		.target_residency = 1,
 370		.enter = &intel_idle,
 371		.enter_s2idle = intel_idle_s2idle, },
 372	{
 373		.name = "C6N",
 374		.desc = "MWAIT 0x58",
 375		.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
 376		.exit_latency = 80,
 377		.target_residency = 275,
 378		.enter = &intel_idle,
 379		.enter_s2idle = intel_idle_s2idle, },
 380	{
 381		.name = "C6S",
 382		.desc = "MWAIT 0x52",
 383		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
 384		.exit_latency = 200,
 385		.target_residency = 560,
 386		.enter = &intel_idle,
 387		.enter_s2idle = intel_idle_s2idle, },
 388	{
 389		.name = "C7",
 390		.desc = "MWAIT 0x60",
 391		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 392		.exit_latency = 1200,
 393		.target_residency = 4000,
 394		.enter = &intel_idle,
 395		.enter_s2idle = intel_idle_s2idle, },
 396	{
 397		.name = "C7S",
 398		.desc = "MWAIT 0x64",
 399		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
 400		.exit_latency = 10000,
 401		.target_residency = 20000,
 402		.enter = &intel_idle,
 403		.enter_s2idle = intel_idle_s2idle, },
 404	{
 405		.enter = NULL }
 406};
 407
 408static struct cpuidle_state ivb_cstates[] __initdata = {
 409	{
 410		.name = "C1",
 411		.desc = "MWAIT 0x00",
 412		.flags = MWAIT2flg(0x00),
 413		.exit_latency = 1,
 414		.target_residency = 1,
 415		.enter = &intel_idle,
 416		.enter_s2idle = intel_idle_s2idle, },
 417	{
 418		.name = "C1E",
 419		.desc = "MWAIT 0x01",
 420		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 421		.exit_latency = 10,
 422		.target_residency = 20,
 423		.enter = &intel_idle,
 424		.enter_s2idle = intel_idle_s2idle, },
 425	{
 426		.name = "C3",
 427		.desc = "MWAIT 0x10",
 428		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 429		.exit_latency = 59,
 430		.target_residency = 156,
 431		.enter = &intel_idle,
 432		.enter_s2idle = intel_idle_s2idle, },
 433	{
 434		.name = "C6",
 435		.desc = "MWAIT 0x20",
 436		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 437		.exit_latency = 80,
 438		.target_residency = 300,
 439		.enter = &intel_idle,
 440		.enter_s2idle = intel_idle_s2idle, },
 441	{
 442		.name = "C7",
 443		.desc = "MWAIT 0x30",
 444		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
 445		.exit_latency = 87,
 446		.target_residency = 300,
 447		.enter = &intel_idle,
 448		.enter_s2idle = intel_idle_s2idle, },
 449	{
 450		.enter = NULL }
 451};
 452
 453static struct cpuidle_state ivt_cstates[] __initdata = {
 454	{
 455		.name = "C1",
 456		.desc = "MWAIT 0x00",
 457		.flags = MWAIT2flg(0x00),
 458		.exit_latency = 1,
 459		.target_residency = 1,
 460		.enter = &intel_idle,
 461		.enter_s2idle = intel_idle_s2idle, },
 462	{
 463		.name = "C1E",
 464		.desc = "MWAIT 0x01",
 465		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 466		.exit_latency = 10,
 467		.target_residency = 80,
 468		.enter = &intel_idle,
 469		.enter_s2idle = intel_idle_s2idle, },
 470	{
 471		.name = "C3",
 472		.desc = "MWAIT 0x10",
 473		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 474		.exit_latency = 59,
 475		.target_residency = 156,
 476		.enter = &intel_idle,
 477		.enter_s2idle = intel_idle_s2idle, },
 478	{
 479		.name = "C6",
 480		.desc = "MWAIT 0x20",
 481		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 482		.exit_latency = 82,
 483		.target_residency = 300,
 484		.enter = &intel_idle,
 485		.enter_s2idle = intel_idle_s2idle, },
 486	{
 487		.enter = NULL }
 488};
 489
 490static struct cpuidle_state ivt_cstates_4s[] __initdata = {
 491	{
 492		.name = "C1",
 493		.desc = "MWAIT 0x00",
 494		.flags = MWAIT2flg(0x00),
 495		.exit_latency = 1,
 496		.target_residency = 1,
 497		.enter = &intel_idle,
 498		.enter_s2idle = intel_idle_s2idle, },
 499	{
 500		.name = "C1E",
 501		.desc = "MWAIT 0x01",
 502		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 503		.exit_latency = 10,
 504		.target_residency = 250,
 505		.enter = &intel_idle,
 506		.enter_s2idle = intel_idle_s2idle, },
 507	{
 508		.name = "C3",
 509		.desc = "MWAIT 0x10",
 510		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 511		.exit_latency = 59,
 512		.target_residency = 300,
 513		.enter = &intel_idle,
 514		.enter_s2idle = intel_idle_s2idle, },
 515	{
 516		.name = "C6",
 517		.desc = "MWAIT 0x20",
 518		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 519		.exit_latency = 84,
 520		.target_residency = 400,
 521		.enter = &intel_idle,
 522		.enter_s2idle = intel_idle_s2idle, },
 523	{
 524		.enter = NULL }
 525};
 526
 527static struct cpuidle_state ivt_cstates_8s[] __initdata = {
 528	{
 529		.name = "C1",
 530		.desc = "MWAIT 0x00",
 531		.flags = MWAIT2flg(0x00),
 532		.exit_latency = 1,
 533		.target_residency = 1,
 534		.enter = &intel_idle,
 535		.enter_s2idle = intel_idle_s2idle, },
 536	{
 537		.name = "C1E",
 538		.desc = "MWAIT 0x01",
 539		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 540		.exit_latency = 10,
 541		.target_residency = 500,
 542		.enter = &intel_idle,
 543		.enter_s2idle = intel_idle_s2idle, },
 544	{
 545		.name = "C3",
 546		.desc = "MWAIT 0x10",
 547		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 548		.exit_latency = 59,
 549		.target_residency = 600,
 550		.enter = &intel_idle,
 551		.enter_s2idle = intel_idle_s2idle, },
 552	{
 553		.name = "C6",
 554		.desc = "MWAIT 0x20",
 555		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 556		.exit_latency = 88,
 557		.target_residency = 700,
 558		.enter = &intel_idle,
 559		.enter_s2idle = intel_idle_s2idle, },
 560	{
 561		.enter = NULL }
 562};
 563
 564static struct cpuidle_state hsw_cstates[] __initdata = {
 565	{
 566		.name = "C1",
 567		.desc = "MWAIT 0x00",
 568		.flags = MWAIT2flg(0x00),
 569		.exit_latency = 2,
 570		.target_residency = 2,
 571		.enter = &intel_idle,
 572		.enter_s2idle = intel_idle_s2idle, },
 573	{
 574		.name = "C1E",
 575		.desc = "MWAIT 0x01",
 576		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 577		.exit_latency = 10,
 578		.target_residency = 20,
 579		.enter = &intel_idle,
 580		.enter_s2idle = intel_idle_s2idle, },
 581	{
 582		.name = "C3",
 583		.desc = "MWAIT 0x10",
 584		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 585		.exit_latency = 33,
 586		.target_residency = 100,
 587		.enter = &intel_idle,
 588		.enter_s2idle = intel_idle_s2idle, },
 589	{
 590		.name = "C6",
 591		.desc = "MWAIT 0x20",
 592		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 593		.exit_latency = 133,
 594		.target_residency = 400,
 595		.enter = &intel_idle,
 596		.enter_s2idle = intel_idle_s2idle, },
 597	{
 598		.name = "C7s",
 599		.desc = "MWAIT 0x32",
 600		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
 601		.exit_latency = 166,
 602		.target_residency = 500,
 603		.enter = &intel_idle,
 604		.enter_s2idle = intel_idle_s2idle, },
 605	{
 606		.name = "C8",
 607		.desc = "MWAIT 0x40",
 608		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 609		.exit_latency = 300,
 610		.target_residency = 900,
 611		.enter = &intel_idle,
 612		.enter_s2idle = intel_idle_s2idle, },
 613	{
 614		.name = "C9",
 615		.desc = "MWAIT 0x50",
 616		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
 617		.exit_latency = 600,
 618		.target_residency = 1800,
 619		.enter = &intel_idle,
 620		.enter_s2idle = intel_idle_s2idle, },
 621	{
 622		.name = "C10",
 623		.desc = "MWAIT 0x60",
 624		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 625		.exit_latency = 2600,
 626		.target_residency = 7700,
 627		.enter = &intel_idle,
 628		.enter_s2idle = intel_idle_s2idle, },
 629	{
 630		.enter = NULL }
 631};
 632static struct cpuidle_state bdw_cstates[] __initdata = {
 633	{
 634		.name = "C1",
 635		.desc = "MWAIT 0x00",
 636		.flags = MWAIT2flg(0x00),
 637		.exit_latency = 2,
 638		.target_residency = 2,
 639		.enter = &intel_idle,
 640		.enter_s2idle = intel_idle_s2idle, },
 641	{
 642		.name = "C1E",
 643		.desc = "MWAIT 0x01",
 644		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 645		.exit_latency = 10,
 646		.target_residency = 20,
 647		.enter = &intel_idle,
 648		.enter_s2idle = intel_idle_s2idle, },
 649	{
 650		.name = "C3",
 651		.desc = "MWAIT 0x10",
 652		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 653		.exit_latency = 40,
 654		.target_residency = 100,
 655		.enter = &intel_idle,
 656		.enter_s2idle = intel_idle_s2idle, },
 657	{
 658		.name = "C6",
 659		.desc = "MWAIT 0x20",
 660		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 661		.exit_latency = 133,
 662		.target_residency = 400,
 663		.enter = &intel_idle,
 664		.enter_s2idle = intel_idle_s2idle, },
 665	{
 666		.name = "C7s",
 667		.desc = "MWAIT 0x32",
 668		.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
 669		.exit_latency = 166,
 670		.target_residency = 500,
 671		.enter = &intel_idle,
 672		.enter_s2idle = intel_idle_s2idle, },
 673	{
 674		.name = "C8",
 675		.desc = "MWAIT 0x40",
 676		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 677		.exit_latency = 300,
 678		.target_residency = 900,
 679		.enter = &intel_idle,
 680		.enter_s2idle = intel_idle_s2idle, },
 681	{
 682		.name = "C9",
 683		.desc = "MWAIT 0x50",
 684		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
 685		.exit_latency = 600,
 686		.target_residency = 1800,
 687		.enter = &intel_idle,
 688		.enter_s2idle = intel_idle_s2idle, },
 689	{
 690		.name = "C10",
 691		.desc = "MWAIT 0x60",
 692		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 693		.exit_latency = 2600,
 694		.target_residency = 7700,
 695		.enter = &intel_idle,
 696		.enter_s2idle = intel_idle_s2idle, },
 697	{
 698		.enter = NULL }
 699};
 700
 701static struct cpuidle_state skl_cstates[] __initdata = {
 702	{
 703		.name = "C1",
 704		.desc = "MWAIT 0x00",
 705		.flags = MWAIT2flg(0x00),
 706		.exit_latency = 2,
 707		.target_residency = 2,
 708		.enter = &intel_idle,
 709		.enter_s2idle = intel_idle_s2idle, },
 710	{
 711		.name = "C1E",
 712		.desc = "MWAIT 0x01",
 713		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 714		.exit_latency = 10,
 715		.target_residency = 20,
 716		.enter = &intel_idle,
 717		.enter_s2idle = intel_idle_s2idle, },
 718	{
 719		.name = "C3",
 720		.desc = "MWAIT 0x10",
 721		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
 722		.exit_latency = 70,
 723		.target_residency = 100,
 724		.enter = &intel_idle,
 725		.enter_s2idle = intel_idle_s2idle, },
 726	{
 727		.name = "C6",
 728		.desc = "MWAIT 0x20",
 729		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 730		.exit_latency = 85,
 731		.target_residency = 200,
 732		.enter = &intel_idle,
 733		.enter_s2idle = intel_idle_s2idle, },
 734	{
 735		.name = "C7s",
 736		.desc = "MWAIT 0x33",
 737		.flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 738		.exit_latency = 124,
 739		.target_residency = 800,
 740		.enter = &intel_idle,
 741		.enter_s2idle = intel_idle_s2idle, },
 742	{
 743		.name = "C8",
 744		.desc = "MWAIT 0x40",
 745		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 746		.exit_latency = 200,
 747		.target_residency = 800,
 748		.enter = &intel_idle,
 749		.enter_s2idle = intel_idle_s2idle, },
 750	{
 751		.name = "C9",
 752		.desc = "MWAIT 0x50",
 753		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 754		.exit_latency = 480,
 755		.target_residency = 5000,
 756		.enter = &intel_idle,
 757		.enter_s2idle = intel_idle_s2idle, },
 758	{
 759		.name = "C10",
 760		.desc = "MWAIT 0x60",
 761		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 762		.exit_latency = 890,
 763		.target_residency = 5000,
 764		.enter = &intel_idle,
 765		.enter_s2idle = intel_idle_s2idle, },
 766	{
 767		.enter = NULL }
 768};
 769
 770static struct cpuidle_state skx_cstates[] __initdata = {
 771	{
 772		.name = "C1",
 773		.desc = "MWAIT 0x00",
 774		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
 775		.exit_latency = 2,
 776		.target_residency = 2,
 777		.enter = &intel_idle,
 778		.enter_s2idle = intel_idle_s2idle, },
 779	{
 780		.name = "C1E",
 781		.desc = "MWAIT 0x01",
 782		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 783		.exit_latency = 10,
 784		.target_residency = 20,
 785		.enter = &intel_idle,
 786		.enter_s2idle = intel_idle_s2idle, },
 787	{
 788		.name = "C6",
 789		.desc = "MWAIT 0x20",
 790		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
 791		.exit_latency = 133,
 792		.target_residency = 600,
 793		.enter = &intel_idle,
 794		.enter_s2idle = intel_idle_s2idle, },
 795	{
 796		.enter = NULL }
 797};
 798
 799static struct cpuidle_state icx_cstates[] __initdata = {
 800	{
 801		.name = "C1",
 802		.desc = "MWAIT 0x00",
 803		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
 804		.exit_latency = 1,
 805		.target_residency = 1,
 806		.enter = &intel_idle,
 807		.enter_s2idle = intel_idle_s2idle, },
 808	{
 809		.name = "C1E",
 810		.desc = "MWAIT 0x01",
 811		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 812		.exit_latency = 4,
 813		.target_residency = 4,
 814		.enter = &intel_idle,
 815		.enter_s2idle = intel_idle_s2idle, },
 816	{
 817		.name = "C6",
 818		.desc = "MWAIT 0x20",
 819		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 820		.exit_latency = 170,
 821		.target_residency = 600,
 822		.enter = &intel_idle,
 823		.enter_s2idle = intel_idle_s2idle, },
 824	{
 825		.enter = NULL }
 826};
 827
 828/*
 829 * On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
 830 * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
 831 * But in this case there is effectively no C1, because C1 requests are
 832 * promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
 833 * and C1E requests end up with C1, so there is effectively no C1E.
 834 *
 835 * By default we enable C1E and disable C1 by marking it with
 836 * 'CPUIDLE_FLAG_UNUSABLE'.
 837 */
 838static struct cpuidle_state adl_cstates[] __initdata = {
 839	{
 840		.name = "C1",
 841		.desc = "MWAIT 0x00",
 842		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 843		.exit_latency = 1,
 844		.target_residency = 1,
 845		.enter = &intel_idle,
 846		.enter_s2idle = intel_idle_s2idle, },
 847	{
 848		.name = "C1E",
 849		.desc = "MWAIT 0x01",
 850		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 851		.exit_latency = 2,
 852		.target_residency = 4,
 853		.enter = &intel_idle,
 854		.enter_s2idle = intel_idle_s2idle, },
 855	{
 856		.name = "C6",
 857		.desc = "MWAIT 0x20",
 858		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 859		.exit_latency = 220,
 860		.target_residency = 600,
 861		.enter = &intel_idle,
 862		.enter_s2idle = intel_idle_s2idle, },
 863	{
 864		.name = "C8",
 865		.desc = "MWAIT 0x40",
 866		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 867		.exit_latency = 280,
 868		.target_residency = 800,
 869		.enter = &intel_idle,
 870		.enter_s2idle = intel_idle_s2idle, },
 871	{
 872		.name = "C10",
 873		.desc = "MWAIT 0x60",
 874		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 875		.exit_latency = 680,
 876		.target_residency = 2000,
 877		.enter = &intel_idle,
 878		.enter_s2idle = intel_idle_s2idle, },
 879	{
 880		.enter = NULL }
 881};
 882
 883static struct cpuidle_state adl_l_cstates[] __initdata = {
 884	{
 885		.name = "C1",
 886		.desc = "MWAIT 0x00",
 887		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 888		.exit_latency = 1,
 889		.target_residency = 1,
 890		.enter = &intel_idle,
 891		.enter_s2idle = intel_idle_s2idle, },
 892	{
 893		.name = "C1E",
 894		.desc = "MWAIT 0x01",
 895		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 896		.exit_latency = 2,
 897		.target_residency = 4,
 898		.enter = &intel_idle,
 899		.enter_s2idle = intel_idle_s2idle, },
 900	{
 901		.name = "C6",
 902		.desc = "MWAIT 0x20",
 903		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 904		.exit_latency = 170,
 905		.target_residency = 500,
 906		.enter = &intel_idle,
 907		.enter_s2idle = intel_idle_s2idle, },
 908	{
 909		.name = "C8",
 910		.desc = "MWAIT 0x40",
 911		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 912		.exit_latency = 200,
 913		.target_residency = 600,
 914		.enter = &intel_idle,
 915		.enter_s2idle = intel_idle_s2idle, },
 916	{
 917		.name = "C10",
 918		.desc = "MWAIT 0x60",
 919		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 920		.exit_latency = 230,
 921		.target_residency = 700,
 922		.enter = &intel_idle,
 923		.enter_s2idle = intel_idle_s2idle, },
 924	{
 925		.enter = NULL }
 926};
 927
 928static struct cpuidle_state mtl_l_cstates[] __initdata = {
 929	{
 930		.name = "C1E",
 931		.desc = "MWAIT 0x01",
 932		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 933		.exit_latency = 1,
 934		.target_residency = 1,
 935		.enter = &intel_idle,
 936		.enter_s2idle = intel_idle_s2idle, },
 937	{
 938		.name = "C6",
 939		.desc = "MWAIT 0x20",
 940		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 941		.exit_latency = 140,
 942		.target_residency = 420,
 943		.enter = &intel_idle,
 944		.enter_s2idle = intel_idle_s2idle, },
 945	{
 946		.name = "C10",
 947		.desc = "MWAIT 0x60",
 948		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 949		.exit_latency = 310,
 950		.target_residency = 930,
 951		.enter = &intel_idle,
 952		.enter_s2idle = intel_idle_s2idle, },
 953	{
 954		.enter = NULL }
 955};
 956
 957static struct cpuidle_state gmt_cstates[] __initdata = {
 958	{
 959		.name = "C1",
 960		.desc = "MWAIT 0x00",
 961		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
 962		.exit_latency = 1,
 963		.target_residency = 1,
 964		.enter = &intel_idle,
 965		.enter_s2idle = intel_idle_s2idle, },
 966	{
 967		.name = "C1E",
 968		.desc = "MWAIT 0x01",
 969		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
 970		.exit_latency = 2,
 971		.target_residency = 4,
 972		.enter = &intel_idle,
 973		.enter_s2idle = intel_idle_s2idle, },
 974	{
 975		.name = "C6",
 976		.desc = "MWAIT 0x20",
 977		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
 978		.exit_latency = 195,
 979		.target_residency = 585,
 980		.enter = &intel_idle,
 981		.enter_s2idle = intel_idle_s2idle, },
 982	{
 983		.name = "C8",
 984		.desc = "MWAIT 0x40",
 985		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
 986		.exit_latency = 260,
 987		.target_residency = 1040,
 988		.enter = &intel_idle,
 989		.enter_s2idle = intel_idle_s2idle, },
 990	{
 991		.name = "C10",
 992		.desc = "MWAIT 0x60",
 993		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
 994		.exit_latency = 660,
 995		.target_residency = 1980,
 996		.enter = &intel_idle,
 997		.enter_s2idle = intel_idle_s2idle, },
 998	{
 999		.enter = NULL }
1000};
1001
1002static struct cpuidle_state spr_cstates[] __initdata = {
1003	{
1004		.name = "C1",
1005		.desc = "MWAIT 0x00",
1006		.flags = MWAIT2flg(0x00),
1007		.exit_latency = 1,
1008		.target_residency = 1,
1009		.enter = &intel_idle,
1010		.enter_s2idle = intel_idle_s2idle, },
1011	{
1012		.name = "C1E",
1013		.desc = "MWAIT 0x01",
1014		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1015		.exit_latency = 2,
1016		.target_residency = 4,
1017		.enter = &intel_idle,
1018		.enter_s2idle = intel_idle_s2idle, },
1019	{
1020		.name = "C6",
1021		.desc = "MWAIT 0x20",
1022		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1023					   CPUIDLE_FLAG_INIT_XSTATE,
1024		.exit_latency = 290,
1025		.target_residency = 800,
1026		.enter = &intel_idle,
1027		.enter_s2idle = intel_idle_s2idle, },
1028	{
1029		.enter = NULL }
1030};
1031
1032static struct cpuidle_state gnr_cstates[] __initdata = {
1033	{
1034		.name = "C1",
1035		.desc = "MWAIT 0x00",
1036		.flags = MWAIT2flg(0x00),
1037		.exit_latency = 1,
1038		.target_residency = 1,
1039		.enter = &intel_idle,
1040		.enter_s2idle = intel_idle_s2idle, },
1041	{
1042		.name = "C1E",
1043		.desc = "MWAIT 0x01",
1044		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1045		.exit_latency = 4,
1046		.target_residency = 4,
1047		.enter = &intel_idle,
1048		.enter_s2idle = intel_idle_s2idle, },
1049	{
1050		.name = "C6",
1051		.desc = "MWAIT 0x20",
1052		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1053					   CPUIDLE_FLAG_INIT_XSTATE |
1054					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1055		.exit_latency = 170,
1056		.target_residency = 650,
1057		.enter = &intel_idle,
1058		.enter_s2idle = intel_idle_s2idle, },
1059	{
1060		.name = "C6P",
1061		.desc = "MWAIT 0x21",
1062		.flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1063					   CPUIDLE_FLAG_INIT_XSTATE |
1064					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1065		.exit_latency = 210,
1066		.target_residency = 1000,
1067		.enter = &intel_idle,
1068		.enter_s2idle = intel_idle_s2idle, },
1069	{
1070		.enter = NULL }
1071};
1072
1073static struct cpuidle_state gnrd_cstates[] __initdata = {
1074	{
1075		.name = "C1",
1076		.desc = "MWAIT 0x00",
1077		.flags = MWAIT2flg(0x00),
1078		.exit_latency = 1,
1079		.target_residency = 1,
1080		.enter = &intel_idle,
1081		.enter_s2idle = intel_idle_s2idle, },
1082	{
1083		.name = "C1E",
1084		.desc = "MWAIT 0x01",
1085		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1086		.exit_latency = 4,
1087		.target_residency = 4,
1088		.enter = &intel_idle,
1089		.enter_s2idle = intel_idle_s2idle, },
1090	{
1091		.name = "C6",
1092		.desc = "MWAIT 0x20",
1093		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1094					   CPUIDLE_FLAG_INIT_XSTATE |
1095					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1096		.exit_latency = 220,
1097		.target_residency = 650,
1098		.enter = &intel_idle,
1099		.enter_s2idle = intel_idle_s2idle, },
1100	{
1101		.name = "C6P",
1102		.desc = "MWAIT 0x21",
1103		.flags = MWAIT2flg(0x21) | CPUIDLE_FLAG_TLB_FLUSHED |
1104					   CPUIDLE_FLAG_INIT_XSTATE |
1105					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1106		.exit_latency = 240,
1107		.target_residency = 750,
1108		.enter = &intel_idle,
1109		.enter_s2idle = intel_idle_s2idle, },
1110	{
1111		.enter = NULL }
1112};
1113
1114static struct cpuidle_state atom_cstates[] __initdata = {
1115	{
1116		.name = "C1E",
1117		.desc = "MWAIT 0x00",
1118		.flags = MWAIT2flg(0x00),
1119		.exit_latency = 10,
1120		.target_residency = 20,
1121		.enter = &intel_idle,
1122		.enter_s2idle = intel_idle_s2idle, },
1123	{
1124		.name = "C2",
1125		.desc = "MWAIT 0x10",
1126		.flags = MWAIT2flg(0x10),
1127		.exit_latency = 20,
1128		.target_residency = 80,
1129		.enter = &intel_idle,
1130		.enter_s2idle = intel_idle_s2idle, },
1131	{
1132		.name = "C4",
1133		.desc = "MWAIT 0x30",
1134		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1135		.exit_latency = 100,
1136		.target_residency = 400,
1137		.enter = &intel_idle,
1138		.enter_s2idle = intel_idle_s2idle, },
1139	{
1140		.name = "C6",
1141		.desc = "MWAIT 0x52",
1142		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1143		.exit_latency = 140,
1144		.target_residency = 560,
1145		.enter = &intel_idle,
1146		.enter_s2idle = intel_idle_s2idle, },
1147	{
1148		.enter = NULL }
1149};
1150static struct cpuidle_state tangier_cstates[] __initdata = {
1151	{
1152		.name = "C1",
1153		.desc = "MWAIT 0x00",
1154		.flags = MWAIT2flg(0x00),
1155		.exit_latency = 1,
1156		.target_residency = 4,
1157		.enter = &intel_idle,
1158		.enter_s2idle = intel_idle_s2idle, },
1159	{
1160		.name = "C4",
1161		.desc = "MWAIT 0x30",
1162		.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1163		.exit_latency = 100,
1164		.target_residency = 400,
1165		.enter = &intel_idle,
1166		.enter_s2idle = intel_idle_s2idle, },
1167	{
1168		.name = "C6",
1169		.desc = "MWAIT 0x52",
1170		.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1171		.exit_latency = 140,
1172		.target_residency = 560,
1173		.enter = &intel_idle,
1174		.enter_s2idle = intel_idle_s2idle, },
1175	{
1176		.name = "C7",
1177		.desc = "MWAIT 0x60",
1178		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1179		.exit_latency = 1200,
1180		.target_residency = 4000,
1181		.enter = &intel_idle,
1182		.enter_s2idle = intel_idle_s2idle, },
1183	{
1184		.name = "C9",
1185		.desc = "MWAIT 0x64",
1186		.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1187		.exit_latency = 10000,
1188		.target_residency = 20000,
1189		.enter = &intel_idle,
1190		.enter_s2idle = intel_idle_s2idle, },
1191	{
1192		.enter = NULL }
1193};
1194static struct cpuidle_state avn_cstates[] __initdata = {
1195	{
1196		.name = "C1",
1197		.desc = "MWAIT 0x00",
1198		.flags = MWAIT2flg(0x00),
1199		.exit_latency = 2,
1200		.target_residency = 2,
1201		.enter = &intel_idle,
1202		.enter_s2idle = intel_idle_s2idle, },
1203	{
1204		.name = "C6",
1205		.desc = "MWAIT 0x51",
1206		.flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1207		.exit_latency = 15,
1208		.target_residency = 45,
1209		.enter = &intel_idle,
1210		.enter_s2idle = intel_idle_s2idle, },
1211	{
1212		.enter = NULL }
1213};
1214static struct cpuidle_state knl_cstates[] __initdata = {
1215	{
1216		.name = "C1",
1217		.desc = "MWAIT 0x00",
1218		.flags = MWAIT2flg(0x00),
1219		.exit_latency = 1,
1220		.target_residency = 2,
1221		.enter = &intel_idle,
1222		.enter_s2idle = intel_idle_s2idle },
1223	{
1224		.name = "C6",
1225		.desc = "MWAIT 0x10",
1226		.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1227		.exit_latency = 120,
1228		.target_residency = 500,
1229		.enter = &intel_idle,
1230		.enter_s2idle = intel_idle_s2idle },
1231	{
1232		.enter = NULL }
1233};
1234
1235static struct cpuidle_state bxt_cstates[] __initdata = {
1236	{
1237		.name = "C1",
1238		.desc = "MWAIT 0x00",
1239		.flags = MWAIT2flg(0x00),
1240		.exit_latency = 2,
1241		.target_residency = 2,
1242		.enter = &intel_idle,
1243		.enter_s2idle = intel_idle_s2idle, },
1244	{
1245		.name = "C1E",
1246		.desc = "MWAIT 0x01",
1247		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1248		.exit_latency = 10,
1249		.target_residency = 20,
1250		.enter = &intel_idle,
1251		.enter_s2idle = intel_idle_s2idle, },
1252	{
1253		.name = "C6",
1254		.desc = "MWAIT 0x20",
1255		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1256		.exit_latency = 133,
1257		.target_residency = 133,
1258		.enter = &intel_idle,
1259		.enter_s2idle = intel_idle_s2idle, },
1260	{
1261		.name = "C7s",
1262		.desc = "MWAIT 0x31",
1263		.flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1264		.exit_latency = 155,
1265		.target_residency = 155,
1266		.enter = &intel_idle,
1267		.enter_s2idle = intel_idle_s2idle, },
1268	{
1269		.name = "C8",
1270		.desc = "MWAIT 0x40",
1271		.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1272		.exit_latency = 1000,
1273		.target_residency = 1000,
1274		.enter = &intel_idle,
1275		.enter_s2idle = intel_idle_s2idle, },
1276	{
1277		.name = "C9",
1278		.desc = "MWAIT 0x50",
1279		.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1280		.exit_latency = 2000,
1281		.target_residency = 2000,
1282		.enter = &intel_idle,
1283		.enter_s2idle = intel_idle_s2idle, },
1284	{
1285		.name = "C10",
1286		.desc = "MWAIT 0x60",
1287		.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1288		.exit_latency = 10000,
1289		.target_residency = 10000,
1290		.enter = &intel_idle,
1291		.enter_s2idle = intel_idle_s2idle, },
1292	{
1293		.enter = NULL }
1294};
1295
1296static struct cpuidle_state dnv_cstates[] __initdata = {
1297	{
1298		.name = "C1",
1299		.desc = "MWAIT 0x00",
1300		.flags = MWAIT2flg(0x00),
1301		.exit_latency = 2,
1302		.target_residency = 2,
1303		.enter = &intel_idle,
1304		.enter_s2idle = intel_idle_s2idle, },
1305	{
1306		.name = "C1E",
1307		.desc = "MWAIT 0x01",
1308		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1309		.exit_latency = 10,
1310		.target_residency = 20,
1311		.enter = &intel_idle,
1312		.enter_s2idle = intel_idle_s2idle, },
1313	{
1314		.name = "C6",
1315		.desc = "MWAIT 0x20",
1316		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1317		.exit_latency = 50,
1318		.target_residency = 500,
1319		.enter = &intel_idle,
1320		.enter_s2idle = intel_idle_s2idle, },
1321	{
1322		.enter = NULL }
1323};
1324
1325/*
1326 * Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1327 * C6, and this is indicated in the CPUID mwait leaf.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1328 */
1329static struct cpuidle_state snr_cstates[] __initdata = {
1330	{
1331		.name = "C1",
1332		.desc = "MWAIT 0x00",
1333		.flags = MWAIT2flg(0x00),
1334		.exit_latency = 2,
1335		.target_residency = 2,
1336		.enter = &intel_idle,
1337		.enter_s2idle = intel_idle_s2idle, },
1338	{
1339		.name = "C1E",
1340		.desc = "MWAIT 0x01",
1341		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1342		.exit_latency = 15,
1343		.target_residency = 25,
1344		.enter = &intel_idle,
1345		.enter_s2idle = intel_idle_s2idle, },
1346	{
1347		.name = "C6",
1348		.desc = "MWAIT 0x20",
1349		.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1350		.exit_latency = 130,
1351		.target_residency = 500,
1352		.enter = &intel_idle,
1353		.enter_s2idle = intel_idle_s2idle, },
1354	{
1355		.enter = NULL }
1356};
1357
1358static struct cpuidle_state grr_cstates[] __initdata = {
1359	{
1360		.name = "C1",
1361		.desc = "MWAIT 0x00",
1362		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1363		.exit_latency = 1,
1364		.target_residency = 1,
1365		.enter = &intel_idle,
1366		.enter_s2idle = intel_idle_s2idle, },
1367	{
1368		.name = "C1E",
1369		.desc = "MWAIT 0x01",
1370		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1371		.exit_latency = 2,
1372		.target_residency = 10,
1373		.enter = &intel_idle,
1374		.enter_s2idle = intel_idle_s2idle, },
1375	{
1376		.name = "C6S",
1377		.desc = "MWAIT 0x22",
1378		.flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1379		.exit_latency = 140,
1380		.target_residency = 500,
1381		.enter = &intel_idle,
1382		.enter_s2idle = intel_idle_s2idle, },
1383	{
1384		.enter = NULL }
1385};
1386
1387static struct cpuidle_state srf_cstates[] __initdata = {
1388	{
1389		.name = "C1",
1390		.desc = "MWAIT 0x00",
1391		.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1392		.exit_latency = 1,
1393		.target_residency = 1,
1394		.enter = &intel_idle,
1395		.enter_s2idle = intel_idle_s2idle, },
1396	{
1397		.name = "C1E",
1398		.desc = "MWAIT 0x01",
1399		.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1400		.exit_latency = 2,
1401		.target_residency = 10,
1402		.enter = &intel_idle,
1403		.enter_s2idle = intel_idle_s2idle, },
1404	{
1405		.name = "C6S",
1406		.desc = "MWAIT 0x22",
1407		.flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED |
1408					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1409		.exit_latency = 270,
1410		.target_residency = 700,
1411		.enter = &intel_idle,
1412		.enter_s2idle = intel_idle_s2idle, },
1413	{
1414		.name = "C6SP",
1415		.desc = "MWAIT 0x23",
1416		.flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED |
1417					   CPUIDLE_FLAG_PARTIAL_HINT_MATCH,
1418		.exit_latency = 310,
1419		.target_residency = 900,
1420		.enter = &intel_idle,
1421		.enter_s2idle = intel_idle_s2idle, },
1422	{
1423		.enter = NULL }
1424};
1425
1426static const struct idle_cpu idle_cpu_nehalem __initconst = {
1427	.state_table = nehalem_cstates,
1428	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1429	.disable_promotion_to_c1e = true,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1430};
1431
1432static const struct idle_cpu idle_cpu_nhx __initconst = {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1433	.state_table = nehalem_cstates,
1434	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1435	.disable_promotion_to_c1e = true,
1436	.use_acpi = true,
1437};
1438
1439static const struct idle_cpu idle_cpu_atom __initconst = {
1440	.state_table = atom_cstates,
1441};
1442
1443static const struct idle_cpu idle_cpu_tangier __initconst = {
1444	.state_table = tangier_cstates,
1445};
1446
1447static const struct idle_cpu idle_cpu_lincroft __initconst = {
1448	.state_table = atom_cstates,
1449	.auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1450};
1451
1452static const struct idle_cpu idle_cpu_snb __initconst = {
1453	.state_table = snb_cstates,
1454	.disable_promotion_to_c1e = true,
1455};
1456
1457static const struct idle_cpu idle_cpu_snx __initconst = {
1458	.state_table = snb_cstates,
1459	.disable_promotion_to_c1e = true,
1460	.use_acpi = true,
1461};
1462
1463static const struct idle_cpu idle_cpu_byt __initconst = {
1464	.state_table = byt_cstates,
1465	.disable_promotion_to_c1e = true,
1466	.byt_auto_demotion_disable_flag = true,
1467};
1468
1469static const struct idle_cpu idle_cpu_cht __initconst = {
1470	.state_table = cht_cstates,
1471	.disable_promotion_to_c1e = true,
1472	.byt_auto_demotion_disable_flag = true,
1473};
1474
1475static const struct idle_cpu idle_cpu_ivb __initconst = {
1476	.state_table = ivb_cstates,
1477	.disable_promotion_to_c1e = true,
1478};
1479
1480static const struct idle_cpu idle_cpu_ivt __initconst = {
1481	.state_table = ivt_cstates,
1482	.disable_promotion_to_c1e = true,
1483	.use_acpi = true,
1484};
1485
1486static const struct idle_cpu idle_cpu_hsw __initconst = {
1487	.state_table = hsw_cstates,
1488	.disable_promotion_to_c1e = true,
1489};
1490
1491static const struct idle_cpu idle_cpu_hsx __initconst = {
1492	.state_table = hsw_cstates,
1493	.disable_promotion_to_c1e = true,
1494	.use_acpi = true,
1495};
1496
1497static const struct idle_cpu idle_cpu_bdw __initconst = {
1498	.state_table = bdw_cstates,
1499	.disable_promotion_to_c1e = true,
1500};
1501
1502static const struct idle_cpu idle_cpu_bdx __initconst = {
1503	.state_table = bdw_cstates,
1504	.disable_promotion_to_c1e = true,
1505	.use_acpi = true,
1506};
1507
1508static const struct idle_cpu idle_cpu_skl __initconst = {
1509	.state_table = skl_cstates,
1510	.disable_promotion_to_c1e = true,
1511};
1512
1513static const struct idle_cpu idle_cpu_skx __initconst = {
1514	.state_table = skx_cstates,
1515	.disable_promotion_to_c1e = true,
1516	.use_acpi = true,
1517};
1518
1519static const struct idle_cpu idle_cpu_icx __initconst = {
1520	.state_table = icx_cstates,
1521	.disable_promotion_to_c1e = true,
1522	.use_acpi = true,
1523};
1524
1525static const struct idle_cpu idle_cpu_adl __initconst = {
1526	.state_table = adl_cstates,
1527};
1528
1529static const struct idle_cpu idle_cpu_adl_l __initconst = {
1530	.state_table = adl_l_cstates,
1531};
1532
1533static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1534	.state_table = mtl_l_cstates,
1535};
1536
1537static const struct idle_cpu idle_cpu_gmt __initconst = {
1538	.state_table = gmt_cstates,
1539};
1540
1541static const struct idle_cpu idle_cpu_spr __initconst = {
1542	.state_table = spr_cstates,
1543	.disable_promotion_to_c1e = true,
1544	.use_acpi = true,
1545};
1546
1547static const struct idle_cpu idle_cpu_gnr __initconst = {
1548	.state_table = gnr_cstates,
1549	.disable_promotion_to_c1e = true,
1550	.use_acpi = true,
1551};
1552
1553static const struct idle_cpu idle_cpu_gnrd __initconst = {
1554	.state_table = gnrd_cstates,
1555	.disable_promotion_to_c1e = true,
1556	.use_acpi = true,
1557};
1558
1559static const struct idle_cpu idle_cpu_avn __initconst = {
1560	.state_table = avn_cstates,
1561	.disable_promotion_to_c1e = true,
1562	.use_acpi = true,
1563};
1564
1565static const struct idle_cpu idle_cpu_knl __initconst = {
1566	.state_table = knl_cstates,
1567	.use_acpi = true,
1568};
1569
1570static const struct idle_cpu idle_cpu_bxt __initconst = {
1571	.state_table = bxt_cstates,
1572	.disable_promotion_to_c1e = true,
1573};
1574
1575static const struct idle_cpu idle_cpu_dnv __initconst = {
1576	.state_table = dnv_cstates,
1577	.disable_promotion_to_c1e = true,
1578	.use_acpi = true,
1579};
1580
1581static const struct idle_cpu idle_cpu_tmt __initconst = {
1582	.disable_promotion_to_c1e = true,
1583};
1584
1585static const struct idle_cpu idle_cpu_snr __initconst = {
1586	.state_table = snr_cstates,
1587	.disable_promotion_to_c1e = true,
1588	.use_acpi = true,
1589};
1590
1591static const struct idle_cpu idle_cpu_grr __initconst = {
1592	.state_table = grr_cstates,
1593	.disable_promotion_to_c1e = true,
1594	.use_acpi = true,
1595};
1596
1597static const struct idle_cpu idle_cpu_srf __initconst = {
1598	.state_table = srf_cstates,
1599	.disable_promotion_to_c1e = true,
1600	.use_acpi = true,
1601};
1602
1603static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1604	X86_MATCH_VFM(INTEL_NEHALEM_EP,		&idle_cpu_nhx),
1605	X86_MATCH_VFM(INTEL_NEHALEM,		&idle_cpu_nehalem),
1606	X86_MATCH_VFM(INTEL_NEHALEM_G,		&idle_cpu_nehalem),
1607	X86_MATCH_VFM(INTEL_WESTMERE,		&idle_cpu_nehalem),
1608	X86_MATCH_VFM(INTEL_WESTMERE_EP,	&idle_cpu_nhx),
1609	X86_MATCH_VFM(INTEL_NEHALEM_EX,		&idle_cpu_nhx),
1610	X86_MATCH_VFM(INTEL_ATOM_BONNELL,	&idle_cpu_atom),
1611	X86_MATCH_VFM(INTEL_ATOM_BONNELL_MID,	&idle_cpu_lincroft),
1612	X86_MATCH_VFM(INTEL_WESTMERE_EX,	&idle_cpu_nhx),
1613	X86_MATCH_VFM(INTEL_SANDYBRIDGE,	&idle_cpu_snb),
1614	X86_MATCH_VFM(INTEL_SANDYBRIDGE_X,	&idle_cpu_snx),
1615	X86_MATCH_VFM(INTEL_ATOM_SALTWELL,	&idle_cpu_atom),
1616	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT,	&idle_cpu_byt),
1617	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1618	X86_MATCH_VFM(INTEL_ATOM_AIRMONT,	&idle_cpu_cht),
1619	X86_MATCH_VFM(INTEL_IVYBRIDGE,		&idle_cpu_ivb),
1620	X86_MATCH_VFM(INTEL_IVYBRIDGE_X,	&idle_cpu_ivt),
1621	X86_MATCH_VFM(INTEL_HASWELL,		&idle_cpu_hsw),
1622	X86_MATCH_VFM(INTEL_HASWELL_X,		&idle_cpu_hsx),
1623	X86_MATCH_VFM(INTEL_HASWELL_L,		&idle_cpu_hsw),
1624	X86_MATCH_VFM(INTEL_HASWELL_G,		&idle_cpu_hsw),
1625	X86_MATCH_VFM(INTEL_ATOM_SILVERMONT_D,	&idle_cpu_avn),
1626	X86_MATCH_VFM(INTEL_BROADWELL,		&idle_cpu_bdw),
1627	X86_MATCH_VFM(INTEL_BROADWELL_G,	&idle_cpu_bdw),
1628	X86_MATCH_VFM(INTEL_BROADWELL_X,	&idle_cpu_bdx),
1629	X86_MATCH_VFM(INTEL_BROADWELL_D,	&idle_cpu_bdx),
1630	X86_MATCH_VFM(INTEL_SKYLAKE_L,		&idle_cpu_skl),
1631	X86_MATCH_VFM(INTEL_SKYLAKE,		&idle_cpu_skl),
1632	X86_MATCH_VFM(INTEL_KABYLAKE_L,		&idle_cpu_skl),
1633	X86_MATCH_VFM(INTEL_KABYLAKE,		&idle_cpu_skl),
1634	X86_MATCH_VFM(INTEL_SKYLAKE_X,		&idle_cpu_skx),
1635	X86_MATCH_VFM(INTEL_ICELAKE_X,		&idle_cpu_icx),
1636	X86_MATCH_VFM(INTEL_ICELAKE_D,		&idle_cpu_icx),
1637	X86_MATCH_VFM(INTEL_ALDERLAKE,		&idle_cpu_adl),
1638	X86_MATCH_VFM(INTEL_ALDERLAKE_L,	&idle_cpu_adl_l),
1639	X86_MATCH_VFM(INTEL_METEORLAKE_L,	&idle_cpu_mtl_l),
1640	X86_MATCH_VFM(INTEL_ATOM_GRACEMONT,	&idle_cpu_gmt),
1641	X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X,	&idle_cpu_spr),
1642	X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X,	&idle_cpu_spr),
1643	X86_MATCH_VFM(INTEL_GRANITERAPIDS_X,	&idle_cpu_gnr),
1644	X86_MATCH_VFM(INTEL_GRANITERAPIDS_D,	&idle_cpu_gnrd),
1645	X86_MATCH_VFM(INTEL_XEON_PHI_KNL,	&idle_cpu_knl),
1646	X86_MATCH_VFM(INTEL_XEON_PHI_KNM,	&idle_cpu_knl),
1647	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT,	&idle_cpu_bxt),
1648	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_PLUS,	&idle_cpu_bxt),
1649	X86_MATCH_VFM(INTEL_ATOM_GOLDMONT_D,	&idle_cpu_dnv),
1650	X86_MATCH_VFM(INTEL_ATOM_TREMONT,       &idle_cpu_tmt),
1651	X86_MATCH_VFM(INTEL_ATOM_TREMONT_L,     &idle_cpu_tmt),
1652	X86_MATCH_VFM(INTEL_ATOM_TREMONT_D,	&idle_cpu_snr),
1653	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT,	&idle_cpu_grr),
1654	X86_MATCH_VFM(INTEL_ATOM_CRESTMONT_X,	&idle_cpu_srf),
1655	{}
1656};
 
1657
1658static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1659	X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
1660	{}
1661};
1662
1663static bool __init intel_idle_max_cstate_reached(int cstate)
1664{
1665	if (cstate + 1 > max_cstate) {
1666		pr_info("max_cstate %d reached\n", max_cstate);
1667		return true;
1668	}
1669	return false;
1670}
1671
1672static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1673{
1674	unsigned long eax = flg2MWAIT(state->flags);
1675
1676	if (boot_cpu_has(X86_FEATURE_ARAT))
1677		return false;
1678
1679	/*
1680	 * Switch over to one-shot tick broadcast if the target C-state
1681	 * is deeper than C1.
1682	 */
1683	return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1684}
1685
1686#ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1687#include <acpi/processor.h>
1688
1689static bool no_acpi __read_mostly;
1690module_param(no_acpi, bool, 0444);
1691MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1692
1693static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1694module_param_named(use_acpi, force_use_acpi, bool, 0444);
1695MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1696
1697static struct acpi_processor_power acpi_state_table __initdata;
1698
1699/**
1700 * intel_idle_cst_usable - Check if the _CST information can be used.
1701 *
1702 * Check if all of the C-states listed by _CST in the max_cstate range are
1703 * ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1704 */
1705static bool __init intel_idle_cst_usable(void)
1706{
1707	int cstate, limit;
1708
1709	limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1710		      acpi_state_table.count);
1711
1712	for (cstate = 1; cstate < limit; cstate++) {
1713		struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1714
1715		if (cx->entry_method != ACPI_CSTATE_FFH)
1716			return false;
1717	}
1718
1719	return true;
1720}
1721
1722static bool __init intel_idle_acpi_cst_extract(void)
1723{
1724	unsigned int cpu;
1725
1726	if (no_acpi) {
1727		pr_debug("Not allowed to use ACPI _CST\n");
1728		return false;
1729	}
1730
1731	for_each_possible_cpu(cpu) {
1732		struct acpi_processor *pr = per_cpu(processors, cpu);
1733
1734		if (!pr)
1735			continue;
1736
1737		if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))
1738			continue;
1739
1740		acpi_state_table.count++;
1741
1742		if (!intel_idle_cst_usable())
1743			continue;
1744
1745		if (!acpi_processor_claim_cst_control())
1746			break;
1747
1748		return true;
1749	}
1750
1751	acpi_state_table.count = 0;
1752	pr_debug("ACPI _CST not found or not usable\n");
1753	return false;
1754}
1755
1756static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1757{
1758	int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1759
1760	/*
1761	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1762	 * the interesting states are ACPI_CSTATE_FFH.
1763	 */
1764	for (cstate = 1; cstate < limit; cstate++) {
1765		struct acpi_processor_cx *cx;
1766		struct cpuidle_state *state;
1767
1768		if (intel_idle_max_cstate_reached(cstate - 1))
1769			break;
1770
1771		cx = &acpi_state_table.states[cstate];
1772
1773		state = &drv->states[drv->state_count++];
1774
1775		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);
1776		strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1777		state->exit_latency = cx->latency;
1778		/*
1779		 * For C1-type C-states use the same number for both the exit
1780		 * latency and target residency, because that is the case for
1781		 * C1 in the majority of the static C-states tables above.
1782		 * For the other types of C-states, however, set the target
1783		 * residency to 3 times the exit latency which should lead to
1784		 * a reasonable balance between energy-efficiency and
1785		 * performance in the majority of interesting cases.
1786		 */
1787		state->target_residency = cx->latency;
1788		if (cx->type > ACPI_STATE_C1)
1789			state->target_residency *= 3;
1790
1791		state->flags = MWAIT2flg(cx->address);
1792		if (cx->type > ACPI_STATE_C2)
1793			state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
 
1794
1795		if (disabled_states_mask & BIT(cstate))
1796			state->flags |= CPUIDLE_FLAG_OFF;
1797
1798		if (intel_idle_state_needs_timer_stop(state))
1799			state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1800
1801		if (cx->type > ACPI_STATE_C1 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1802			mark_tsc_unstable("TSC halts in idle");
1803
1804		state->enter = intel_idle;
1805		state->enter_s2idle = intel_idle_s2idle;
1806	}
1807}
1808
1809static bool __init intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
 
 
 
 
1810{
1811	int cstate, limit;
1812
1813	/*
1814	 * If there are no _CST C-states, do not disable any C-states by
1815	 * default.
1816	 */
1817	if (!acpi_state_table.count)
1818		return false;
1819
1820	limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1821	/*
1822	 * If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1823	 * the interesting states are ACPI_CSTATE_FFH.
1824	 */
1825	for (cstate = 1; cstate < limit; cstate++) {
1826		u32 acpi_hint = acpi_state_table.states[cstate].address;
1827		u32 table_hint = mwait_hint;
1828
1829		if (flags & CPUIDLE_FLAG_PARTIAL_HINT_MATCH) {
1830			acpi_hint &= ~MWAIT_SUBSTATE_MASK;
1831			table_hint &= ~MWAIT_SUBSTATE_MASK;
1832		}
1833
1834		if (acpi_hint == table_hint)
1835			return false;
 
1836	}
1837	return true;
1838}
1839#else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1840#define force_use_acpi	(false)
1841
1842static inline bool intel_idle_acpi_cst_extract(void) { return false; }
1843static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
1844static inline bool intel_idle_off_by_default(unsigned int flags, u32 mwait_hint)
1845{
1846	return false;
1847}
1848#endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1849
1850/**
1851 * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1852 *
1853 * Tune IVT multi-socket targets.
1854 * Assumption: num_sockets == (max_package_num + 1).
1855 */
1856static void __init ivt_idle_state_table_update(void)
1857{
1858	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1859	int cpu, package_num, num_sockets = 1;
1860
1861	for_each_online_cpu(cpu) {
1862		package_num = topology_physical_package_id(cpu);
1863		if (package_num + 1 > num_sockets) {
1864			num_sockets = package_num + 1;
1865
1866			if (num_sockets > 4) {
1867				cpuidle_state_table = ivt_cstates_8s;
1868				return;
1869			}
1870		}
1871	}
1872
1873	if (num_sockets > 2)
1874		cpuidle_state_table = ivt_cstates_4s;
1875
1876	/* else, 1 and 2 socket systems use default ivt_cstates */
1877}
1878
1879/**
1880 * irtl_2_usec - IRTL to microseconds conversion.
1881 * @irtl: IRTL MSR value.
1882 *
1883 * Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1884 */
1885static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1886{
1887	static const unsigned int irtl_ns_units[] __initconst = {
1888		1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1889	};
1890	unsigned long long ns;
1891
1892	if (!irtl)
1893		return 0;
1894
1895	ns = irtl_ns_units[(irtl >> 10) & 0x7];
1896
1897	return div_u64((irtl & 0x3FF) * ns, NSEC_PER_USEC);
1898}
1899
1900/**
1901 * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1902 *
1903 * On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1904 * definitive maximum latency and use the same value for target_residency.
1905 */
1906static void __init bxt_idle_state_table_update(void)
1907{
1908	unsigned long long msr;
1909	unsigned int usec;
1910
1911	rdmsrl(MSR_PKGC6_IRTL, msr);
1912	usec = irtl_2_usec(msr);
1913	if (usec) {
1914		bxt_cstates[2].exit_latency = usec;
1915		bxt_cstates[2].target_residency = usec;
1916	}
1917
1918	rdmsrl(MSR_PKGC7_IRTL, msr);
1919	usec = irtl_2_usec(msr);
1920	if (usec) {
1921		bxt_cstates[3].exit_latency = usec;
1922		bxt_cstates[3].target_residency = usec;
1923	}
1924
1925	rdmsrl(MSR_PKGC8_IRTL, msr);
1926	usec = irtl_2_usec(msr);
1927	if (usec) {
1928		bxt_cstates[4].exit_latency = usec;
1929		bxt_cstates[4].target_residency = usec;
1930	}
1931
1932	rdmsrl(MSR_PKGC9_IRTL, msr);
1933	usec = irtl_2_usec(msr);
1934	if (usec) {
1935		bxt_cstates[5].exit_latency = usec;
1936		bxt_cstates[5].target_residency = usec;
1937	}
1938
1939	rdmsrl(MSR_PKGC10_IRTL, msr);
1940	usec = irtl_2_usec(msr);
1941	if (usec) {
1942		bxt_cstates[6].exit_latency = usec;
1943		bxt_cstates[6].target_residency = usec;
1944	}
1945
1946}
1947
1948/**
1949 * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1950 *
1951 * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
 
1952 */
1953static void __init sklh_idle_state_table_update(void)
1954{
1955	unsigned long long msr;
1956	unsigned int eax, ebx, ecx, edx;
1957
1958
1959	/* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1960	if (max_cstate <= 7)
1961		return;
1962
1963	/* if PC10 not present in CPUID.MWAIT.EDX */
1964	if ((mwait_substates & (0xF << 28)) == 0)
1965		return;
1966
1967	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1968
1969	/* PC10 is not enabled in PKG C-state limit */
1970	if ((msr & 0xF) != 8)
1971		return;
1972
1973	ecx = 0;
1974	cpuid(7, &eax, &ebx, &ecx, &edx);
1975
1976	/* if SGX is present */
1977	if (ebx & (1 << 2)) {
1978
1979		rdmsrl(MSR_IA32_FEAT_CTL, msr);
1980
1981		/* if SGX is enabled */
1982		if (msr & (1 << 18))
1983			return;
1984	}
1985
1986	skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C8-SKL */
1987	skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C9-SKL */
1988}
1989
1990/**
1991 * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1992 * idle states table.
1993 */
1994static void __init skx_idle_state_table_update(void)
1995{
1996	unsigned long long msr;
1997
1998	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1999
2000	/*
2001	 * 000b: C0/C1 (no package C-state support)
2002	 * 001b: C2
2003	 * 010b: C6 (non-retention)
2004	 * 011b: C6 (retention)
2005	 * 111b: No Package C state limits.
2006	 */
2007	if ((msr & 0x7) < 2) {
2008		/*
2009		 * Uses the CC6 + PC0 latency and 3 times of
2010		 * latency for target_residency if the PC6
2011		 * is disabled in BIOS. This is consistent
2012		 * with how intel_idle driver uses _CST
2013		 * to set the target_residency.
2014		 */
2015		skx_cstates[2].exit_latency = 92;
2016		skx_cstates[2].target_residency = 276;
2017	}
2018}
2019
2020/**
2021 * adl_idle_state_table_update - Adjust AlderLake idle states table.
2022 */
2023static void __init adl_idle_state_table_update(void)
2024{
2025	/* Check if user prefers C1 over C1E. */
2026	if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
2027		cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
2028		cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
2029
2030		/* Disable C1E by clearing the "C1E promotion" bit. */
2031		c1e_promotion = C1E_PROMOTION_DISABLE;
2032		return;
 
 
 
2033	}
2034
2035	/* Make sure C1E is enabled by default */
2036	c1e_promotion = C1E_PROMOTION_ENABLE;
2037}
2038
2039/**
2040 * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
 
2041 */
2042static void __init spr_idle_state_table_update(void)
2043{
2044	unsigned long long msr;
2045
2046	/*
2047	 * By default, the C6 state assumes the worst-case scenario of package
2048	 * C6. However, if PC6 is disabled, we update the numbers to match
2049	 * core C6.
2050	 */
2051	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
2052
2053	/* Limit value 2 and above allow for PC6. */
2054	if ((msr & 0x7) < 2) {
2055		spr_cstates[2].exit_latency = 190;
2056		spr_cstates[2].target_residency = 600;
2057	}
2058}
2059
2060static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
2061{
2062	unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &
2063					MWAIT_CSTATE_MASK;
2064	unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
2065					MWAIT_SUBSTATE_MASK;
2066
2067	/* Ignore the C-state if there are NO sub-states in CPUID for it. */
2068	if (num_substates == 0)
2069		return false;
2070
2071	if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
2072		mark_tsc_unstable("TSC halts in idle states deeper than C2");
2073
2074	return true;
2075}
2076
2077static void state_update_enter_method(struct cpuidle_state *state, int cstate)
2078{
2079	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
2080		/*
2081		 * Combining with XSTATE with IBRS or IRQ_ENABLE flags
2082		 * is not currently supported but this driver.
2083		 */
2084		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
2085		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2086		state->enter = intel_idle_xstate;
2087		return;
2088	}
2089
2090	if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
2091			((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
2092		/*
2093		 * IBRS mitigation requires that C-states are entered
2094		 * with interrupts disabled.
2095		 */
2096		if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
2097			state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
2098		WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
2099		state->enter = intel_idle_ibrs;
2100		return;
2101	}
2102
2103	if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
2104		state->enter = intel_idle_irq;
2105		return;
2106	}
2107
2108	if (force_irq_on) {
2109		pr_info("forced intel_idle_irq for state %d\n", cstate);
2110		state->enter = intel_idle_irq;
2111	}
2112}
2113
2114static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
2115{
2116	int cstate;
 
2117
2118	switch (boot_cpu_data.x86_vfm) {
2119	case INTEL_IVYBRIDGE_X:
2120		ivt_idle_state_table_update();
2121		break;
2122	case INTEL_ATOM_GOLDMONT:
2123	case INTEL_ATOM_GOLDMONT_PLUS:
2124		bxt_idle_state_table_update();
2125		break;
2126	case INTEL_SKYLAKE:
2127		sklh_idle_state_table_update();
2128		break;
2129	case INTEL_SKYLAKE_X:
2130		skx_idle_state_table_update();
2131		break;
2132	case INTEL_SAPPHIRERAPIDS_X:
2133	case INTEL_EMERALDRAPIDS_X:
2134		spr_idle_state_table_update();
2135		break;
2136	case INTEL_ALDERLAKE:
2137	case INTEL_ALDERLAKE_L:
2138	case INTEL_ATOM_GRACEMONT:
2139		adl_idle_state_table_update();
2140		break;
2141	}
2142
2143	for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2144		struct cpuidle_state *state;
2145		unsigned int mwait_hint;
2146
2147		if (intel_idle_max_cstate_reached(cstate))
 
2148			break;
2149
2150		if (!cpuidle_state_table[cstate].enter &&
2151		    !cpuidle_state_table[cstate].enter_s2idle)
 
2152			break;
2153
2154		/* If marked as unusable, skip this state. */
2155		if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2156			pr_debug("state %s is disabled\n",
2157				 cpuidle_state_table[cstate].name);
2158			continue;
2159		}
2160
2161		mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2162		if (!intel_idle_verify_cstate(mwait_hint))
2163			continue;
 
 
 
2164
2165		/* Structure copy. */
2166		drv->states[drv->state_count] = cpuidle_state_table[cstate];
2167		state = &drv->states[drv->state_count];
2168
2169		state_update_enter_method(state, cstate);
 
 
 
 
 
2170
2171
2172		if ((disabled_states_mask & BIT(drv->state_count)) ||
2173		    ((icpu->use_acpi || force_use_acpi) &&
2174		     intel_idle_off_by_default(state->flags, mwait_hint) &&
2175		     !(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2176			state->flags |= CPUIDLE_FLAG_OFF;
2177
2178		if (intel_idle_state_needs_timer_stop(state))
2179			state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2180
2181		drv->state_count++;
2182	}
2183
2184	if (icpu->byt_auto_demotion_disable_flag) {
2185		wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, 0);
2186		wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, 0);
2187	}
2188}
2189
2190/**
2191 * intel_idle_cpuidle_driver_init - Create the list of available idle states.
2192 * @drv: cpuidle driver structure to initialize.
2193 */
2194static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2195{
2196	cpuidle_poll_state_init(drv);
2197
2198	if (disabled_states_mask & BIT(0))
2199		drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2200
2201	drv->state_count = 1;
2202
2203	if (icpu && icpu->state_table)
2204		intel_idle_init_cstates_icpu(drv);
2205	else
2206		intel_idle_init_cstates_acpi(drv);
2207}
2208
2209static void auto_demotion_disable(void)
2210{
2211	unsigned long long msr_bits;
2212
2213	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2214	msr_bits &= ~auto_demotion_disable_flags;
2215	wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2216}
2217
2218static void c1e_promotion_enable(void)
2219{
2220	unsigned long long msr_bits;
2221
2222	rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2223	msr_bits |= 0x2;
2224	wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2225}
2226
2227static void c1e_promotion_disable(void)
2228{
2229	unsigned long long msr_bits;
2230
2231	rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2232	msr_bits &= ~0x2;
2233	wrmsrl(MSR_IA32_POWER_CTL, msr_bits);
2234}
2235
2236/**
2237 * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2238 * @cpu: CPU to initialize.
2239 *
2240 * Register a cpuidle device object for @cpu and update its MSRs in accordance
2241 * with the processor model flags.
2242 */
2243static int intel_idle_cpu_init(unsigned int cpu)
2244{
2245	struct cpuidle_device *dev;
2246
2247	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
 
2248	dev->cpu = cpu;
2249
2250	if (cpuidle_register_device(dev)) {
2251		pr_debug("cpuidle_register_device %d failed!\n", cpu);
2252		return -EIO;
2253	}
2254
2255	if (auto_demotion_disable_flags)
2256		auto_demotion_disable();
2257
2258	if (c1e_promotion == C1E_PROMOTION_ENABLE)
2259		c1e_promotion_enable();
2260	else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2261		c1e_promotion_disable();
2262
2263	return 0;
2264}
2265
2266static int intel_idle_cpu_online(unsigned int cpu)
2267{
2268	struct cpuidle_device *dev;
2269
2270	if (!boot_cpu_has(X86_FEATURE_ARAT))
2271		tick_broadcast_enable();
2272
2273	/*
2274	 * Some systems can hotplug a cpu at runtime after
2275	 * the kernel has booted, we have to initialize the
2276	 * driver in this case
2277	 */
2278	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2279	if (!dev->registered)
2280		return intel_idle_cpu_init(cpu);
2281
2282	return 0;
2283}
2284
2285/**
2286 * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2287 */
2288static void __init intel_idle_cpuidle_devices_uninit(void)
2289{
2290	int i;
2291
2292	for_each_online_cpu(i)
2293		cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2294}
2295
2296static int __init intel_idle_init(void)
2297{
2298	const struct x86_cpu_id *id;
2299	unsigned int eax, ebx, ecx;
2300	int retval;
2301
2302	/* Do not load intel_idle at all for now if idle= is passed */
2303	if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2304		return -ENODEV;
2305
2306	if (max_cstate == 0) {
2307		pr_debug("disabled\n");
2308		return -EPERM;
2309	}
2310
2311	id = x86_match_cpu(intel_idle_ids);
2312	if (id) {
2313		if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2314			pr_debug("Please enable MWAIT in BIOS SETUP\n");
2315			return -ENODEV;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2316		}
2317	} else {
2318		id = x86_match_cpu(intel_mwait_ids);
2319		if (!id)
2320			return -ENODEV;
2321	}
 
2322
2323	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
2324		return -ENODEV;
2325
2326	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &mwait_substates);
2327
2328	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2329	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2330	    !mwait_substates)
2331			return -ENODEV;
2332
2333	pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2334
2335	icpu = (const struct idle_cpu *)id->driver_data;
2336	if (icpu) {
2337		if (icpu->state_table)
2338			cpuidle_state_table = icpu->state_table;
2339		else if (!intel_idle_acpi_cst_extract())
2340			return -ENODEV;
2341
2342		auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2343		if (icpu->disable_promotion_to_c1e)
2344			c1e_promotion = C1E_PROMOTION_DISABLE;
2345		if (icpu->use_acpi || force_use_acpi)
2346			intel_idle_acpi_cst_extract();
2347	} else if (!intel_idle_acpi_cst_extract()) {
2348		return -ENODEV;
2349	}
2350
2351	pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2352		 boot_cpu_data.x86_model);
 
 
2353
2354	intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2355	if (!intel_idle_cpuidle_devices)
2356		return -ENOMEM;
2357
2358	intel_idle_cpuidle_driver_init(&intel_idle_driver);
 
 
2359
2360	retval = cpuidle_register_driver(&intel_idle_driver);
2361	if (retval) {
2362		struct cpuidle_driver *drv = cpuidle_get_driver();
2363		printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2364		       drv ? drv->name : "none");
2365		goto init_driver_fail;
2366	}
2367
2368	retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",
2369				   intel_idle_cpu_online, NULL);
2370	if (retval < 0)
2371		goto hp_setup_fail;
2372
2373	pr_debug("Local APIC timer is reliable in %s\n",
2374		 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2375
2376	return 0;
2377
2378hp_setup_fail:
2379	intel_idle_cpuidle_devices_uninit();
2380	cpuidle_unregister_driver(&intel_idle_driver);
2381init_driver_fail:
2382	free_percpu(intel_idle_cpuidle_devices);
2383	return retval;
2384
2385}
2386device_initcall(intel_idle_init);
2387
2388/*
2389 * We are not really modular, but we used to support that.  Meaning we also
2390 * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2391 * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2392 * is the easiest way (currently) to continue doing that.
2393 */
2394module_param(max_cstate, int, 0444);
2395/*
2396 * The positions of the bits that are set in this number are the indices of the
2397 * idle states to be disabled by default (as reflected by the names of the
2398 * corresponding idle state directories in sysfs, "state0", "state1" ...
2399 * "state<i>" ..., where <i> is the index of the given state).
2400 */
2401module_param_named(states_off, disabled_states_mask, uint, 0444);
2402MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2403/*
2404 * Some platforms come with mutually exclusive C-states, so that if one is
2405 * enabled, the other C-states must not be used. Example: C1 and C1E on
2406 * Sapphire Rapids platform. This parameter allows for selecting the
2407 * preferred C-states among the groups of mutually exclusive C-states - the
2408 * selected C-states will be registered, the other C-states from the mutually
2409 * exclusive group won't be registered. If the platform has no mutually
2410 * exclusive C-states, this parameter has no effect.
2411 */
2412module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2413MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2414/*
2415 * Debugging option that forces the driver to enter all C-states with
2416 * interrupts enabled. Does not apply to C-states with
2417 * 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2418 */
2419module_param(force_irq_on, bool, 0444);
2420/*
2421 * Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2422 * CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2423 */
2424module_param(ibrs_off, bool, 0444);
2425MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");