1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * itmt.c: Support Intel Turbo Boost Max Technology 3.0
  4 *
  5 * (C) Copyright 2016 Intel Corporation
  6 * Author: Tim Chen <tim.c.chen@linux.intel.com>
  7 *
 
 
 
 
 
  8 * On platforms supporting Intel Turbo Boost Max Technology 3.0, (ITMT),
  9 * the maximum turbo frequencies of some cores in a CPU package may be
 10 * higher than for the other cores in the same package.  In that case,
 11 * better performance can be achieved by making the scheduler prefer
 12 * to run tasks on the CPUs with higher max turbo frequencies.
 13 *
 14 * This file provides functions and data structures for enabling the
 15 * scheduler to favor scheduling on cores can be boosted to a higher
 16 * frequency under ITMT.
 17 */
 18
 19#include <linux/sched.h>
 20#include <linux/cpumask.h>
 21#include <linux/cpuset.h>
 22#include <linux/mutex.h>
 
 23#include <linux/sysctl.h>
 24#include <linux/nodemask.h>
 25
 26static DEFINE_MUTEX(itmt_update_mutex);
 27DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
 28
 29/* Boolean to track if system has ITMT capabilities */
 30static bool __read_mostly sched_itmt_capable;
 31
 32/*
 33 * Boolean to control whether we want to move processes to cpu capable
 34 * of higher turbo frequency for cpus supporting Intel Turbo Boost Max
 35 * Technology 3.0.
 36 *
 37 * It can be set via /proc/sys/kernel/sched_itmt_enabled
 38 */
 39unsigned int __read_mostly sysctl_sched_itmt_enabled;
 40
 41static int sched_itmt_update_handler(struct ctl_table *table, int write,
 42				     void *buffer, size_t *lenp, loff_t *ppos)
 
 43{
 44	unsigned int old_sysctl;
 45	int ret;
 46
 47	mutex_lock(&itmt_update_mutex);
 48
 49	if (!sched_itmt_capable) {
 50		mutex_unlock(&itmt_update_mutex);
 51		return -EINVAL;
 52	}
 53
 54	old_sysctl = sysctl_sched_itmt_enabled;
 55	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 56
 57	if (!ret && write && old_sysctl != sysctl_sched_itmt_enabled) {
 58		x86_topology_update = true;
 59		rebuild_sched_domains();
 60	}
 61
 62	mutex_unlock(&itmt_update_mutex);
 63
 64	return ret;
 65}
 66
 
 
 67static struct ctl_table itmt_kern_table[] = {
 68	{
 69		.procname	= "sched_itmt_enabled",
 70		.data		= &sysctl_sched_itmt_enabled,
 71		.maxlen		= sizeof(unsigned int),
 72		.mode		= 0644,
 73		.proc_handler	= sched_itmt_update_handler,
 74		.extra1		= SYSCTL_ZERO,
 75		.extra2		= SYSCTL_ONE,
 76	},
 77	{}
 78};
 79
 80static struct ctl_table itmt_root_table[] = {
 81	{
 82		.procname	= "kernel",
 83		.mode		= 0555,
 84		.child		= itmt_kern_table,
 85	},
 86	{}
 87};
 88
 89static struct ctl_table_header *itmt_sysctl_header;
 90
 91/**
 92 * sched_set_itmt_support() - Indicate platform supports ITMT
 93 *
 94 * This function is used by the OS to indicate to scheduler that the platform
 95 * is capable of supporting the ITMT feature.
 96 *
 97 * The current scheme has the pstate driver detects if the system
 98 * is ITMT capable and call sched_set_itmt_support.
 99 *
100 * This must be done only after sched_set_itmt_core_prio
101 * has been called to set the cpus' priorities.
102 * It must not be called with cpu hot plug lock
103 * held as we need to acquire the lock to rebuild sched domains
104 * later.
105 *
106 * Return: 0 on success
107 */
108int sched_set_itmt_support(void)
109{
110	mutex_lock(&itmt_update_mutex);
111
112	if (sched_itmt_capable) {
113		mutex_unlock(&itmt_update_mutex);
114		return 0;
115	}
116
117	itmt_sysctl_header = register_sysctl_table(itmt_root_table);
118	if (!itmt_sysctl_header) {
119		mutex_unlock(&itmt_update_mutex);
120		return -ENOMEM;
121	}
122
123	sched_itmt_capable = true;
124
125	sysctl_sched_itmt_enabled = 1;
126
127	x86_topology_update = true;
128	rebuild_sched_domains();
 
 
129
130	mutex_unlock(&itmt_update_mutex);
131
132	return 0;
133}
134
135/**
136 * sched_clear_itmt_support() - Revoke platform's support of ITMT
137 *
138 * This function is used by the OS to indicate that it has
139 * revoked the platform's support of ITMT feature.
140 *
141 * It must not be called with cpu hot plug lock
142 * held as we need to acquire the lock to rebuild sched domains
143 * later.
144 */
145void sched_clear_itmt_support(void)
146{
147	mutex_lock(&itmt_update_mutex);
148
149	if (!sched_itmt_capable) {
150		mutex_unlock(&itmt_update_mutex);
151		return;
152	}
153	sched_itmt_capable = false;
154
155	if (itmt_sysctl_header) {
156		unregister_sysctl_table(itmt_sysctl_header);
157		itmt_sysctl_header = NULL;
158	}
159
160	if (sysctl_sched_itmt_enabled) {
161		/* disable sched_itmt if we are no longer ITMT capable */
162		sysctl_sched_itmt_enabled = 0;
163		x86_topology_update = true;
164		rebuild_sched_domains();
165	}
166
167	mutex_unlock(&itmt_update_mutex);
168}
169
170int arch_asym_cpu_priority(int cpu)
171{
172	return per_cpu(sched_core_priority, cpu);
173}
174
175/**
176 * sched_set_itmt_core_prio() - Set CPU priority based on ITMT
177 * @prio:	Priority of cpu core
178 * @core_cpu:	The cpu number associated with the core
179 *
180 * The pstate driver will find out the max boost frequency
181 * and call this function to set a priority proportional
182 * to the max boost frequency. CPU with higher boost
183 * frequency will receive higher priority.
184 *
185 * No need to rebuild sched domain after updating
186 * the CPU priorities. The sched domains have no
187 * dependency on CPU priorities.
188 */
189void sched_set_itmt_core_prio(int prio, int core_cpu)
190{
191	int cpu, i = 1;
192
193	for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) {
194		int smt_prio;
195
196		/*
197		 * Ensure that the siblings are moved to the end
198		 * of the priority chain and only used when
199		 * all other high priority cpus are out of capacity.
200		 */
201		smt_prio = prio * smp_num_siblings / (i * i);
202		per_cpu(sched_core_priority, cpu) = smt_prio;
203		i++;
204	}
205}

 
  1/*
  2 * itmt.c: Support Intel Turbo Boost Max Technology 3.0
  3 *
  4 * (C) Copyright 2016 Intel Corporation
  5 * Author: Tim Chen <tim.c.chen@linux.intel.com>
  6 *
  7 * This program is free software; you can redistribute it and/or
  8 * modify it under the terms of the GNU General Public License
  9 * as published by the Free Software Foundation; version 2
 10 * of the License.
 11 *
 12 * On platforms supporting Intel Turbo Boost Max Technology 3.0, (ITMT),
 13 * the maximum turbo frequencies of some cores in a CPU package may be
 14 * higher than for the other cores in the same package.  In that case,
 15 * better performance can be achieved by making the scheduler prefer
 16 * to run tasks on the CPUs with higher max turbo frequencies.
 17 *
 18 * This file provides functions and data structures for enabling the
 19 * scheduler to favor scheduling on cores can be boosted to a higher
 20 * frequency under ITMT.
 21 */
 22
 23#include <linux/sched.h>
 24#include <linux/cpumask.h>
 25#include <linux/cpuset.h>
 26#include <linux/mutex.h>
 27#include <linux/sched.h>
 28#include <linux/sysctl.h>
 29#include <linux/nodemask.h>
 30
 31static DEFINE_MUTEX(itmt_update_mutex);
 32DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
 33
 34/* Boolean to track if system has ITMT capabilities */
 35static bool __read_mostly sched_itmt_capable;
 36
 37/*
 38 * Boolean to control whether we want to move processes to cpu capable
 39 * of higher turbo frequency for cpus supporting Intel Turbo Boost Max
 40 * Technology 3.0.
 41 *
 42 * It can be set via /proc/sys/kernel/sched_itmt_enabled
 43 */
 44unsigned int __read_mostly sysctl_sched_itmt_enabled;
 45
 46static int sched_itmt_update_handler(struct ctl_table *table, int write,
 47				     void __user *buffer, size_t *lenp,
 48				     loff_t *ppos)
 49{
 50	unsigned int old_sysctl;
 51	int ret;
 52
 53	mutex_lock(&itmt_update_mutex);
 54
 55	if (!sched_itmt_capable) {
 56		mutex_unlock(&itmt_update_mutex);
 57		return -EINVAL;
 58	}
 59
 60	old_sysctl = sysctl_sched_itmt_enabled;
 61	ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
 62
 63	if (!ret && write && old_sysctl != sysctl_sched_itmt_enabled) {
 64		x86_topology_update = true;
 65		rebuild_sched_domains();
 66	}
 67
 68	mutex_unlock(&itmt_update_mutex);
 69
 70	return ret;
 71}
 72
 73static unsigned int zero;
 74static unsigned int one = 1;
 75static struct ctl_table itmt_kern_table[] = {
 76	{
 77		.procname	= "sched_itmt_enabled",
 78		.data		= &sysctl_sched_itmt_enabled,
 79		.maxlen		= sizeof(unsigned int),
 80		.mode		= 0644,
 81		.proc_handler	= sched_itmt_update_handler,
 82		.extra1		= &zero,
 83		.extra2		= &one,
 84	},
 85	{}
 86};
 87
 88static struct ctl_table itmt_root_table[] = {
 89	{
 90		.procname	= "kernel",
 91		.mode		= 0555,
 92		.child		= itmt_kern_table,
 93	},
 94	{}
 95};
 96
 97static struct ctl_table_header *itmt_sysctl_header;
 98
 99/**
100 * sched_set_itmt_support() - Indicate platform supports ITMT
101 *
102 * This function is used by the OS to indicate to scheduler that the platform
103 * is capable of supporting the ITMT feature.
104 *
105 * The current scheme has the pstate driver detects if the system
106 * is ITMT capable and call sched_set_itmt_support.
107 *
108 * This must be done only after sched_set_itmt_core_prio
109 * has been called to set the cpus' priorities.
110 * It must not be called with cpu hot plug lock
111 * held as we need to acquire the lock to rebuild sched domains
112 * later.
113 *
114 * Return: 0 on success
115 */
116int sched_set_itmt_support(void)
117{
118	mutex_lock(&itmt_update_mutex);
119
120	if (sched_itmt_capable) {
121		mutex_unlock(&itmt_update_mutex);
122		return 0;
123	}
124
125	itmt_sysctl_header = register_sysctl_table(itmt_root_table);
126	if (!itmt_sysctl_header) {
127		mutex_unlock(&itmt_update_mutex);
128		return -ENOMEM;
129	}
130
131	sched_itmt_capable = true;
132
133	sysctl_sched_itmt_enabled = 1;
134
135	if (sysctl_sched_itmt_enabled) {
136		x86_topology_update = true;
137		rebuild_sched_domains();
138	}
139
140	mutex_unlock(&itmt_update_mutex);
141
142	return 0;
143}
144
145/**
146 * sched_clear_itmt_support() - Revoke platform's support of ITMT
147 *
148 * This function is used by the OS to indicate that it has
149 * revoked the platform's support of ITMT feature.
150 *
151 * It must not be called with cpu hot plug lock
152 * held as we need to acquire the lock to rebuild sched domains
153 * later.
154 */
155void sched_clear_itmt_support(void)
156{
157	mutex_lock(&itmt_update_mutex);
158
159	if (!sched_itmt_capable) {
160		mutex_unlock(&itmt_update_mutex);
161		return;
162	}
163	sched_itmt_capable = false;
164
165	if (itmt_sysctl_header) {
166		unregister_sysctl_table(itmt_sysctl_header);
167		itmt_sysctl_header = NULL;
168	}
169
170	if (sysctl_sched_itmt_enabled) {
171		/* disable sched_itmt if we are no longer ITMT capable */
172		sysctl_sched_itmt_enabled = 0;
173		x86_topology_update = true;
174		rebuild_sched_domains();
175	}
176
177	mutex_unlock(&itmt_update_mutex);
178}
179
180int arch_asym_cpu_priority(int cpu)
181{
182	return per_cpu(sched_core_priority, cpu);
183}
184
185/**
186 * sched_set_itmt_core_prio() - Set CPU priority based on ITMT
187 * @prio:	Priority of cpu core
188 * @core_cpu:	The cpu number associated with the core
189 *
190 * The pstate driver will find out the max boost frequency
191 * and call this function to set a priority proportional
192 * to the max boost frequency. CPU with higher boost
193 * frequency will receive higher priority.
194 *
195 * No need to rebuild sched domain after updating
196 * the CPU priorities. The sched domains have no
197 * dependency on CPU priorities.
198 */
199void sched_set_itmt_core_prio(int prio, int core_cpu)
200{
201	int cpu, i = 1;
202
203	for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) {
204		int smt_prio;
205
206		/*
207		 * Ensure that the siblings are moved to the end
208		 * of the priority chain and only used when
209		 * all other high priority cpus are out of capacity.
210		 */
211		smt_prio = prio * smp_num_siblings / i;
212		per_cpu(sched_core_priority, cpu) = smt_prio;
213		i++;
214	}
215}