Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include <linux/string_helpers.h>
7
8#include <drm/i915_drm.h>
9
10#include "display/intel_display.h"
11#include "display/intel_display_irq.h"
12#include "i915_drv.h"
13#include "i915_irq.h"
14#include "i915_reg.h"
15#include "intel_breadcrumbs.h"
16#include "intel_gt.h"
17#include "intel_gt_clock_utils.h"
18#include "intel_gt_irq.h"
19#include "intel_gt_pm.h"
20#include "intel_gt_pm_irq.h"
21#include "intel_gt_print.h"
22#include "intel_gt_regs.h"
23#include "intel_mchbar_regs.h"
24#include "intel_pcode.h"
25#include "intel_rps.h"
26#include "vlv_sideband.h"
27#include "../../../platform/x86/intel_ips.h"
28
29#define BUSY_MAX_EI 20u /* ms */
30
31/*
32 * Lock protecting IPS related data structures
33 */
34static DEFINE_SPINLOCK(mchdev_lock);
35
36static struct intel_gt *rps_to_gt(struct intel_rps *rps)
37{
38 return container_of(rps, struct intel_gt, rps);
39}
40
41static struct drm_i915_private *rps_to_i915(struct intel_rps *rps)
42{
43 return rps_to_gt(rps)->i915;
44}
45
46static struct intel_uncore *rps_to_uncore(struct intel_rps *rps)
47{
48 return rps_to_gt(rps)->uncore;
49}
50
51static struct intel_guc_slpc *rps_to_slpc(struct intel_rps *rps)
52{
53 struct intel_gt *gt = rps_to_gt(rps);
54
55 return >->uc.guc.slpc;
56}
57
58static bool rps_uses_slpc(struct intel_rps *rps)
59{
60 struct intel_gt *gt = rps_to_gt(rps);
61
62 return intel_uc_uses_guc_slpc(>->uc);
63}
64
65static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask)
66{
67 return mask & ~rps->pm_intrmsk_mbz;
68}
69
70static void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val)
71{
72 intel_uncore_write_fw(uncore, reg, val);
73}
74
75static void rps_timer(struct timer_list *t)
76{
77 struct intel_rps *rps = from_timer(rps, t, timer);
78 struct intel_gt *gt = rps_to_gt(rps);
79 struct intel_engine_cs *engine;
80 ktime_t dt, last, timestamp;
81 enum intel_engine_id id;
82 s64 max_busy[3] = {};
83
84 timestamp = 0;
85 for_each_engine(engine, gt, id) {
86 s64 busy;
87 int i;
88
89 dt = intel_engine_get_busy_time(engine, ×tamp);
90 last = engine->stats.rps;
91 engine->stats.rps = dt;
92
93 busy = ktime_to_ns(ktime_sub(dt, last));
94 for (i = 0; i < ARRAY_SIZE(max_busy); i++) {
95 if (busy > max_busy[i])
96 swap(busy, max_busy[i]);
97 }
98 }
99 last = rps->pm_timestamp;
100 rps->pm_timestamp = timestamp;
101
102 if (intel_rps_is_active(rps)) {
103 s64 busy;
104 int i;
105
106 dt = ktime_sub(timestamp, last);
107
108 /*
109 * Our goal is to evaluate each engine independently, so we run
110 * at the lowest clocks required to sustain the heaviest
111 * workload. However, a task may be split into sequential
112 * dependent operations across a set of engines, such that
113 * the independent contributions do not account for high load,
114 * but overall the task is GPU bound. For example, consider
115 * video decode on vcs followed by colour post-processing
116 * on vecs, followed by general post-processing on rcs.
117 * Since multi-engines being active does imply a single
118 * continuous workload across all engines, we hedge our
119 * bets by only contributing a factor of the distributed
120 * load into our busyness calculation.
121 */
122 busy = max_busy[0];
123 for (i = 1; i < ARRAY_SIZE(max_busy); i++) {
124 if (!max_busy[i])
125 break;
126
127 busy += div_u64(max_busy[i], 1 << i);
128 }
129 GT_TRACE(gt,
130 "busy:%lld [%d%%], max:[%lld, %lld, %lld], interval:%d\n",
131 busy, (int)div64_u64(100 * busy, dt),
132 max_busy[0], max_busy[1], max_busy[2],
133 rps->pm_interval);
134
135 if (100 * busy > rps->power.up_threshold * dt &&
136 rps->cur_freq < rps->max_freq_softlimit) {
137 rps->pm_iir |= GEN6_PM_RP_UP_THRESHOLD;
138 rps->pm_interval = 1;
139 queue_work(gt->i915->unordered_wq, &rps->work);
140 } else if (100 * busy < rps->power.down_threshold * dt &&
141 rps->cur_freq > rps->min_freq_softlimit) {
142 rps->pm_iir |= GEN6_PM_RP_DOWN_THRESHOLD;
143 rps->pm_interval = 1;
144 queue_work(gt->i915->unordered_wq, &rps->work);
145 } else {
146 rps->last_adj = 0;
147 }
148
149 mod_timer(&rps->timer,
150 jiffies + msecs_to_jiffies(rps->pm_interval));
151 rps->pm_interval = min(rps->pm_interval * 2, BUSY_MAX_EI);
152 }
153}
154
155static void rps_start_timer(struct intel_rps *rps)
156{
157 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
158 rps->pm_interval = 1;
159 mod_timer(&rps->timer, jiffies + 1);
160}
161
162static void rps_stop_timer(struct intel_rps *rps)
163{
164 del_timer_sync(&rps->timer);
165 rps->pm_timestamp = ktime_sub(ktime_get(), rps->pm_timestamp);
166 cancel_work_sync(&rps->work);
167}
168
169static u32 rps_pm_mask(struct intel_rps *rps, u8 val)
170{
171 u32 mask = 0;
172
173 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */
174 if (val > rps->min_freq_softlimit)
175 mask |= (GEN6_PM_RP_UP_EI_EXPIRED |
176 GEN6_PM_RP_DOWN_THRESHOLD |
177 GEN6_PM_RP_DOWN_TIMEOUT);
178
179 if (val < rps->max_freq_softlimit)
180 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD;
181
182 mask &= rps->pm_events;
183
184 return rps_pm_sanitize_mask(rps, ~mask);
185}
186
187static void rps_reset_ei(struct intel_rps *rps)
188{
189 memset(&rps->ei, 0, sizeof(rps->ei));
190}
191
192static void rps_enable_interrupts(struct intel_rps *rps)
193{
194 struct intel_gt *gt = rps_to_gt(rps);
195
196 GEM_BUG_ON(rps_uses_slpc(rps));
197
198 GT_TRACE(gt, "interrupts:on rps->pm_events: %x, rps_pm_mask:%x\n",
199 rps->pm_events, rps_pm_mask(rps, rps->last_freq));
200
201 rps_reset_ei(rps);
202
203 spin_lock_irq(gt->irq_lock);
204 gen6_gt_pm_enable_irq(gt, rps->pm_events);
205 spin_unlock_irq(gt->irq_lock);
206
207 intel_uncore_write(gt->uncore,
208 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq));
209}
210
211static void gen6_rps_reset_interrupts(struct intel_rps *rps)
212{
213 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS);
214}
215
216static void gen11_rps_reset_interrupts(struct intel_rps *rps)
217{
218 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM))
219 ;
220}
221
222static void rps_reset_interrupts(struct intel_rps *rps)
223{
224 struct intel_gt *gt = rps_to_gt(rps);
225
226 spin_lock_irq(gt->irq_lock);
227 if (GRAPHICS_VER(gt->i915) >= 11)
228 gen11_rps_reset_interrupts(rps);
229 else
230 gen6_rps_reset_interrupts(rps);
231
232 rps->pm_iir = 0;
233 spin_unlock_irq(gt->irq_lock);
234}
235
236static void rps_disable_interrupts(struct intel_rps *rps)
237{
238 struct intel_gt *gt = rps_to_gt(rps);
239
240 intel_uncore_write(gt->uncore,
241 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u));
242
243 spin_lock_irq(gt->irq_lock);
244 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS);
245 spin_unlock_irq(gt->irq_lock);
246
247 intel_synchronize_irq(gt->i915);
248
249 /*
250 * Now that we will not be generating any more work, flush any
251 * outstanding tasks. As we are called on the RPS idle path,
252 * we will reset the GPU to minimum frequencies, so the current
253 * state of the worker can be discarded.
254 */
255 cancel_work_sync(&rps->work);
256
257 rps_reset_interrupts(rps);
258 GT_TRACE(gt, "interrupts:off\n");
259}
260
261static const struct cparams {
262 u16 i;
263 u16 t;
264 u16 m;
265 u16 c;
266} cparams[] = {
267 { 1, 1333, 301, 28664 },
268 { 1, 1066, 294, 24460 },
269 { 1, 800, 294, 25192 },
270 { 0, 1333, 276, 27605 },
271 { 0, 1066, 276, 27605 },
272 { 0, 800, 231, 23784 },
273};
274
275static void gen5_rps_init(struct intel_rps *rps)
276{
277 struct drm_i915_private *i915 = rps_to_i915(rps);
278 struct intel_uncore *uncore = rps_to_uncore(rps);
279 u8 fmax, fmin, fstart;
280 u32 rgvmodectl;
281 int c_m, i;
282
283 if (i915->fsb_freq <= 3200)
284 c_m = 0;
285 else if (i915->fsb_freq <= 4800)
286 c_m = 1;
287 else
288 c_m = 2;
289
290 for (i = 0; i < ARRAY_SIZE(cparams); i++) {
291 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) {
292 rps->ips.m = cparams[i].m;
293 rps->ips.c = cparams[i].c;
294 break;
295 }
296 }
297
298 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
299
300 /* Set up min, max, and cur for interrupt handling */
301 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
302 fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
303 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
304 MEMMODE_FSTART_SHIFT;
305 drm_dbg(&i915->drm, "fmax: %d, fmin: %d, fstart: %d\n",
306 fmax, fmin, fstart);
307
308 rps->min_freq = fmax;
309 rps->efficient_freq = fstart;
310 rps->max_freq = fmin;
311}
312
313static unsigned long
314__ips_chipset_val(struct intel_ips *ips)
315{
316 struct intel_uncore *uncore =
317 rps_to_uncore(container_of(ips, struct intel_rps, ips));
318 unsigned long now = jiffies_to_msecs(jiffies), dt;
319 unsigned long result;
320 u64 total, delta;
321
322 lockdep_assert_held(&mchdev_lock);
323
324 /*
325 * Prevent division-by-zero if we are asking too fast.
326 * Also, we don't get interesting results if we are polling
327 * faster than once in 10ms, so just return the saved value
328 * in such cases.
329 */
330 dt = now - ips->last_time1;
331 if (dt <= 10)
332 return ips->chipset_power;
333
334 /* FIXME: handle per-counter overflow */
335 total = intel_uncore_read(uncore, DMIEC);
336 total += intel_uncore_read(uncore, DDREC);
337 total += intel_uncore_read(uncore, CSIEC);
338
339 delta = total - ips->last_count1;
340
341 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10);
342
343 ips->last_count1 = total;
344 ips->last_time1 = now;
345
346 ips->chipset_power = result;
347
348 return result;
349}
350
351static unsigned long ips_mch_val(struct intel_uncore *uncore)
352{
353 unsigned int m, x, b;
354 u32 tsfs;
355
356 tsfs = intel_uncore_read(uncore, TSFS);
357 x = intel_uncore_read8(uncore, TR1);
358
359 b = tsfs & TSFS_INTR_MASK;
360 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT;
361
362 return m * x / 127 - b;
363}
364
365static int _pxvid_to_vd(u8 pxvid)
366{
367 if (pxvid == 0)
368 return 0;
369
370 if (pxvid >= 8 && pxvid < 31)
371 pxvid = 31;
372
373 return (pxvid + 2) * 125;
374}
375
376static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid)
377{
378 const int vd = _pxvid_to_vd(pxvid);
379
380 if (INTEL_INFO(i915)->is_mobile)
381 return max(vd - 1125, 0);
382
383 return vd;
384}
385
386static void __gen5_ips_update(struct intel_ips *ips)
387{
388 struct intel_uncore *uncore =
389 rps_to_uncore(container_of(ips, struct intel_rps, ips));
390 u64 now, delta, dt;
391 u32 count;
392
393 lockdep_assert_held(&mchdev_lock);
394
395 now = ktime_get_raw_ns();
396 dt = now - ips->last_time2;
397 do_div(dt, NSEC_PER_MSEC);
398
399 /* Don't divide by 0 */
400 if (dt <= 10)
401 return;
402
403 count = intel_uncore_read(uncore, GFXEC);
404 delta = count - ips->last_count2;
405
406 ips->last_count2 = count;
407 ips->last_time2 = now;
408
409 /* More magic constants... */
410 ips->gfx_power = div_u64(delta * 1181, dt * 10);
411}
412
413static void gen5_rps_update(struct intel_rps *rps)
414{
415 spin_lock_irq(&mchdev_lock);
416 __gen5_ips_update(&rps->ips);
417 spin_unlock_irq(&mchdev_lock);
418}
419
420static unsigned int gen5_invert_freq(struct intel_rps *rps,
421 unsigned int val)
422{
423 /* Invert the frequency bin into an ips delay */
424 val = rps->max_freq - val;
425 val = rps->min_freq + val;
426
427 return val;
428}
429
430static int __gen5_rps_set(struct intel_rps *rps, u8 val)
431{
432 struct intel_uncore *uncore = rps_to_uncore(rps);
433 u16 rgvswctl;
434
435 lockdep_assert_held(&mchdev_lock);
436
437 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
438 if (rgvswctl & MEMCTL_CMD_STS) {
439 drm_dbg(&rps_to_i915(rps)->drm,
440 "gpu busy, RCS change rejected\n");
441 return -EBUSY; /* still busy with another command */
442 }
443
444 /* Invert the frequency bin into an ips delay */
445 val = gen5_invert_freq(rps, val);
446
447 rgvswctl =
448 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
449 (val << MEMCTL_FREQ_SHIFT) |
450 MEMCTL_SFCAVM;
451 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
452 intel_uncore_posting_read16(uncore, MEMSWCTL);
453
454 rgvswctl |= MEMCTL_CMD_STS;
455 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl);
456
457 return 0;
458}
459
460static int gen5_rps_set(struct intel_rps *rps, u8 val)
461{
462 int err;
463
464 spin_lock_irq(&mchdev_lock);
465 err = __gen5_rps_set(rps, val);
466 spin_unlock_irq(&mchdev_lock);
467
468 return err;
469}
470
471static unsigned long intel_pxfreq(u32 vidfreq)
472{
473 int div = (vidfreq & 0x3f0000) >> 16;
474 int post = (vidfreq & 0x3000) >> 12;
475 int pre = (vidfreq & 0x7);
476
477 if (!pre)
478 return 0;
479
480 return div * 133333 / (pre << post);
481}
482
483static unsigned int init_emon(struct intel_uncore *uncore)
484{
485 u8 pxw[16];
486 int i;
487
488 /* Disable to program */
489 intel_uncore_write(uncore, ECR, 0);
490 intel_uncore_posting_read(uncore, ECR);
491
492 /* Program energy weights for various events */
493 intel_uncore_write(uncore, SDEW, 0x15040d00);
494 intel_uncore_write(uncore, CSIEW0, 0x007f0000);
495 intel_uncore_write(uncore, CSIEW1, 0x1e220004);
496 intel_uncore_write(uncore, CSIEW2, 0x04000004);
497
498 for (i = 0; i < 5; i++)
499 intel_uncore_write(uncore, PEW(i), 0);
500 for (i = 0; i < 3; i++)
501 intel_uncore_write(uncore, DEW(i), 0);
502
503 /* Program P-state weights to account for frequency power adjustment */
504 for (i = 0; i < 16; i++) {
505 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i));
506 unsigned int freq = intel_pxfreq(pxvidfreq);
507 unsigned int vid =
508 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
509 unsigned int val;
510
511 val = vid * vid * freq / 1000 * 255;
512 val /= 127 * 127 * 900;
513
514 pxw[i] = val;
515 }
516 /* Render standby states get 0 weight */
517 pxw[14] = 0;
518 pxw[15] = 0;
519
520 for (i = 0; i < 4; i++) {
521 intel_uncore_write(uncore, PXW(i),
522 pxw[i * 4 + 0] << 24 |
523 pxw[i * 4 + 1] << 16 |
524 pxw[i * 4 + 2] << 8 |
525 pxw[i * 4 + 3] << 0);
526 }
527
528 /* Adjust magic regs to magic values (more experimental results) */
529 intel_uncore_write(uncore, OGW0, 0);
530 intel_uncore_write(uncore, OGW1, 0);
531 intel_uncore_write(uncore, EG0, 0x00007f00);
532 intel_uncore_write(uncore, EG1, 0x0000000e);
533 intel_uncore_write(uncore, EG2, 0x000e0000);
534 intel_uncore_write(uncore, EG3, 0x68000300);
535 intel_uncore_write(uncore, EG4, 0x42000000);
536 intel_uncore_write(uncore, EG5, 0x00140031);
537 intel_uncore_write(uncore, EG6, 0);
538 intel_uncore_write(uncore, EG7, 0);
539
540 for (i = 0; i < 8; i++)
541 intel_uncore_write(uncore, PXWL(i), 0);
542
543 /* Enable PMON + select events */
544 intel_uncore_write(uncore, ECR, 0x80000019);
545
546 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK;
547}
548
549static bool gen5_rps_enable(struct intel_rps *rps)
550{
551 struct drm_i915_private *i915 = rps_to_i915(rps);
552 struct intel_uncore *uncore = rps_to_uncore(rps);
553 u8 fstart, vstart;
554 u32 rgvmodectl;
555
556 spin_lock_irq(&mchdev_lock);
557
558 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL);
559
560 /* Enable temp reporting */
561 intel_uncore_write16(uncore, PMMISC,
562 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN);
563 intel_uncore_write16(uncore, TSC1,
564 intel_uncore_read16(uncore, TSC1) | TSE);
565
566 /* 100ms RC evaluation intervals */
567 intel_uncore_write(uncore, RCUPEI, 100000);
568 intel_uncore_write(uncore, RCDNEI, 100000);
569
570 /* Set max/min thresholds to 90ms and 80ms respectively */
571 intel_uncore_write(uncore, RCBMAXAVG, 90000);
572 intel_uncore_write(uncore, RCBMINAVG, 80000);
573
574 intel_uncore_write(uncore, MEMIHYST, 1);
575
576 /* Set up min, max, and cur for interrupt handling */
577 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
578 MEMMODE_FSTART_SHIFT;
579
580 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) &
581 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT;
582
583 intel_uncore_write(uncore,
584 MEMINTREN,
585 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
586
587 intel_uncore_write(uncore, VIDSTART, vstart);
588 intel_uncore_posting_read(uncore, VIDSTART);
589
590 rgvmodectl |= MEMMODE_SWMODE_EN;
591 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl);
592
593 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) &
594 MEMCTL_CMD_STS) == 0, 10))
595 drm_err(&uncore->i915->drm,
596 "stuck trying to change perf mode\n");
597 mdelay(1);
598
599 __gen5_rps_set(rps, rps->cur_freq);
600
601 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC);
602 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC);
603 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC);
604 rps->ips.last_time1 = jiffies_to_msecs(jiffies);
605
606 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC);
607 rps->ips.last_time2 = ktime_get_raw_ns();
608
609 spin_lock(&i915->irq_lock);
610 ilk_enable_display_irq(i915, DE_PCU_EVENT);
611 spin_unlock(&i915->irq_lock);
612
613 spin_unlock_irq(&mchdev_lock);
614
615 rps->ips.corr = init_emon(uncore);
616
617 return true;
618}
619
620static void gen5_rps_disable(struct intel_rps *rps)
621{
622 struct drm_i915_private *i915 = rps_to_i915(rps);
623 struct intel_uncore *uncore = rps_to_uncore(rps);
624 u16 rgvswctl;
625
626 spin_lock_irq(&mchdev_lock);
627
628 spin_lock(&i915->irq_lock);
629 ilk_disable_display_irq(i915, DE_PCU_EVENT);
630 spin_unlock(&i915->irq_lock);
631
632 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL);
633
634 /* Ack interrupts, disable EFC interrupt */
635 intel_uncore_rmw(uncore, MEMINTREN, MEMINT_EVAL_CHG_EN, 0);
636 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
637
638 /* Go back to the starting frequency */
639 __gen5_rps_set(rps, rps->idle_freq);
640 mdelay(1);
641 rgvswctl |= MEMCTL_CMD_STS;
642 intel_uncore_write(uncore, MEMSWCTL, rgvswctl);
643 mdelay(1);
644
645 spin_unlock_irq(&mchdev_lock);
646}
647
648static u32 rps_limits(struct intel_rps *rps, u8 val)
649{
650 u32 limits;
651
652 /*
653 * Only set the down limit when we've reached the lowest level to avoid
654 * getting more interrupts, otherwise leave this clear. This prevents a
655 * race in the hw when coming out of rc6: There's a tiny window where
656 * the hw runs at the minimal clock before selecting the desired
657 * frequency, if the down threshold expires in that window we will not
658 * receive a down interrupt.
659 */
660 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
661 limits = rps->max_freq_softlimit << 23;
662 if (val <= rps->min_freq_softlimit)
663 limits |= rps->min_freq_softlimit << 14;
664 } else {
665 limits = rps->max_freq_softlimit << 24;
666 if (val <= rps->min_freq_softlimit)
667 limits |= rps->min_freq_softlimit << 16;
668 }
669
670 return limits;
671}
672
673static void rps_set_power(struct intel_rps *rps, int new_power)
674{
675 struct intel_gt *gt = rps_to_gt(rps);
676 struct intel_uncore *uncore = gt->uncore;
677 u32 ei_up = 0, ei_down = 0;
678
679 lockdep_assert_held(&rps->power.mutex);
680
681 if (new_power == rps->power.mode)
682 return;
683
684 /* Note the units here are not exactly 1us, but 1280ns. */
685 switch (new_power) {
686 case LOW_POWER:
687 ei_up = 16000;
688 ei_down = 32000;
689 break;
690
691 case BETWEEN:
692 ei_up = 13000;
693 ei_down = 32000;
694 break;
695
696 case HIGH_POWER:
697 ei_up = 10000;
698 ei_down = 32000;
699 break;
700 }
701
702 /* When byt can survive without system hang with dynamic
703 * sw freq adjustments, this restriction can be lifted.
704 */
705 if (IS_VALLEYVIEW(gt->i915))
706 goto skip_hw_write;
707
708 GT_TRACE(gt,
709 "changing power mode [%d], up %d%% @ %dus, down %d%% @ %dus\n",
710 new_power,
711 rps->power.up_threshold, ei_up,
712 rps->power.down_threshold, ei_down);
713
714 set(uncore, GEN6_RP_UP_EI,
715 intel_gt_ns_to_pm_interval(gt, ei_up * 1000));
716 set(uncore, GEN6_RP_UP_THRESHOLD,
717 intel_gt_ns_to_pm_interval(gt,
718 ei_up * rps->power.up_threshold * 10));
719
720 set(uncore, GEN6_RP_DOWN_EI,
721 intel_gt_ns_to_pm_interval(gt, ei_down * 1000));
722 set(uncore, GEN6_RP_DOWN_THRESHOLD,
723 intel_gt_ns_to_pm_interval(gt,
724 ei_down *
725 rps->power.down_threshold * 10));
726
727 set(uncore, GEN6_RP_CONTROL,
728 (GRAPHICS_VER(gt->i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) |
729 GEN6_RP_MEDIA_HW_NORMAL_MODE |
730 GEN6_RP_MEDIA_IS_GFX |
731 GEN6_RP_ENABLE |
732 GEN6_RP_UP_BUSY_AVG |
733 GEN6_RP_DOWN_IDLE_AVG);
734
735skip_hw_write:
736 rps->power.mode = new_power;
737}
738
739static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val)
740{
741 int new_power;
742
743 new_power = rps->power.mode;
744 switch (rps->power.mode) {
745 case LOW_POWER:
746 if (val > rps->efficient_freq + 1 &&
747 val > rps->cur_freq)
748 new_power = BETWEEN;
749 break;
750
751 case BETWEEN:
752 if (val <= rps->efficient_freq &&
753 val < rps->cur_freq)
754 new_power = LOW_POWER;
755 else if (val >= rps->rp0_freq &&
756 val > rps->cur_freq)
757 new_power = HIGH_POWER;
758 break;
759
760 case HIGH_POWER:
761 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 &&
762 val < rps->cur_freq)
763 new_power = BETWEEN;
764 break;
765 }
766 /* Max/min bins are special */
767 if (val <= rps->min_freq_softlimit)
768 new_power = LOW_POWER;
769 if (val >= rps->max_freq_softlimit)
770 new_power = HIGH_POWER;
771
772 mutex_lock(&rps->power.mutex);
773 if (rps->power.interactive)
774 new_power = HIGH_POWER;
775 rps_set_power(rps, new_power);
776 mutex_unlock(&rps->power.mutex);
777}
778
779void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive)
780{
781 GT_TRACE(rps_to_gt(rps), "mark interactive: %s\n",
782 str_yes_no(interactive));
783
784 mutex_lock(&rps->power.mutex);
785 if (interactive) {
786 if (!rps->power.interactive++ && intel_rps_is_active(rps))
787 rps_set_power(rps, HIGH_POWER);
788 } else {
789 GEM_BUG_ON(!rps->power.interactive);
790 rps->power.interactive--;
791 }
792 mutex_unlock(&rps->power.mutex);
793}
794
795static int gen6_rps_set(struct intel_rps *rps, u8 val)
796{
797 struct intel_uncore *uncore = rps_to_uncore(rps);
798 struct drm_i915_private *i915 = rps_to_i915(rps);
799 u32 swreq;
800
801 GEM_BUG_ON(rps_uses_slpc(rps));
802
803 if (GRAPHICS_VER(i915) >= 9)
804 swreq = GEN9_FREQUENCY(val);
805 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
806 swreq = HSW_FREQUENCY(val);
807 else
808 swreq = (GEN6_FREQUENCY(val) |
809 GEN6_OFFSET(0) |
810 GEN6_AGGRESSIVE_TURBO);
811 set(uncore, GEN6_RPNSWREQ, swreq);
812
813 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d, swreq:%x\n",
814 val, intel_gpu_freq(rps, val), swreq);
815
816 return 0;
817}
818
819static int vlv_rps_set(struct intel_rps *rps, u8 val)
820{
821 struct drm_i915_private *i915 = rps_to_i915(rps);
822 int err;
823
824 vlv_punit_get(i915);
825 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val);
826 vlv_punit_put(i915);
827
828 GT_TRACE(rps_to_gt(rps), "set val:%x, freq:%d\n",
829 val, intel_gpu_freq(rps, val));
830
831 return err;
832}
833
834static int rps_set(struct intel_rps *rps, u8 val, bool update)
835{
836 struct drm_i915_private *i915 = rps_to_i915(rps);
837 int err;
838
839 if (val == rps->last_freq)
840 return 0;
841
842 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
843 err = vlv_rps_set(rps, val);
844 else if (GRAPHICS_VER(i915) >= 6)
845 err = gen6_rps_set(rps, val);
846 else
847 err = gen5_rps_set(rps, val);
848 if (err)
849 return err;
850
851 if (update && GRAPHICS_VER(i915) >= 6)
852 gen6_rps_set_thresholds(rps, val);
853 rps->last_freq = val;
854
855 return 0;
856}
857
858void intel_rps_unpark(struct intel_rps *rps)
859{
860 if (!intel_rps_is_enabled(rps))
861 return;
862
863 GT_TRACE(rps_to_gt(rps), "unpark:%x\n", rps->cur_freq);
864
865 /*
866 * Use the user's desired frequency as a guide, but for better
867 * performance, jump directly to RPe as our starting frequency.
868 */
869 mutex_lock(&rps->lock);
870
871 intel_rps_set_active(rps);
872 intel_rps_set(rps,
873 clamp(rps->cur_freq,
874 rps->min_freq_softlimit,
875 rps->max_freq_softlimit));
876
877 mutex_unlock(&rps->lock);
878
879 rps->pm_iir = 0;
880 if (intel_rps_has_interrupts(rps))
881 rps_enable_interrupts(rps);
882 if (intel_rps_uses_timer(rps))
883 rps_start_timer(rps);
884
885 if (GRAPHICS_VER(rps_to_i915(rps)) == 5)
886 gen5_rps_update(rps);
887}
888
889void intel_rps_park(struct intel_rps *rps)
890{
891 int adj;
892
893 if (!intel_rps_is_enabled(rps))
894 return;
895
896 if (!intel_rps_clear_active(rps))
897 return;
898
899 if (intel_rps_uses_timer(rps))
900 rps_stop_timer(rps);
901 if (intel_rps_has_interrupts(rps))
902 rps_disable_interrupts(rps);
903
904 if (rps->last_freq <= rps->idle_freq)
905 return;
906
907 /*
908 * The punit delays the write of the frequency and voltage until it
909 * determines the GPU is awake. During normal usage we don't want to
910 * waste power changing the frequency if the GPU is sleeping (rc6).
911 * However, the GPU and driver is now idle and we do not want to delay
912 * switching to minimum voltage (reducing power whilst idle) as we do
913 * not expect to be woken in the near future and so must flush the
914 * change by waking the device.
915 *
916 * We choose to take the media powerwell (either would do to trick the
917 * punit into committing the voltage change) as that takes a lot less
918 * power than the render powerwell.
919 */
920 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA);
921 rps_set(rps, rps->idle_freq, false);
922 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA);
923
924 /*
925 * Since we will try and restart from the previously requested
926 * frequency on unparking, treat this idle point as a downclock
927 * interrupt and reduce the frequency for resume. If we park/unpark
928 * more frequently than the rps worker can run, we will not respond
929 * to any EI and never see a change in frequency.
930 *
931 * (Note we accommodate Cherryview's limitation of only using an
932 * even bin by applying it to all.)
933 */
934 adj = rps->last_adj;
935 if (adj < 0)
936 adj *= 2;
937 else /* CHV needs even encode values */
938 adj = -2;
939 rps->last_adj = adj;
940 rps->cur_freq = max_t(int, rps->cur_freq + adj, rps->min_freq);
941 if (rps->cur_freq < rps->efficient_freq) {
942 rps->cur_freq = rps->efficient_freq;
943 rps->last_adj = 0;
944 }
945
946 GT_TRACE(rps_to_gt(rps), "park:%x\n", rps->cur_freq);
947}
948
949u32 intel_rps_get_boost_frequency(struct intel_rps *rps)
950{
951 struct intel_guc_slpc *slpc;
952
953 if (rps_uses_slpc(rps)) {
954 slpc = rps_to_slpc(rps);
955
956 return slpc->boost_freq;
957 } else {
958 return intel_gpu_freq(rps, rps->boost_freq);
959 }
960}
961
962static int rps_set_boost_freq(struct intel_rps *rps, u32 val)
963{
964 bool boost = false;
965
966 /* Validate against (static) hardware limits */
967 val = intel_freq_opcode(rps, val);
968 if (val < rps->min_freq || val > rps->max_freq)
969 return -EINVAL;
970
971 mutex_lock(&rps->lock);
972 if (val != rps->boost_freq) {
973 rps->boost_freq = val;
974 boost = atomic_read(&rps->num_waiters);
975 }
976 mutex_unlock(&rps->lock);
977 if (boost)
978 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);
979
980 return 0;
981}
982
983int intel_rps_set_boost_frequency(struct intel_rps *rps, u32 freq)
984{
985 struct intel_guc_slpc *slpc;
986
987 if (rps_uses_slpc(rps)) {
988 slpc = rps_to_slpc(rps);
989
990 return intel_guc_slpc_set_boost_freq(slpc, freq);
991 } else {
992 return rps_set_boost_freq(rps, freq);
993 }
994}
995
996void intel_rps_dec_waiters(struct intel_rps *rps)
997{
998 struct intel_guc_slpc *slpc;
999
1000 if (rps_uses_slpc(rps)) {
1001 slpc = rps_to_slpc(rps);
1002
1003 intel_guc_slpc_dec_waiters(slpc);
1004 } else {
1005 atomic_dec(&rps->num_waiters);
1006 }
1007}
1008
1009void intel_rps_boost(struct i915_request *rq)
1010{
1011 struct intel_guc_slpc *slpc;
1012
1013 if (i915_request_signaled(rq) || i915_request_has_waitboost(rq))
1014 return;
1015
1016 /* Serializes with i915_request_retire() */
1017 if (!test_and_set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags)) {
1018 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps;
1019
1020 if (rps_uses_slpc(rps)) {
1021 slpc = rps_to_slpc(rps);
1022
1023 if (slpc->min_freq_softlimit >= slpc->boost_freq)
1024 return;
1025
1026 /* Return if old value is non zero */
1027 if (!atomic_fetch_inc(&slpc->num_waiters)) {
1028 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1029 rq->fence.context, rq->fence.seqno);
1030 queue_work(rps_to_gt(rps)->i915->unordered_wq,
1031 &slpc->boost_work);
1032 }
1033
1034 return;
1035 }
1036
1037 if (atomic_fetch_inc(&rps->num_waiters))
1038 return;
1039
1040 if (!intel_rps_is_active(rps))
1041 return;
1042
1043 GT_TRACE(rps_to_gt(rps), "boost fence:%llx:%llx\n",
1044 rq->fence.context, rq->fence.seqno);
1045
1046 if (READ_ONCE(rps->cur_freq) < rps->boost_freq)
1047 queue_work(rps_to_gt(rps)->i915->unordered_wq, &rps->work);
1048
1049 WRITE_ONCE(rps->boosts, rps->boosts + 1); /* debug only */
1050 }
1051}
1052
1053int intel_rps_set(struct intel_rps *rps, u8 val)
1054{
1055 int err;
1056
1057 lockdep_assert_held(&rps->lock);
1058 GEM_BUG_ON(val > rps->max_freq);
1059 GEM_BUG_ON(val < rps->min_freq);
1060
1061 if (intel_rps_is_active(rps)) {
1062 err = rps_set(rps, val, true);
1063 if (err)
1064 return err;
1065
1066 /*
1067 * Make sure we continue to get interrupts
1068 * until we hit the minimum or maximum frequencies.
1069 */
1070 if (intel_rps_has_interrupts(rps)) {
1071 struct intel_uncore *uncore = rps_to_uncore(rps);
1072
1073 set(uncore,
1074 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val));
1075
1076 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val));
1077 }
1078 }
1079
1080 rps->cur_freq = val;
1081 return 0;
1082}
1083
1084static u32 intel_rps_read_state_cap(struct intel_rps *rps)
1085{
1086 struct drm_i915_private *i915 = rps_to_i915(rps);
1087 struct intel_uncore *uncore = rps_to_uncore(rps);
1088
1089 if (IS_PONTEVECCHIO(i915))
1090 return intel_uncore_read(uncore, PVC_RP_STATE_CAP);
1091 else if (IS_XEHPSDV(i915))
1092 return intel_uncore_read(uncore, XEHPSDV_RP_STATE_CAP);
1093 else if (IS_GEN9_LP(i915))
1094 return intel_uncore_read(uncore, BXT_RP_STATE_CAP);
1095 else
1096 return intel_uncore_read(uncore, GEN6_RP_STATE_CAP);
1097}
1098
1099static void
1100mtl_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1101{
1102 struct intel_uncore *uncore = rps_to_uncore(rps);
1103 u32 rp_state_cap = rps_to_gt(rps)->type == GT_MEDIA ?
1104 intel_uncore_read(uncore, MTL_MEDIAP_STATE_CAP) :
1105 intel_uncore_read(uncore, MTL_RP_STATE_CAP);
1106 u32 rpe = rps_to_gt(rps)->type == GT_MEDIA ?
1107 intel_uncore_read(uncore, MTL_MPE_FREQUENCY) :
1108 intel_uncore_read(uncore, MTL_GT_RPE_FREQUENCY);
1109
1110 /* MTL values are in units of 16.67 MHz */
1111 caps->rp0_freq = REG_FIELD_GET(MTL_RP0_CAP_MASK, rp_state_cap);
1112 caps->min_freq = REG_FIELD_GET(MTL_RPN_CAP_MASK, rp_state_cap);
1113 caps->rp1_freq = REG_FIELD_GET(MTL_RPE_MASK, rpe);
1114}
1115
1116static void
1117__gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1118{
1119 struct drm_i915_private *i915 = rps_to_i915(rps);
1120 u32 rp_state_cap;
1121
1122 rp_state_cap = intel_rps_read_state_cap(rps);
1123
1124 /* static values from HW: RP0 > RP1 > RPn (min_freq) */
1125 if (IS_GEN9_LP(i915)) {
1126 caps->rp0_freq = (rp_state_cap >> 16) & 0xff;
1127 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1128 caps->min_freq = (rp_state_cap >> 0) & 0xff;
1129 } else {
1130 caps->rp0_freq = (rp_state_cap >> 0) & 0xff;
1131 if (GRAPHICS_VER(i915) >= 10)
1132 caps->rp1_freq = REG_FIELD_GET(RPE_MASK,
1133 intel_uncore_read(to_gt(i915)->uncore,
1134 GEN10_FREQ_INFO_REC));
1135 else
1136 caps->rp1_freq = (rp_state_cap >> 8) & 0xff;
1137 caps->min_freq = (rp_state_cap >> 16) & 0xff;
1138 }
1139
1140 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1141 /*
1142 * In this case rp_state_cap register reports frequencies in
1143 * units of 50 MHz. Convert these to the actual "hw unit", i.e.
1144 * units of 16.67 MHz
1145 */
1146 caps->rp0_freq *= GEN9_FREQ_SCALER;
1147 caps->rp1_freq *= GEN9_FREQ_SCALER;
1148 caps->min_freq *= GEN9_FREQ_SCALER;
1149 }
1150}
1151
1152/**
1153 * gen6_rps_get_freq_caps - Get freq caps exposed by HW
1154 * @rps: the intel_rps structure
1155 * @caps: returned freq caps
1156 *
1157 * Returned "caps" frequencies should be converted to MHz using
1158 * intel_gpu_freq()
1159 */
1160void gen6_rps_get_freq_caps(struct intel_rps *rps, struct intel_rps_freq_caps *caps)
1161{
1162 struct drm_i915_private *i915 = rps_to_i915(rps);
1163
1164 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
1165 return mtl_get_freq_caps(rps, caps);
1166 else
1167 return __gen6_rps_get_freq_caps(rps, caps);
1168}
1169
1170static void gen6_rps_init(struct intel_rps *rps)
1171{
1172 struct drm_i915_private *i915 = rps_to_i915(rps);
1173 struct intel_rps_freq_caps caps;
1174
1175 gen6_rps_get_freq_caps(rps, &caps);
1176 rps->rp0_freq = caps.rp0_freq;
1177 rps->rp1_freq = caps.rp1_freq;
1178 rps->min_freq = caps.min_freq;
1179
1180 /* hw_max = RP0 until we check for overclocking */
1181 rps->max_freq = rps->rp0_freq;
1182
1183 rps->efficient_freq = rps->rp1_freq;
1184 if (IS_HASWELL(i915) || IS_BROADWELL(i915) ||
1185 IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11) {
1186 u32 ddcc_status = 0;
1187 u32 mult = 1;
1188
1189 if (IS_GEN9_BC(i915) || GRAPHICS_VER(i915) >= 11)
1190 mult = GEN9_FREQ_SCALER;
1191 if (snb_pcode_read(rps_to_gt(rps)->uncore,
1192 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
1193 &ddcc_status, NULL) == 0)
1194 rps->efficient_freq =
1195 clamp_t(u32,
1196 ((ddcc_status >> 8) & 0xff) * mult,
1197 rps->min_freq,
1198 rps->max_freq);
1199 }
1200}
1201
1202static bool rps_reset(struct intel_rps *rps)
1203{
1204 struct drm_i915_private *i915 = rps_to_i915(rps);
1205
1206 /* force a reset */
1207 rps->power.mode = -1;
1208 rps->last_freq = -1;
1209
1210 if (rps_set(rps, rps->min_freq, true)) {
1211 drm_err(&i915->drm, "Failed to reset RPS to initial values\n");
1212 return false;
1213 }
1214
1215 rps->cur_freq = rps->min_freq;
1216 return true;
1217}
1218
1219/* See the Gen9_GT_PM_Programming_Guide doc for the below */
1220static bool gen9_rps_enable(struct intel_rps *rps)
1221{
1222 struct intel_gt *gt = rps_to_gt(rps);
1223 struct intel_uncore *uncore = gt->uncore;
1224
1225 /* Program defaults and thresholds for RPS */
1226 if (GRAPHICS_VER(gt->i915) == 9)
1227 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1228 GEN9_FREQUENCY(rps->rp1_freq));
1229
1230 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa);
1231
1232 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1233
1234 return rps_reset(rps);
1235}
1236
1237static bool gen8_rps_enable(struct intel_rps *rps)
1238{
1239 struct intel_uncore *uncore = rps_to_uncore(rps);
1240
1241 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ,
1242 HSW_FREQUENCY(rps->rp1_freq));
1243
1244 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1245
1246 rps->pm_events = GEN6_PM_RP_UP_THRESHOLD | GEN6_PM_RP_DOWN_THRESHOLD;
1247
1248 return rps_reset(rps);
1249}
1250
1251static bool gen6_rps_enable(struct intel_rps *rps)
1252{
1253 struct intel_uncore *uncore = rps_to_uncore(rps);
1254
1255 /* Power down if completely idle for over 50ms */
1256 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000);
1257 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1258
1259 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1260 GEN6_PM_RP_DOWN_THRESHOLD |
1261 GEN6_PM_RP_DOWN_TIMEOUT);
1262
1263 return rps_reset(rps);
1264}
1265
1266static int chv_rps_max_freq(struct intel_rps *rps)
1267{
1268 struct drm_i915_private *i915 = rps_to_i915(rps);
1269 struct intel_gt *gt = rps_to_gt(rps);
1270 u32 val;
1271
1272 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1273
1274 switch (gt->info.sseu.eu_total) {
1275 case 8:
1276 /* (2 * 4) config */
1277 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT;
1278 break;
1279 case 12:
1280 /* (2 * 6) config */
1281 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT;
1282 break;
1283 case 16:
1284 /* (2 * 8) config */
1285 default:
1286 /* Setting (2 * 8) Min RP0 for any other combination */
1287 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT;
1288 break;
1289 }
1290
1291 return val & FB_GFX_FREQ_FUSE_MASK;
1292}
1293
1294static int chv_rps_rpe_freq(struct intel_rps *rps)
1295{
1296 struct drm_i915_private *i915 = rps_to_i915(rps);
1297 u32 val;
1298
1299 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG);
1300 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT;
1301
1302 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK;
1303}
1304
1305static int chv_rps_guar_freq(struct intel_rps *rps)
1306{
1307 struct drm_i915_private *i915 = rps_to_i915(rps);
1308 u32 val;
1309
1310 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE);
1311
1312 return val & FB_GFX_FREQ_FUSE_MASK;
1313}
1314
1315static u32 chv_rps_min_freq(struct intel_rps *rps)
1316{
1317 struct drm_i915_private *i915 = rps_to_i915(rps);
1318 u32 val;
1319
1320 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE);
1321 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT;
1322
1323 return val & FB_GFX_FREQ_FUSE_MASK;
1324}
1325
1326static bool chv_rps_enable(struct intel_rps *rps)
1327{
1328 struct intel_uncore *uncore = rps_to_uncore(rps);
1329 struct drm_i915_private *i915 = rps_to_i915(rps);
1330 u32 val;
1331
1332 /* 1: Program defaults and thresholds for RPS*/
1333 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1334 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1335 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1336 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1337 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1338
1339 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1340
1341 /* 2: Enable RPS */
1342 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1343 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1344 GEN6_RP_MEDIA_IS_GFX |
1345 GEN6_RP_ENABLE |
1346 GEN6_RP_UP_BUSY_AVG |
1347 GEN6_RP_DOWN_IDLE_AVG);
1348
1349 rps->pm_events = (GEN6_PM_RP_UP_THRESHOLD |
1350 GEN6_PM_RP_DOWN_THRESHOLD |
1351 GEN6_PM_RP_DOWN_TIMEOUT);
1352
1353 /* Setting Fixed Bias */
1354 vlv_punit_get(i915);
1355
1356 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50;
1357 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1358
1359 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1360
1361 vlv_punit_put(i915);
1362
1363 /* RPS code assumes GPLL is used */
1364 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1365 "GPLL not enabled\n");
1366
1367 drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1368 str_yes_no(val & GPLLENABLE));
1369 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1370
1371 return rps_reset(rps);
1372}
1373
1374static int vlv_rps_guar_freq(struct intel_rps *rps)
1375{
1376 struct drm_i915_private *i915 = rps_to_i915(rps);
1377 u32 val, rp1;
1378
1379 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1380
1381 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK;
1382 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT;
1383
1384 return rp1;
1385}
1386
1387static int vlv_rps_max_freq(struct intel_rps *rps)
1388{
1389 struct drm_i915_private *i915 = rps_to_i915(rps);
1390 u32 val, rp0;
1391
1392 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE);
1393
1394 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT;
1395 /* Clamp to max */
1396 rp0 = min_t(u32, rp0, 0xea);
1397
1398 return rp0;
1399}
1400
1401static int vlv_rps_rpe_freq(struct intel_rps *rps)
1402{
1403 struct drm_i915_private *i915 = rps_to_i915(rps);
1404 u32 val, rpe;
1405
1406 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO);
1407 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT;
1408 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI);
1409 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5;
1410
1411 return rpe;
1412}
1413
1414static int vlv_rps_min_freq(struct intel_rps *rps)
1415{
1416 struct drm_i915_private *i915 = rps_to_i915(rps);
1417 u32 val;
1418
1419 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff;
1420 /*
1421 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value
1422 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on
1423 * a BYT-M B0 the above register contains 0xbf. Moreover when setting
1424 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0
1425 * to make sure it matches what Punit accepts.
1426 */
1427 return max_t(u32, val, 0xc0);
1428}
1429
1430static bool vlv_rps_enable(struct intel_rps *rps)
1431{
1432 struct intel_uncore *uncore = rps_to_uncore(rps);
1433 struct drm_i915_private *i915 = rps_to_i915(rps);
1434 u32 val;
1435
1436 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000);
1437 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400);
1438 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000);
1439 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000);
1440 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000);
1441
1442 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10);
1443
1444 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL,
1445 GEN6_RP_MEDIA_TURBO |
1446 GEN6_RP_MEDIA_HW_NORMAL_MODE |
1447 GEN6_RP_MEDIA_IS_GFX |
1448 GEN6_RP_ENABLE |
1449 GEN6_RP_UP_BUSY_AVG |
1450 GEN6_RP_DOWN_IDLE_CONT);
1451
1452 /* WaGsvRC0ResidencyMethod:vlv */
1453 rps->pm_events = GEN6_PM_RP_UP_EI_EXPIRED;
1454
1455 vlv_punit_get(i915);
1456
1457 /* Setting Fixed Bias */
1458 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875;
1459 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val);
1460
1461 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
1462
1463 vlv_punit_put(i915);
1464
1465 /* RPS code assumes GPLL is used */
1466 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0,
1467 "GPLL not enabled\n");
1468
1469 drm_dbg(&i915->drm, "GPLL enabled? %s\n",
1470 str_yes_no(val & GPLLENABLE));
1471 drm_dbg(&i915->drm, "GPU status: 0x%08x\n", val);
1472
1473 return rps_reset(rps);
1474}
1475
1476static unsigned long __ips_gfx_val(struct intel_ips *ips)
1477{
1478 struct intel_rps *rps = container_of(ips, typeof(*rps), ips);
1479 struct intel_uncore *uncore = rps_to_uncore(rps);
1480 unsigned int t, state1, state2;
1481 u32 pxvid, ext_v;
1482 u64 corr, corr2;
1483
1484 lockdep_assert_held(&mchdev_lock);
1485
1486 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq));
1487 pxvid = (pxvid >> 24) & 0x7f;
1488 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid);
1489
1490 state1 = ext_v;
1491
1492 /* Revel in the empirically derived constants */
1493
1494 /* Correction factor in 1/100000 units */
1495 t = ips_mch_val(uncore);
1496 if (t > 80)
1497 corr = t * 2349 + 135940;
1498 else if (t >= 50)
1499 corr = t * 964 + 29317;
1500 else /* < 50 */
1501 corr = t * 301 + 1004;
1502
1503 corr = div_u64(corr * 150142 * state1, 10000) - 78642;
1504 corr2 = div_u64(corr, 100000) * ips->corr;
1505
1506 state2 = div_u64(corr2 * state1, 10000);
1507 state2 /= 100; /* convert to mW */
1508
1509 __gen5_ips_update(ips);
1510
1511 return ips->gfx_power + state2;
1512}
1513
1514static bool has_busy_stats(struct intel_rps *rps)
1515{
1516 struct intel_engine_cs *engine;
1517 enum intel_engine_id id;
1518
1519 for_each_engine(engine, rps_to_gt(rps), id) {
1520 if (!intel_engine_supports_stats(engine))
1521 return false;
1522 }
1523
1524 return true;
1525}
1526
1527void intel_rps_enable(struct intel_rps *rps)
1528{
1529 struct drm_i915_private *i915 = rps_to_i915(rps);
1530 struct intel_uncore *uncore = rps_to_uncore(rps);
1531 bool enabled = false;
1532
1533 if (!HAS_RPS(i915))
1534 return;
1535
1536 if (rps_uses_slpc(rps))
1537 return;
1538
1539 intel_gt_check_clock_frequency(rps_to_gt(rps));
1540
1541 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1542 if (rps->max_freq <= rps->min_freq)
1543 /* leave disabled, no room for dynamic reclocking */;
1544 else if (IS_CHERRYVIEW(i915))
1545 enabled = chv_rps_enable(rps);
1546 else if (IS_VALLEYVIEW(i915))
1547 enabled = vlv_rps_enable(rps);
1548 else if (GRAPHICS_VER(i915) >= 9)
1549 enabled = gen9_rps_enable(rps);
1550 else if (GRAPHICS_VER(i915) >= 8)
1551 enabled = gen8_rps_enable(rps);
1552 else if (GRAPHICS_VER(i915) >= 6)
1553 enabled = gen6_rps_enable(rps);
1554 else if (IS_IRONLAKE_M(i915))
1555 enabled = gen5_rps_enable(rps);
1556 else
1557 MISSING_CASE(GRAPHICS_VER(i915));
1558 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1559 if (!enabled)
1560 return;
1561
1562 GT_TRACE(rps_to_gt(rps),
1563 "min:%x, max:%x, freq:[%d, %d], thresholds:[%u, %u]\n",
1564 rps->min_freq, rps->max_freq,
1565 intel_gpu_freq(rps, rps->min_freq),
1566 intel_gpu_freq(rps, rps->max_freq),
1567 rps->power.up_threshold,
1568 rps->power.down_threshold);
1569
1570 GEM_BUG_ON(rps->max_freq < rps->min_freq);
1571 GEM_BUG_ON(rps->idle_freq > rps->max_freq);
1572
1573 GEM_BUG_ON(rps->efficient_freq < rps->min_freq);
1574 GEM_BUG_ON(rps->efficient_freq > rps->max_freq);
1575
1576 if (has_busy_stats(rps))
1577 intel_rps_set_timer(rps);
1578 else if (GRAPHICS_VER(i915) >= 6 && GRAPHICS_VER(i915) <= 11)
1579 intel_rps_set_interrupts(rps);
1580 else
1581 /* Ironlake currently uses intel_ips.ko */ {}
1582
1583 intel_rps_set_enabled(rps);
1584}
1585
1586static void gen6_rps_disable(struct intel_rps *rps)
1587{
1588 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0);
1589}
1590
1591void intel_rps_disable(struct intel_rps *rps)
1592{
1593 struct drm_i915_private *i915 = rps_to_i915(rps);
1594
1595 if (!intel_rps_is_enabled(rps))
1596 return;
1597
1598 intel_rps_clear_enabled(rps);
1599 intel_rps_clear_interrupts(rps);
1600 intel_rps_clear_timer(rps);
1601
1602 if (GRAPHICS_VER(i915) >= 6)
1603 gen6_rps_disable(rps);
1604 else if (IS_IRONLAKE_M(i915))
1605 gen5_rps_disable(rps);
1606}
1607
1608static int byt_gpu_freq(struct intel_rps *rps, int val)
1609{
1610 /*
1611 * N = val - 0xb7
1612 * Slow = Fast = GPLL ref * N
1613 */
1614 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000);
1615}
1616
1617static int byt_freq_opcode(struct intel_rps *rps, int val)
1618{
1619 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7;
1620}
1621
1622static int chv_gpu_freq(struct intel_rps *rps, int val)
1623{
1624 /*
1625 * N = val / 2
1626 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2
1627 */
1628 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000);
1629}
1630
1631static int chv_freq_opcode(struct intel_rps *rps, int val)
1632{
1633 /* CHV needs even values */
1634 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2;
1635}
1636
1637int intel_gpu_freq(struct intel_rps *rps, int val)
1638{
1639 struct drm_i915_private *i915 = rps_to_i915(rps);
1640
1641 if (GRAPHICS_VER(i915) >= 9)
1642 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER,
1643 GEN9_FREQ_SCALER);
1644 else if (IS_CHERRYVIEW(i915))
1645 return chv_gpu_freq(rps, val);
1646 else if (IS_VALLEYVIEW(i915))
1647 return byt_gpu_freq(rps, val);
1648 else if (GRAPHICS_VER(i915) >= 6)
1649 return val * GT_FREQUENCY_MULTIPLIER;
1650 else
1651 return val;
1652}
1653
1654int intel_freq_opcode(struct intel_rps *rps, int val)
1655{
1656 struct drm_i915_private *i915 = rps_to_i915(rps);
1657
1658 if (GRAPHICS_VER(i915) >= 9)
1659 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER,
1660 GT_FREQUENCY_MULTIPLIER);
1661 else if (IS_CHERRYVIEW(i915))
1662 return chv_freq_opcode(rps, val);
1663 else if (IS_VALLEYVIEW(i915))
1664 return byt_freq_opcode(rps, val);
1665 else if (GRAPHICS_VER(i915) >= 6)
1666 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER);
1667 else
1668 return val;
1669}
1670
1671static void vlv_init_gpll_ref_freq(struct intel_rps *rps)
1672{
1673 struct drm_i915_private *i915 = rps_to_i915(rps);
1674
1675 rps->gpll_ref_freq =
1676 vlv_get_cck_clock(i915, "GPLL ref",
1677 CCK_GPLL_CLOCK_CONTROL,
1678 i915->czclk_freq);
1679
1680 drm_dbg(&i915->drm, "GPLL reference freq: %d kHz\n",
1681 rps->gpll_ref_freq);
1682}
1683
1684static void vlv_rps_init(struct intel_rps *rps)
1685{
1686 struct drm_i915_private *i915 = rps_to_i915(rps);
1687
1688 vlv_iosf_sb_get(i915,
1689 BIT(VLV_IOSF_SB_PUNIT) |
1690 BIT(VLV_IOSF_SB_NC) |
1691 BIT(VLV_IOSF_SB_CCK));
1692
1693 vlv_init_gpll_ref_freq(rps);
1694
1695 rps->max_freq = vlv_rps_max_freq(rps);
1696 rps->rp0_freq = rps->max_freq;
1697 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1698 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1699
1700 rps->efficient_freq = vlv_rps_rpe_freq(rps);
1701 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1702 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1703
1704 rps->rp1_freq = vlv_rps_guar_freq(rps);
1705 drm_dbg(&i915->drm, "RP1(Guar Freq) GPU freq: %d MHz (%u)\n",
1706 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1707
1708 rps->min_freq = vlv_rps_min_freq(rps);
1709 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1710 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1711
1712 vlv_iosf_sb_put(i915,
1713 BIT(VLV_IOSF_SB_PUNIT) |
1714 BIT(VLV_IOSF_SB_NC) |
1715 BIT(VLV_IOSF_SB_CCK));
1716}
1717
1718static void chv_rps_init(struct intel_rps *rps)
1719{
1720 struct drm_i915_private *i915 = rps_to_i915(rps);
1721
1722 vlv_iosf_sb_get(i915,
1723 BIT(VLV_IOSF_SB_PUNIT) |
1724 BIT(VLV_IOSF_SB_NC) |
1725 BIT(VLV_IOSF_SB_CCK));
1726
1727 vlv_init_gpll_ref_freq(rps);
1728
1729 rps->max_freq = chv_rps_max_freq(rps);
1730 rps->rp0_freq = rps->max_freq;
1731 drm_dbg(&i915->drm, "max GPU freq: %d MHz (%u)\n",
1732 intel_gpu_freq(rps, rps->max_freq), rps->max_freq);
1733
1734 rps->efficient_freq = chv_rps_rpe_freq(rps);
1735 drm_dbg(&i915->drm, "RPe GPU freq: %d MHz (%u)\n",
1736 intel_gpu_freq(rps, rps->efficient_freq), rps->efficient_freq);
1737
1738 rps->rp1_freq = chv_rps_guar_freq(rps);
1739 drm_dbg(&i915->drm, "RP1(Guar) GPU freq: %d MHz (%u)\n",
1740 intel_gpu_freq(rps, rps->rp1_freq), rps->rp1_freq);
1741
1742 rps->min_freq = chv_rps_min_freq(rps);
1743 drm_dbg(&i915->drm, "min GPU freq: %d MHz (%u)\n",
1744 intel_gpu_freq(rps, rps->min_freq), rps->min_freq);
1745
1746 vlv_iosf_sb_put(i915,
1747 BIT(VLV_IOSF_SB_PUNIT) |
1748 BIT(VLV_IOSF_SB_NC) |
1749 BIT(VLV_IOSF_SB_CCK));
1750
1751 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq |
1752 rps->rp1_freq | rps->min_freq) & 1,
1753 "Odd GPU freq values\n");
1754}
1755
1756static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei)
1757{
1758 ei->ktime = ktime_get_raw();
1759 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT);
1760 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT);
1761}
1762
1763static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir)
1764{
1765 struct intel_uncore *uncore = rps_to_uncore(rps);
1766 const struct intel_rps_ei *prev = &rps->ei;
1767 struct intel_rps_ei now;
1768 u32 events = 0;
1769
1770 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0)
1771 return 0;
1772
1773 vlv_c0_read(uncore, &now);
1774
1775 if (prev->ktime) {
1776 u64 time, c0;
1777 u32 render, media;
1778
1779 time = ktime_us_delta(now.ktime, prev->ktime);
1780
1781 time *= rps_to_i915(rps)->czclk_freq;
1782
1783 /* Workload can be split between render + media,
1784 * e.g. SwapBuffers being blitted in X after being rendered in
1785 * mesa. To account for this we need to combine both engines
1786 * into our activity counter.
1787 */
1788 render = now.render_c0 - prev->render_c0;
1789 media = now.media_c0 - prev->media_c0;
1790 c0 = max(render, media);
1791 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */
1792
1793 if (c0 > time * rps->power.up_threshold)
1794 events = GEN6_PM_RP_UP_THRESHOLD;
1795 else if (c0 < time * rps->power.down_threshold)
1796 events = GEN6_PM_RP_DOWN_THRESHOLD;
1797 }
1798
1799 rps->ei = now;
1800 return events;
1801}
1802
1803static void rps_work(struct work_struct *work)
1804{
1805 struct intel_rps *rps = container_of(work, typeof(*rps), work);
1806 struct intel_gt *gt = rps_to_gt(rps);
1807 struct drm_i915_private *i915 = rps_to_i915(rps);
1808 bool client_boost = false;
1809 int new_freq, adj, min, max;
1810 u32 pm_iir = 0;
1811
1812 spin_lock_irq(gt->irq_lock);
1813 pm_iir = fetch_and_zero(&rps->pm_iir) & rps->pm_events;
1814 client_boost = atomic_read(&rps->num_waiters);
1815 spin_unlock_irq(gt->irq_lock);
1816
1817 /* Make sure we didn't queue anything we're not going to process. */
1818 if (!pm_iir && !client_boost)
1819 goto out;
1820
1821 mutex_lock(&rps->lock);
1822 if (!intel_rps_is_active(rps)) {
1823 mutex_unlock(&rps->lock);
1824 return;
1825 }
1826
1827 pm_iir |= vlv_wa_c0_ei(rps, pm_iir);
1828
1829 adj = rps->last_adj;
1830 new_freq = rps->cur_freq;
1831 min = rps->min_freq_softlimit;
1832 max = rps->max_freq_softlimit;
1833 if (client_boost)
1834 max = rps->max_freq;
1835
1836 GT_TRACE(gt,
1837 "pm_iir:%x, client_boost:%s, last:%d, cur:%x, min:%x, max:%x\n",
1838 pm_iir, str_yes_no(client_boost),
1839 adj, new_freq, min, max);
1840
1841 if (client_boost && new_freq < rps->boost_freq) {
1842 new_freq = rps->boost_freq;
1843 adj = 0;
1844 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
1845 if (adj > 0)
1846 adj *= 2;
1847 else /* CHV needs even encode values */
1848 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1;
1849
1850 if (new_freq >= rps->max_freq_softlimit)
1851 adj = 0;
1852 } else if (client_boost) {
1853 adj = 0;
1854 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
1855 if (rps->cur_freq > rps->efficient_freq)
1856 new_freq = rps->efficient_freq;
1857 else if (rps->cur_freq > rps->min_freq_softlimit)
1858 new_freq = rps->min_freq_softlimit;
1859 adj = 0;
1860 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1861 if (adj < 0)
1862 adj *= 2;
1863 else /* CHV needs even encode values */
1864 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1;
1865
1866 if (new_freq <= rps->min_freq_softlimit)
1867 adj = 0;
1868 } else { /* unknown event */
1869 adj = 0;
1870 }
1871
1872 /*
1873 * sysfs frequency limits may have snuck in while
1874 * servicing the interrupt
1875 */
1876 new_freq += adj;
1877 new_freq = clamp_t(int, new_freq, min, max);
1878
1879 if (intel_rps_set(rps, new_freq)) {
1880 drm_dbg(&i915->drm, "Failed to set new GPU frequency\n");
1881 adj = 0;
1882 }
1883 rps->last_adj = adj;
1884
1885 mutex_unlock(&rps->lock);
1886
1887out:
1888 spin_lock_irq(gt->irq_lock);
1889 gen6_gt_pm_unmask_irq(gt, rps->pm_events);
1890 spin_unlock_irq(gt->irq_lock);
1891}
1892
1893void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1894{
1895 struct intel_gt *gt = rps_to_gt(rps);
1896 const u32 events = rps->pm_events & pm_iir;
1897
1898 lockdep_assert_held(gt->irq_lock);
1899
1900 if (unlikely(!events))
1901 return;
1902
1903 GT_TRACE(gt, "irq events:%x\n", events);
1904
1905 gen6_gt_pm_mask_irq(gt, events);
1906
1907 rps->pm_iir |= events;
1908 queue_work(gt->i915->unordered_wq, &rps->work);
1909}
1910
1911void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir)
1912{
1913 struct intel_gt *gt = rps_to_gt(rps);
1914 u32 events;
1915
1916 events = pm_iir & rps->pm_events;
1917 if (events) {
1918 spin_lock(gt->irq_lock);
1919
1920 GT_TRACE(gt, "irq events:%x\n", events);
1921
1922 gen6_gt_pm_mask_irq(gt, events);
1923 rps->pm_iir |= events;
1924
1925 queue_work(gt->i915->unordered_wq, &rps->work);
1926 spin_unlock(gt->irq_lock);
1927 }
1928
1929 if (GRAPHICS_VER(gt->i915) >= 8)
1930 return;
1931
1932 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1933 intel_engine_cs_irq(gt->engine[VECS0], pm_iir >> 10);
1934
1935 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT)
1936 drm_dbg(&rps_to_i915(rps)->drm,
1937 "Command parser error, pm_iir 0x%08x\n", pm_iir);
1938}
1939
1940void gen5_rps_irq_handler(struct intel_rps *rps)
1941{
1942 struct intel_uncore *uncore = rps_to_uncore(rps);
1943 u32 busy_up, busy_down, max_avg, min_avg;
1944 u8 new_freq;
1945
1946 spin_lock(&mchdev_lock);
1947
1948 intel_uncore_write16(uncore,
1949 MEMINTRSTS,
1950 intel_uncore_read(uncore, MEMINTRSTS));
1951
1952 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG);
1953 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG);
1954 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG);
1955 max_avg = intel_uncore_read(uncore, RCBMAXAVG);
1956 min_avg = intel_uncore_read(uncore, RCBMINAVG);
1957
1958 /* Handle RCS change request from hw */
1959 new_freq = rps->cur_freq;
1960 if (busy_up > max_avg)
1961 new_freq++;
1962 else if (busy_down < min_avg)
1963 new_freq--;
1964 new_freq = clamp(new_freq,
1965 rps->min_freq_softlimit,
1966 rps->max_freq_softlimit);
1967
1968 if (new_freq != rps->cur_freq && !__gen5_rps_set(rps, new_freq))
1969 rps->cur_freq = new_freq;
1970
1971 spin_unlock(&mchdev_lock);
1972}
1973
1974void intel_rps_init_early(struct intel_rps *rps)
1975{
1976 mutex_init(&rps->lock);
1977 mutex_init(&rps->power.mutex);
1978
1979 INIT_WORK(&rps->work, rps_work);
1980 timer_setup(&rps->timer, rps_timer, 0);
1981
1982 atomic_set(&rps->num_waiters, 0);
1983}
1984
1985void intel_rps_init(struct intel_rps *rps)
1986{
1987 struct drm_i915_private *i915 = rps_to_i915(rps);
1988
1989 if (rps_uses_slpc(rps))
1990 return;
1991
1992 if (IS_CHERRYVIEW(i915))
1993 chv_rps_init(rps);
1994 else if (IS_VALLEYVIEW(i915))
1995 vlv_rps_init(rps);
1996 else if (GRAPHICS_VER(i915) >= 6)
1997 gen6_rps_init(rps);
1998 else if (IS_IRONLAKE_M(i915))
1999 gen5_rps_init(rps);
2000
2001 /* Derive initial user preferences/limits from the hardware limits */
2002 rps->max_freq_softlimit = rps->max_freq;
2003 rps_to_gt(rps)->defaults.max_freq = rps->max_freq_softlimit;
2004 rps->min_freq_softlimit = rps->min_freq;
2005 rps_to_gt(rps)->defaults.min_freq = rps->min_freq_softlimit;
2006
2007 /* After setting max-softlimit, find the overclock max freq */
2008 if (GRAPHICS_VER(i915) == 6 || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) {
2009 u32 params = 0;
2010
2011 snb_pcode_read(rps_to_gt(rps)->uncore, GEN6_READ_OC_PARAMS, ¶ms, NULL);
2012 if (params & BIT(31)) { /* OC supported */
2013 drm_dbg(&i915->drm,
2014 "Overclocking supported, max: %dMHz, overclock: %dMHz\n",
2015 (rps->max_freq & 0xff) * 50,
2016 (params & 0xff) * 50);
2017 rps->max_freq = params & 0xff;
2018 }
2019 }
2020
2021 /* Set default thresholds in % */
2022 rps->power.up_threshold = 95;
2023 rps_to_gt(rps)->defaults.rps_up_threshold = rps->power.up_threshold;
2024 rps->power.down_threshold = 85;
2025 rps_to_gt(rps)->defaults.rps_down_threshold = rps->power.down_threshold;
2026
2027 /* Finally allow us to boost to max by default */
2028 rps->boost_freq = rps->max_freq;
2029 rps->idle_freq = rps->min_freq;
2030
2031 /* Start in the middle, from here we will autotune based on workload */
2032 rps->cur_freq = rps->efficient_freq;
2033
2034 rps->pm_intrmsk_mbz = 0;
2035
2036 /*
2037 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer
2038 * if GEN6_PM_UP_EI_EXPIRED is masked.
2039 *
2040 * TODO: verify if this can be reproduced on VLV,CHV.
2041 */
2042 if (GRAPHICS_VER(i915) <= 7)
2043 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED;
2044
2045 if (GRAPHICS_VER(i915) >= 8 && GRAPHICS_VER(i915) < 11)
2046 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC;
2047
2048 /* GuC needs ARAT expired interrupt unmasked */
2049 if (intel_uc_uses_guc_submission(&rps_to_gt(rps)->uc))
2050 rps->pm_intrmsk_mbz |= ARAT_EXPIRED_INTRMSK;
2051}
2052
2053void intel_rps_sanitize(struct intel_rps *rps)
2054{
2055 if (rps_uses_slpc(rps))
2056 return;
2057
2058 if (GRAPHICS_VER(rps_to_i915(rps)) >= 6)
2059 rps_disable_interrupts(rps);
2060}
2061
2062u32 intel_rps_read_rpstat(struct intel_rps *rps)
2063{
2064 struct drm_i915_private *i915 = rps_to_i915(rps);
2065 i915_reg_t rpstat;
2066
2067 rpstat = (GRAPHICS_VER(i915) >= 12) ? GEN12_RPSTAT1 : GEN6_RPSTAT1;
2068
2069 return intel_uncore_read(rps_to_gt(rps)->uncore, rpstat);
2070}
2071
2072static u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat)
2073{
2074 struct drm_i915_private *i915 = rps_to_i915(rps);
2075 u32 cagf;
2076
2077 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
2078 cagf = REG_FIELD_GET(MTL_CAGF_MASK, rpstat);
2079 else if (GRAPHICS_VER(i915) >= 12)
2080 cagf = REG_FIELD_GET(GEN12_CAGF_MASK, rpstat);
2081 else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915))
2082 cagf = REG_FIELD_GET(RPE_MASK, rpstat);
2083 else if (GRAPHICS_VER(i915) >= 9)
2084 cagf = REG_FIELD_GET(GEN9_CAGF_MASK, rpstat);
2085 else if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2086 cagf = REG_FIELD_GET(HSW_CAGF_MASK, rpstat);
2087 else if (GRAPHICS_VER(i915) >= 6)
2088 cagf = REG_FIELD_GET(GEN6_CAGF_MASK, rpstat);
2089 else
2090 cagf = gen5_invert_freq(rps, REG_FIELD_GET(MEMSTAT_PSTATE_MASK, rpstat));
2091
2092 return cagf;
2093}
2094
2095static u32 __read_cagf(struct intel_rps *rps, bool take_fw)
2096{
2097 struct drm_i915_private *i915 = rps_to_i915(rps);
2098 struct intel_uncore *uncore = rps_to_uncore(rps);
2099 i915_reg_t r = INVALID_MMIO_REG;
2100 u32 freq;
2101
2102 /*
2103 * For Gen12+ reading freq from HW does not need a forcewake and
2104 * registers will return 0 freq when GT is in RC6
2105 */
2106 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) {
2107 r = MTL_MIRROR_TARGET_WP1;
2108 } else if (GRAPHICS_VER(i915) >= 12) {
2109 r = GEN12_RPSTAT1;
2110 } else if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
2111 vlv_punit_get(i915);
2112 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS);
2113 vlv_punit_put(i915);
2114 } else if (GRAPHICS_VER(i915) >= 6) {
2115 r = GEN6_RPSTAT1;
2116 } else {
2117 r = MEMSTAT_ILK;
2118 }
2119
2120 if (i915_mmio_reg_valid(r))
2121 freq = take_fw ? intel_uncore_read(uncore, r) : intel_uncore_read_fw(uncore, r);
2122
2123 return intel_rps_get_cagf(rps, freq);
2124}
2125
2126static u32 read_cagf(struct intel_rps *rps)
2127{
2128 return __read_cagf(rps, true);
2129}
2130
2131u32 intel_rps_read_actual_frequency(struct intel_rps *rps)
2132{
2133 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2134 intel_wakeref_t wakeref;
2135 u32 freq = 0;
2136
2137 with_intel_runtime_pm_if_in_use(rpm, wakeref)
2138 freq = intel_gpu_freq(rps, read_cagf(rps));
2139
2140 return freq;
2141}
2142
2143u32 intel_rps_read_actual_frequency_fw(struct intel_rps *rps)
2144{
2145 return intel_gpu_freq(rps, __read_cagf(rps, false));
2146}
2147
2148static u32 intel_rps_read_punit_req(struct intel_rps *rps)
2149{
2150 struct intel_uncore *uncore = rps_to_uncore(rps);
2151 struct intel_runtime_pm *rpm = rps_to_uncore(rps)->rpm;
2152 intel_wakeref_t wakeref;
2153 u32 freq = 0;
2154
2155 with_intel_runtime_pm_if_in_use(rpm, wakeref)
2156 freq = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2157
2158 return freq;
2159}
2160
2161static u32 intel_rps_get_req(u32 pureq)
2162{
2163 u32 req = pureq >> GEN9_SW_REQ_UNSLICE_RATIO_SHIFT;
2164
2165 return req;
2166}
2167
2168u32 intel_rps_read_punit_req_frequency(struct intel_rps *rps)
2169{
2170 u32 freq = intel_rps_get_req(intel_rps_read_punit_req(rps));
2171
2172 return intel_gpu_freq(rps, freq);
2173}
2174
2175u32 intel_rps_get_requested_frequency(struct intel_rps *rps)
2176{
2177 if (rps_uses_slpc(rps))
2178 return intel_rps_read_punit_req_frequency(rps);
2179 else
2180 return intel_gpu_freq(rps, rps->cur_freq);
2181}
2182
2183u32 intel_rps_get_max_frequency(struct intel_rps *rps)
2184{
2185 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2186
2187 if (rps_uses_slpc(rps))
2188 return slpc->max_freq_softlimit;
2189 else
2190 return intel_gpu_freq(rps, rps->max_freq_softlimit);
2191}
2192
2193/**
2194 * intel_rps_get_max_raw_freq - returns the max frequency in some raw format.
2195 * @rps: the intel_rps structure
2196 *
2197 * Returns the max frequency in a raw format. In newer platforms raw is in
2198 * units of 50 MHz.
2199 */
2200u32 intel_rps_get_max_raw_freq(struct intel_rps *rps)
2201{
2202 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2203 u32 freq;
2204
2205 if (rps_uses_slpc(rps)) {
2206 return DIV_ROUND_CLOSEST(slpc->rp0_freq,
2207 GT_FREQUENCY_MULTIPLIER);
2208 } else {
2209 freq = rps->max_freq;
2210 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2211 /* Convert GT frequency to 50 MHz units */
2212 freq /= GEN9_FREQ_SCALER;
2213 }
2214 return freq;
2215 }
2216}
2217
2218u32 intel_rps_get_rp0_frequency(struct intel_rps *rps)
2219{
2220 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2221
2222 if (rps_uses_slpc(rps))
2223 return slpc->rp0_freq;
2224 else
2225 return intel_gpu_freq(rps, rps->rp0_freq);
2226}
2227
2228u32 intel_rps_get_rp1_frequency(struct intel_rps *rps)
2229{
2230 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2231
2232 if (rps_uses_slpc(rps))
2233 return slpc->rp1_freq;
2234 else
2235 return intel_gpu_freq(rps, rps->rp1_freq);
2236}
2237
2238u32 intel_rps_get_rpn_frequency(struct intel_rps *rps)
2239{
2240 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2241
2242 if (rps_uses_slpc(rps))
2243 return slpc->min_freq;
2244 else
2245 return intel_gpu_freq(rps, rps->min_freq);
2246}
2247
2248static void rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2249{
2250 struct intel_gt *gt = rps_to_gt(rps);
2251 struct drm_i915_private *i915 = gt->i915;
2252 struct intel_uncore *uncore = gt->uncore;
2253 struct intel_rps_freq_caps caps;
2254 u32 rp_state_limits;
2255 u32 gt_perf_status;
2256 u32 rpmodectl, rpinclimit, rpdeclimit;
2257 u32 rpstat, cagf, reqf;
2258 u32 rpcurupei, rpcurup, rpprevup;
2259 u32 rpcurdownei, rpcurdown, rpprevdown;
2260 u32 rpupei, rpupt, rpdownei, rpdownt;
2261 u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
2262
2263 rp_state_limits = intel_uncore_read(uncore, GEN6_RP_STATE_LIMITS);
2264 gen6_rps_get_freq_caps(rps, &caps);
2265 if (IS_GEN9_LP(i915))
2266 gt_perf_status = intel_uncore_read(uncore, BXT_GT_PERF_STATUS);
2267 else
2268 gt_perf_status = intel_uncore_read(uncore, GEN6_GT_PERF_STATUS);
2269
2270 /* RPSTAT1 is in the GT power well */
2271 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
2272
2273 reqf = intel_uncore_read(uncore, GEN6_RPNSWREQ);
2274 if (GRAPHICS_VER(i915) >= 9) {
2275 reqf >>= 23;
2276 } else {
2277 reqf &= ~GEN6_TURBO_DISABLE;
2278 if (IS_HASWELL(i915) || IS_BROADWELL(i915))
2279 reqf >>= 24;
2280 else
2281 reqf >>= 25;
2282 }
2283 reqf = intel_gpu_freq(rps, reqf);
2284
2285 rpmodectl = intel_uncore_read(uncore, GEN6_RP_CONTROL);
2286 rpinclimit = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2287 rpdeclimit = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2288
2289 rpstat = intel_rps_read_rpstat(rps);
2290 rpcurupei = intel_uncore_read(uncore, GEN6_RP_CUR_UP_EI) & GEN6_CURICONT_MASK;
2291 rpcurup = intel_uncore_read(uncore, GEN6_RP_CUR_UP) & GEN6_CURBSYTAVG_MASK;
2292 rpprevup = intel_uncore_read(uncore, GEN6_RP_PREV_UP) & GEN6_CURBSYTAVG_MASK;
2293 rpcurdownei = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN_EI) & GEN6_CURIAVG_MASK;
2294 rpcurdown = intel_uncore_read(uncore, GEN6_RP_CUR_DOWN) & GEN6_CURBSYTAVG_MASK;
2295 rpprevdown = intel_uncore_read(uncore, GEN6_RP_PREV_DOWN) & GEN6_CURBSYTAVG_MASK;
2296
2297 rpupei = intel_uncore_read(uncore, GEN6_RP_UP_EI);
2298 rpupt = intel_uncore_read(uncore, GEN6_RP_UP_THRESHOLD);
2299
2300 rpdownei = intel_uncore_read(uncore, GEN6_RP_DOWN_EI);
2301 rpdownt = intel_uncore_read(uncore, GEN6_RP_DOWN_THRESHOLD);
2302
2303 cagf = intel_rps_read_actual_frequency(rps);
2304
2305 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
2306
2307 if (GRAPHICS_VER(i915) >= 11) {
2308 pm_ier = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_ENABLE);
2309 pm_imr = intel_uncore_read(uncore, GEN11_GPM_WGBOXPERF_INTR_MASK);
2310 /*
2311 * The equivalent to the PM ISR & IIR cannot be read
2312 * without affecting the current state of the system
2313 */
2314 pm_isr = 0;
2315 pm_iir = 0;
2316 } else if (GRAPHICS_VER(i915) >= 8) {
2317 pm_ier = intel_uncore_read(uncore, GEN8_GT_IER(2));
2318 pm_imr = intel_uncore_read(uncore, GEN8_GT_IMR(2));
2319 pm_isr = intel_uncore_read(uncore, GEN8_GT_ISR(2));
2320 pm_iir = intel_uncore_read(uncore, GEN8_GT_IIR(2));
2321 } else {
2322 pm_ier = intel_uncore_read(uncore, GEN6_PMIER);
2323 pm_imr = intel_uncore_read(uncore, GEN6_PMIMR);
2324 pm_isr = intel_uncore_read(uncore, GEN6_PMISR);
2325 pm_iir = intel_uncore_read(uncore, GEN6_PMIIR);
2326 }
2327 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2328
2329 drm_printf(p, "Video Turbo Mode: %s\n",
2330 str_yes_no(rpmodectl & GEN6_RP_MEDIA_TURBO));
2331 drm_printf(p, "HW control enabled: %s\n",
2332 str_yes_no(rpmodectl & GEN6_RP_ENABLE));
2333 drm_printf(p, "SW control enabled: %s\n",
2334 str_yes_no((rpmodectl & GEN6_RP_MEDIA_MODE_MASK) == GEN6_RP_MEDIA_SW_MODE));
2335
2336 drm_printf(p, "PM IER=0x%08x IMR=0x%08x, MASK=0x%08x\n",
2337 pm_ier, pm_imr, pm_mask);
2338 if (GRAPHICS_VER(i915) <= 10)
2339 drm_printf(p, "PM ISR=0x%08x IIR=0x%08x\n",
2340 pm_isr, pm_iir);
2341 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2342 rps->pm_intrmsk_mbz);
2343 drm_printf(p, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
2344 drm_printf(p, "Render p-state ratio: %d\n",
2345 (gt_perf_status & (GRAPHICS_VER(i915) >= 9 ? 0x1ff00 : 0xff00)) >> 8);
2346 drm_printf(p, "Render p-state VID: %d\n",
2347 gt_perf_status & 0xff);
2348 drm_printf(p, "Render p-state limit: %d\n",
2349 rp_state_limits & 0xff);
2350 drm_printf(p, "RPSTAT1: 0x%08x\n", rpstat);
2351 drm_printf(p, "RPMODECTL: 0x%08x\n", rpmodectl);
2352 drm_printf(p, "RPINCLIMIT: 0x%08x\n", rpinclimit);
2353 drm_printf(p, "RPDECLIMIT: 0x%08x\n", rpdeclimit);
2354 drm_printf(p, "RPNSWREQ: %dMHz\n", reqf);
2355 drm_printf(p, "CAGF: %dMHz\n", cagf);
2356 drm_printf(p, "RP CUR UP EI: %d (%lldns)\n",
2357 rpcurupei,
2358 intel_gt_pm_interval_to_ns(gt, rpcurupei));
2359 drm_printf(p, "RP CUR UP: %d (%lldns)\n",
2360 rpcurup, intel_gt_pm_interval_to_ns(gt, rpcurup));
2361 drm_printf(p, "RP PREV UP: %d (%lldns)\n",
2362 rpprevup, intel_gt_pm_interval_to_ns(gt, rpprevup));
2363 drm_printf(p, "Up threshold: %d%%\n",
2364 rps->power.up_threshold);
2365 drm_printf(p, "RP UP EI: %d (%lldns)\n",
2366 rpupei, intel_gt_pm_interval_to_ns(gt, rpupei));
2367 drm_printf(p, "RP UP THRESHOLD: %d (%lldns)\n",
2368 rpupt, intel_gt_pm_interval_to_ns(gt, rpupt));
2369
2370 drm_printf(p, "RP CUR DOWN EI: %d (%lldns)\n",
2371 rpcurdownei,
2372 intel_gt_pm_interval_to_ns(gt, rpcurdownei));
2373 drm_printf(p, "RP CUR DOWN: %d (%lldns)\n",
2374 rpcurdown,
2375 intel_gt_pm_interval_to_ns(gt, rpcurdown));
2376 drm_printf(p, "RP PREV DOWN: %d (%lldns)\n",
2377 rpprevdown,
2378 intel_gt_pm_interval_to_ns(gt, rpprevdown));
2379 drm_printf(p, "Down threshold: %d%%\n",
2380 rps->power.down_threshold);
2381 drm_printf(p, "RP DOWN EI: %d (%lldns)\n",
2382 rpdownei, intel_gt_pm_interval_to_ns(gt, rpdownei));
2383 drm_printf(p, "RP DOWN THRESHOLD: %d (%lldns)\n",
2384 rpdownt, intel_gt_pm_interval_to_ns(gt, rpdownt));
2385
2386 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2387 intel_gpu_freq(rps, caps.min_freq));
2388 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2389 intel_gpu_freq(rps, caps.rp1_freq));
2390 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2391 intel_gpu_freq(rps, caps.rp0_freq));
2392 drm_printf(p, "Max overclocked frequency: %dMHz\n",
2393 intel_gpu_freq(rps, rps->max_freq));
2394
2395 drm_printf(p, "Current freq: %d MHz\n",
2396 intel_gpu_freq(rps, rps->cur_freq));
2397 drm_printf(p, "Actual freq: %d MHz\n", cagf);
2398 drm_printf(p, "Idle freq: %d MHz\n",
2399 intel_gpu_freq(rps, rps->idle_freq));
2400 drm_printf(p, "Min freq: %d MHz\n",
2401 intel_gpu_freq(rps, rps->min_freq));
2402 drm_printf(p, "Boost freq: %d MHz\n",
2403 intel_gpu_freq(rps, rps->boost_freq));
2404 drm_printf(p, "Max freq: %d MHz\n",
2405 intel_gpu_freq(rps, rps->max_freq));
2406 drm_printf(p,
2407 "efficient (RPe) frequency: %d MHz\n",
2408 intel_gpu_freq(rps, rps->efficient_freq));
2409}
2410
2411static void slpc_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2412{
2413 struct intel_gt *gt = rps_to_gt(rps);
2414 struct intel_uncore *uncore = gt->uncore;
2415 struct intel_rps_freq_caps caps;
2416 u32 pm_mask;
2417
2418 gen6_rps_get_freq_caps(rps, &caps);
2419 pm_mask = intel_uncore_read(uncore, GEN6_PMINTRMSK);
2420
2421 drm_printf(p, "PM MASK=0x%08x\n", pm_mask);
2422 drm_printf(p, "pm_intrmsk_mbz: 0x%08x\n",
2423 rps->pm_intrmsk_mbz);
2424 drm_printf(p, "RPSTAT1: 0x%08x\n", intel_rps_read_rpstat(rps));
2425 drm_printf(p, "RPNSWREQ: %dMHz\n", intel_rps_get_requested_frequency(rps));
2426 drm_printf(p, "Lowest (RPN) frequency: %dMHz\n",
2427 intel_gpu_freq(rps, caps.min_freq));
2428 drm_printf(p, "Nominal (RP1) frequency: %dMHz\n",
2429 intel_gpu_freq(rps, caps.rp1_freq));
2430 drm_printf(p, "Max non-overclocked (RP0) frequency: %dMHz\n",
2431 intel_gpu_freq(rps, caps.rp0_freq));
2432 drm_printf(p, "Current freq: %d MHz\n",
2433 intel_rps_get_requested_frequency(rps));
2434 drm_printf(p, "Actual freq: %d MHz\n",
2435 intel_rps_read_actual_frequency(rps));
2436 drm_printf(p, "Min freq: %d MHz\n",
2437 intel_rps_get_min_frequency(rps));
2438 drm_printf(p, "Boost freq: %d MHz\n",
2439 intel_rps_get_boost_frequency(rps));
2440 drm_printf(p, "Max freq: %d MHz\n",
2441 intel_rps_get_max_frequency(rps));
2442 drm_printf(p,
2443 "efficient (RPe) frequency: %d MHz\n",
2444 intel_gpu_freq(rps, caps.rp1_freq));
2445}
2446
2447void gen6_rps_frequency_dump(struct intel_rps *rps, struct drm_printer *p)
2448{
2449 if (rps_uses_slpc(rps))
2450 return slpc_frequency_dump(rps, p);
2451 else
2452 return rps_frequency_dump(rps, p);
2453}
2454
2455static int set_max_freq(struct intel_rps *rps, u32 val)
2456{
2457 struct drm_i915_private *i915 = rps_to_i915(rps);
2458 int ret = 0;
2459
2460 mutex_lock(&rps->lock);
2461
2462 val = intel_freq_opcode(rps, val);
2463 if (val < rps->min_freq ||
2464 val > rps->max_freq ||
2465 val < rps->min_freq_softlimit) {
2466 ret = -EINVAL;
2467 goto unlock;
2468 }
2469
2470 if (val > rps->rp0_freq)
2471 drm_dbg(&i915->drm, "User requested overclocking to %d\n",
2472 intel_gpu_freq(rps, val));
2473
2474 rps->max_freq_softlimit = val;
2475
2476 val = clamp_t(int, rps->cur_freq,
2477 rps->min_freq_softlimit,
2478 rps->max_freq_softlimit);
2479
2480 /*
2481 * We still need *_set_rps to process the new max_delay and
2482 * update the interrupt limits and PMINTRMSK even though
2483 * frequency request may be unchanged.
2484 */
2485 intel_rps_set(rps, val);
2486
2487unlock:
2488 mutex_unlock(&rps->lock);
2489
2490 return ret;
2491}
2492
2493int intel_rps_set_max_frequency(struct intel_rps *rps, u32 val)
2494{
2495 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2496
2497 if (rps_uses_slpc(rps))
2498 return intel_guc_slpc_set_max_freq(slpc, val);
2499 else
2500 return set_max_freq(rps, val);
2501}
2502
2503u32 intel_rps_get_min_frequency(struct intel_rps *rps)
2504{
2505 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2506
2507 if (rps_uses_slpc(rps))
2508 return slpc->min_freq_softlimit;
2509 else
2510 return intel_gpu_freq(rps, rps->min_freq_softlimit);
2511}
2512
2513/**
2514 * intel_rps_get_min_raw_freq - returns the min frequency in some raw format.
2515 * @rps: the intel_rps structure
2516 *
2517 * Returns the min frequency in a raw format. In newer platforms raw is in
2518 * units of 50 MHz.
2519 */
2520u32 intel_rps_get_min_raw_freq(struct intel_rps *rps)
2521{
2522 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2523 u32 freq;
2524
2525 if (rps_uses_slpc(rps)) {
2526 return DIV_ROUND_CLOSEST(slpc->min_freq,
2527 GT_FREQUENCY_MULTIPLIER);
2528 } else {
2529 freq = rps->min_freq;
2530 if (GRAPHICS_VER(rps_to_i915(rps)) >= 9) {
2531 /* Convert GT frequency to 50 MHz units */
2532 freq /= GEN9_FREQ_SCALER;
2533 }
2534 return freq;
2535 }
2536}
2537
2538static int set_min_freq(struct intel_rps *rps, u32 val)
2539{
2540 int ret = 0;
2541
2542 mutex_lock(&rps->lock);
2543
2544 val = intel_freq_opcode(rps, val);
2545 if (val < rps->min_freq ||
2546 val > rps->max_freq ||
2547 val > rps->max_freq_softlimit) {
2548 ret = -EINVAL;
2549 goto unlock;
2550 }
2551
2552 rps->min_freq_softlimit = val;
2553
2554 val = clamp_t(int, rps->cur_freq,
2555 rps->min_freq_softlimit,
2556 rps->max_freq_softlimit);
2557
2558 /*
2559 * We still need *_set_rps to process the new min_delay and
2560 * update the interrupt limits and PMINTRMSK even though
2561 * frequency request may be unchanged.
2562 */
2563 intel_rps_set(rps, val);
2564
2565unlock:
2566 mutex_unlock(&rps->lock);
2567
2568 return ret;
2569}
2570
2571int intel_rps_set_min_frequency(struct intel_rps *rps, u32 val)
2572{
2573 struct intel_guc_slpc *slpc = rps_to_slpc(rps);
2574
2575 if (rps_uses_slpc(rps))
2576 return intel_guc_slpc_set_min_freq(slpc, val);
2577 else
2578 return set_min_freq(rps, val);
2579}
2580
2581u8 intel_rps_get_up_threshold(struct intel_rps *rps)
2582{
2583 return rps->power.up_threshold;
2584}
2585
2586static int rps_set_threshold(struct intel_rps *rps, u8 *threshold, u8 val)
2587{
2588 int ret;
2589
2590 if (val > 100)
2591 return -EINVAL;
2592
2593 ret = mutex_lock_interruptible(&rps->lock);
2594 if (ret)
2595 return ret;
2596
2597 if (*threshold == val)
2598 goto out_unlock;
2599
2600 *threshold = val;
2601
2602 /* Force reset. */
2603 rps->last_freq = -1;
2604 mutex_lock(&rps->power.mutex);
2605 rps->power.mode = -1;
2606 mutex_unlock(&rps->power.mutex);
2607
2608 intel_rps_set(rps, clamp(rps->cur_freq,
2609 rps->min_freq_softlimit,
2610 rps->max_freq_softlimit));
2611
2612out_unlock:
2613 mutex_unlock(&rps->lock);
2614
2615 return ret;
2616}
2617
2618int intel_rps_set_up_threshold(struct intel_rps *rps, u8 threshold)
2619{
2620 return rps_set_threshold(rps, &rps->power.up_threshold, threshold);
2621}
2622
2623u8 intel_rps_get_down_threshold(struct intel_rps *rps)
2624{
2625 return rps->power.down_threshold;
2626}
2627
2628int intel_rps_set_down_threshold(struct intel_rps *rps, u8 threshold)
2629{
2630 return rps_set_threshold(rps, &rps->power.down_threshold, threshold);
2631}
2632
2633static void intel_rps_set_manual(struct intel_rps *rps, bool enable)
2634{
2635 struct intel_uncore *uncore = rps_to_uncore(rps);
2636 u32 state = enable ? GEN9_RPSWCTL_ENABLE : GEN9_RPSWCTL_DISABLE;
2637
2638 /* Allow punit to process software requests */
2639 intel_uncore_write(uncore, GEN6_RP_CONTROL, state);
2640}
2641
2642void intel_rps_raise_unslice(struct intel_rps *rps)
2643{
2644 struct intel_uncore *uncore = rps_to_uncore(rps);
2645
2646 mutex_lock(&rps->lock);
2647
2648 if (rps_uses_slpc(rps)) {
2649 /* RP limits have not been initialized yet for SLPC path */
2650 struct intel_rps_freq_caps caps;
2651
2652 gen6_rps_get_freq_caps(rps, &caps);
2653
2654 intel_rps_set_manual(rps, true);
2655 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2656 ((caps.rp0_freq <<
2657 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2658 GEN9_IGNORE_SLICE_RATIO));
2659 intel_rps_set_manual(rps, false);
2660 } else {
2661 intel_rps_set(rps, rps->rp0_freq);
2662 }
2663
2664 mutex_unlock(&rps->lock);
2665}
2666
2667void intel_rps_lower_unslice(struct intel_rps *rps)
2668{
2669 struct intel_uncore *uncore = rps_to_uncore(rps);
2670
2671 mutex_lock(&rps->lock);
2672
2673 if (rps_uses_slpc(rps)) {
2674 /* RP limits have not been initialized yet for SLPC path */
2675 struct intel_rps_freq_caps caps;
2676
2677 gen6_rps_get_freq_caps(rps, &caps);
2678
2679 intel_rps_set_manual(rps, true);
2680 intel_uncore_write(uncore, GEN6_RPNSWREQ,
2681 ((caps.min_freq <<
2682 GEN9_SW_REQ_UNSLICE_RATIO_SHIFT) |
2683 GEN9_IGNORE_SLICE_RATIO));
2684 intel_rps_set_manual(rps, false);
2685 } else {
2686 intel_rps_set(rps, rps->min_freq);
2687 }
2688
2689 mutex_unlock(&rps->lock);
2690}
2691
2692static u32 rps_read_mmio(struct intel_rps *rps, i915_reg_t reg32)
2693{
2694 struct intel_gt *gt = rps_to_gt(rps);
2695 intel_wakeref_t wakeref;
2696 u32 val;
2697
2698 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
2699 val = intel_uncore_read(gt->uncore, reg32);
2700
2701 return val;
2702}
2703
2704bool rps_read_mask_mmio(struct intel_rps *rps,
2705 i915_reg_t reg32, u32 mask)
2706{
2707 return rps_read_mmio(rps, reg32) & mask;
2708}
2709
2710/* External interface for intel_ips.ko */
2711
2712static struct drm_i915_private __rcu *ips_mchdev;
2713
2714/*
2715 * Tells the intel_ips driver that the i915 driver is now loaded, if
2716 * IPS got loaded first.
2717 *
2718 * This awkward dance is so that neither module has to depend on the
2719 * other in order for IPS to do the appropriate communication of
2720 * GPU turbo limits to i915.
2721 */
2722static void
2723ips_ping_for_i915_load(void)
2724{
2725 void (*link)(void);
2726
2727 link = symbol_get(ips_link_to_i915_driver);
2728 if (link) {
2729 link();
2730 symbol_put(ips_link_to_i915_driver);
2731 }
2732}
2733
2734void intel_rps_driver_register(struct intel_rps *rps)
2735{
2736 struct intel_gt *gt = rps_to_gt(rps);
2737
2738 /*
2739 * We only register the i915 ips part with intel-ips once everything is
2740 * set up, to avoid intel-ips sneaking in and reading bogus values.
2741 */
2742 if (GRAPHICS_VER(gt->i915) == 5) {
2743 GEM_BUG_ON(ips_mchdev);
2744 rcu_assign_pointer(ips_mchdev, gt->i915);
2745 ips_ping_for_i915_load();
2746 }
2747}
2748
2749void intel_rps_driver_unregister(struct intel_rps *rps)
2750{
2751 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps))
2752 rcu_assign_pointer(ips_mchdev, NULL);
2753}
2754
2755static struct drm_i915_private *mchdev_get(void)
2756{
2757 struct drm_i915_private *i915;
2758
2759 rcu_read_lock();
2760 i915 = rcu_dereference(ips_mchdev);
2761 if (i915 && !kref_get_unless_zero(&i915->drm.ref))
2762 i915 = NULL;
2763 rcu_read_unlock();
2764
2765 return i915;
2766}
2767
2768/**
2769 * i915_read_mch_val - return value for IPS use
2770 *
2771 * Calculate and return a value for the IPS driver to use when deciding whether
2772 * we have thermal and power headroom to increase CPU or GPU power budget.
2773 */
2774unsigned long i915_read_mch_val(void)
2775{
2776 struct drm_i915_private *i915;
2777 unsigned long chipset_val = 0;
2778 unsigned long graphics_val = 0;
2779 intel_wakeref_t wakeref;
2780
2781 i915 = mchdev_get();
2782 if (!i915)
2783 return 0;
2784
2785 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
2786 struct intel_ips *ips = &to_gt(i915)->rps.ips;
2787
2788 spin_lock_irq(&mchdev_lock);
2789 chipset_val = __ips_chipset_val(ips);
2790 graphics_val = __ips_gfx_val(ips);
2791 spin_unlock_irq(&mchdev_lock);
2792 }
2793
2794 drm_dev_put(&i915->drm);
2795 return chipset_val + graphics_val;
2796}
2797EXPORT_SYMBOL_GPL(i915_read_mch_val);
2798
2799/**
2800 * i915_gpu_raise - raise GPU frequency limit
2801 *
2802 * Raise the limit; IPS indicates we have thermal headroom.
2803 */
2804bool i915_gpu_raise(void)
2805{
2806 struct drm_i915_private *i915;
2807 struct intel_rps *rps;
2808
2809 i915 = mchdev_get();
2810 if (!i915)
2811 return false;
2812
2813 rps = &to_gt(i915)->rps;
2814
2815 spin_lock_irq(&mchdev_lock);
2816 if (rps->max_freq_softlimit < rps->max_freq)
2817 rps->max_freq_softlimit++;
2818 spin_unlock_irq(&mchdev_lock);
2819
2820 drm_dev_put(&i915->drm);
2821 return true;
2822}
2823EXPORT_SYMBOL_GPL(i915_gpu_raise);
2824
2825/**
2826 * i915_gpu_lower - lower GPU frequency limit
2827 *
2828 * IPS indicates we're close to a thermal limit, so throttle back the GPU
2829 * frequency maximum.
2830 */
2831bool i915_gpu_lower(void)
2832{
2833 struct drm_i915_private *i915;
2834 struct intel_rps *rps;
2835
2836 i915 = mchdev_get();
2837 if (!i915)
2838 return false;
2839
2840 rps = &to_gt(i915)->rps;
2841
2842 spin_lock_irq(&mchdev_lock);
2843 if (rps->max_freq_softlimit > rps->min_freq)
2844 rps->max_freq_softlimit--;
2845 spin_unlock_irq(&mchdev_lock);
2846
2847 drm_dev_put(&i915->drm);
2848 return true;
2849}
2850EXPORT_SYMBOL_GPL(i915_gpu_lower);
2851
2852/**
2853 * i915_gpu_busy - indicate GPU business to IPS
2854 *
2855 * Tell the IPS driver whether or not the GPU is busy.
2856 */
2857bool i915_gpu_busy(void)
2858{
2859 struct drm_i915_private *i915;
2860 bool ret;
2861
2862 i915 = mchdev_get();
2863 if (!i915)
2864 return false;
2865
2866 ret = to_gt(i915)->awake;
2867
2868 drm_dev_put(&i915->drm);
2869 return ret;
2870}
2871EXPORT_SYMBOL_GPL(i915_gpu_busy);
2872
2873/**
2874 * i915_gpu_turbo_disable - disable graphics turbo
2875 *
2876 * Disable graphics turbo by resetting the max frequency and setting the
2877 * current frequency to the default.
2878 */
2879bool i915_gpu_turbo_disable(void)
2880{
2881 struct drm_i915_private *i915;
2882 struct intel_rps *rps;
2883 bool ret;
2884
2885 i915 = mchdev_get();
2886 if (!i915)
2887 return false;
2888
2889 rps = &to_gt(i915)->rps;
2890
2891 spin_lock_irq(&mchdev_lock);
2892 rps->max_freq_softlimit = rps->min_freq;
2893 ret = !__gen5_rps_set(&to_gt(i915)->rps, rps->min_freq);
2894 spin_unlock_irq(&mchdev_lock);
2895
2896 drm_dev_put(&i915->drm);
2897 return ret;
2898}
2899EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable);
2900
2901#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2902#include "selftest_rps.c"
2903#include "selftest_slpc.c"
2904#endif