Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include <drm/drm_atomic_state_helper.h>
7
8#include "i915_drv.h"
9#include "i915_reg.h"
10#include "i915_utils.h"
11#include "intel_atomic.h"
12#include "intel_bw.h"
13#include "intel_cdclk.h"
14#include "intel_display_core.h"
15#include "intel_display_types.h"
16#include "skl_watermark.h"
17#include "intel_mchbar_regs.h"
18#include "intel_pcode.h"
19
20/* Parameters for Qclk Geyserville (QGV) */
21struct intel_qgv_point {
22 u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
23};
24
25struct intel_psf_gv_point {
26 u8 clk; /* clock in multiples of 16.6666 MHz */
27};
28
29struct intel_qgv_info {
30 struct intel_qgv_point points[I915_NUM_QGV_POINTS];
31 struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
32 u8 num_points;
33 u8 num_psf_points;
34 u8 t_bl;
35 u8 max_numchannels;
36 u8 channel_width;
37 u8 deinterleave;
38};
39
40static int dg1_mchbar_read_qgv_point_info(struct drm_i915_private *dev_priv,
41 struct intel_qgv_point *sp,
42 int point)
43{
44 u32 dclk_ratio, dclk_reference;
45 u32 val;
46
47 val = intel_uncore_read(&dev_priv->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
48 dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
49 if (val & DG1_QCLK_REFERENCE)
50 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
51 else
52 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
53 sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
54
55 val = intel_uncore_read(&dev_priv->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
56 if (val & DG1_GEAR_TYPE)
57 sp->dclk *= 2;
58
59 if (sp->dclk == 0)
60 return -EINVAL;
61
62 val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
63 sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
64 sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
65
66 val = intel_uncore_read(&dev_priv->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
67 sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
68 sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
69
70 sp->t_rc = sp->t_rp + sp->t_ras;
71
72 return 0;
73}
74
75static int icl_pcode_read_qgv_point_info(struct drm_i915_private *dev_priv,
76 struct intel_qgv_point *sp,
77 int point)
78{
79 u32 val = 0, val2 = 0;
80 u16 dclk;
81 int ret;
82
83 ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
84 ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
85 &val, &val2);
86 if (ret)
87 return ret;
88
89 dclk = val & 0xffff;
90 sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(dev_priv) > 11 ? 500 : 0), 1000);
91 sp->t_rp = (val & 0xff0000) >> 16;
92 sp->t_rcd = (val & 0xff000000) >> 24;
93
94 sp->t_rdpre = val2 & 0xff;
95 sp->t_ras = (val2 & 0xff00) >> 8;
96
97 sp->t_rc = sp->t_rp + sp->t_ras;
98
99 return 0;
100}
101
102static int adls_pcode_read_psf_gv_point_info(struct drm_i915_private *dev_priv,
103 struct intel_psf_gv_point *points)
104{
105 u32 val = 0;
106 int ret;
107 int i;
108
109 ret = snb_pcode_read(&dev_priv->uncore, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
110 ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
111 if (ret)
112 return ret;
113
114 for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
115 points[i].clk = val & 0xff;
116 val >>= 8;
117 }
118
119 return 0;
120}
121
122int icl_pcode_restrict_qgv_points(struct drm_i915_private *dev_priv,
123 u32 points_mask)
124{
125 int ret;
126
127 /* bspec says to keep retrying for at least 1 ms */
128 ret = skl_pcode_request(&dev_priv->uncore, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
129 points_mask,
130 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
131 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
132 1);
133
134 if (ret < 0) {
135 drm_err(&dev_priv->drm, "Failed to disable qgv points (%d) points: 0x%x\n", ret, points_mask);
136 return ret;
137 }
138
139 return 0;
140}
141
142static int mtl_read_qgv_point_info(struct drm_i915_private *dev_priv,
143 struct intel_qgv_point *sp, int point)
144{
145 u32 val, val2;
146 u16 dclk;
147
148 val = intel_uncore_read(&dev_priv->uncore,
149 MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
150 val2 = intel_uncore_read(&dev_priv->uncore,
151 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
152 dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
153 sp->dclk = DIV_ROUND_UP((16667 * dclk), 1000);
154 sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
155 sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
156
157 sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
158 sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
159
160 sp->t_rc = sp->t_rp + sp->t_ras;
161
162 return 0;
163}
164
165static int
166intel_read_qgv_point_info(struct drm_i915_private *dev_priv,
167 struct intel_qgv_point *sp,
168 int point)
169{
170 if (DISPLAY_VER(dev_priv) >= 14)
171 return mtl_read_qgv_point_info(dev_priv, sp, point);
172 else if (IS_DG1(dev_priv))
173 return dg1_mchbar_read_qgv_point_info(dev_priv, sp, point);
174 else
175 return icl_pcode_read_qgv_point_info(dev_priv, sp, point);
176}
177
178static int icl_get_qgv_points(struct drm_i915_private *dev_priv,
179 struct intel_qgv_info *qi,
180 bool is_y_tile)
181{
182 const struct dram_info *dram_info = &dev_priv->dram_info;
183 int i, ret;
184
185 qi->num_points = dram_info->num_qgv_points;
186 qi->num_psf_points = dram_info->num_psf_gv_points;
187
188 if (DISPLAY_VER(dev_priv) >= 14) {
189 switch (dram_info->type) {
190 case INTEL_DRAM_DDR4:
191 qi->t_bl = 4;
192 qi->max_numchannels = 2;
193 qi->channel_width = 64;
194 qi->deinterleave = 2;
195 break;
196 case INTEL_DRAM_DDR5:
197 qi->t_bl = 8;
198 qi->max_numchannels = 4;
199 qi->channel_width = 32;
200 qi->deinterleave = 2;
201 break;
202 case INTEL_DRAM_LPDDR4:
203 case INTEL_DRAM_LPDDR5:
204 qi->t_bl = 16;
205 qi->max_numchannels = 8;
206 qi->channel_width = 16;
207 qi->deinterleave = 4;
208 break;
209 default:
210 MISSING_CASE(dram_info->type);
211 return -EINVAL;
212 }
213 } else if (DISPLAY_VER(dev_priv) >= 12) {
214 switch (dram_info->type) {
215 case INTEL_DRAM_DDR4:
216 qi->t_bl = is_y_tile ? 8 : 4;
217 qi->max_numchannels = 2;
218 qi->channel_width = 64;
219 qi->deinterleave = is_y_tile ? 1 : 2;
220 break;
221 case INTEL_DRAM_DDR5:
222 qi->t_bl = is_y_tile ? 16 : 8;
223 qi->max_numchannels = 4;
224 qi->channel_width = 32;
225 qi->deinterleave = is_y_tile ? 1 : 2;
226 break;
227 case INTEL_DRAM_LPDDR4:
228 if (IS_ROCKETLAKE(dev_priv)) {
229 qi->t_bl = 8;
230 qi->max_numchannels = 4;
231 qi->channel_width = 32;
232 qi->deinterleave = 2;
233 break;
234 }
235 fallthrough;
236 case INTEL_DRAM_LPDDR5:
237 qi->t_bl = 16;
238 qi->max_numchannels = 8;
239 qi->channel_width = 16;
240 qi->deinterleave = is_y_tile ? 2 : 4;
241 break;
242 default:
243 qi->t_bl = 16;
244 qi->max_numchannels = 1;
245 break;
246 }
247 } else if (DISPLAY_VER(dev_priv) == 11) {
248 qi->t_bl = dev_priv->dram_info.type == INTEL_DRAM_DDR4 ? 4 : 8;
249 qi->max_numchannels = 1;
250 }
251
252 if (drm_WARN_ON(&dev_priv->drm,
253 qi->num_points > ARRAY_SIZE(qi->points)))
254 qi->num_points = ARRAY_SIZE(qi->points);
255
256 for (i = 0; i < qi->num_points; i++) {
257 struct intel_qgv_point *sp = &qi->points[i];
258
259 ret = intel_read_qgv_point_info(dev_priv, sp, i);
260 if (ret)
261 return ret;
262
263 drm_dbg_kms(&dev_priv->drm,
264 "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
265 i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
266 sp->t_rcd, sp->t_rc);
267 }
268
269 if (qi->num_psf_points > 0) {
270 ret = adls_pcode_read_psf_gv_point_info(dev_priv, qi->psf_points);
271 if (ret) {
272 drm_err(&dev_priv->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
273 qi->num_psf_points = 0;
274 }
275
276 for (i = 0; i < qi->num_psf_points; i++)
277 drm_dbg_kms(&dev_priv->drm,
278 "PSF GV %d: CLK=%d \n",
279 i, qi->psf_points[i].clk);
280 }
281
282 return 0;
283}
284
285static int adl_calc_psf_bw(int clk)
286{
287 /*
288 * clk is multiples of 16.666MHz (100/6)
289 * According to BSpec PSF GV bandwidth is
290 * calculated as BW = 64 * clk * 16.666Mhz
291 */
292 return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
293}
294
295static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
296{
297 u16 dclk = 0;
298 int i;
299
300 for (i = 0; i < qi->num_points; i++)
301 dclk = max(dclk, qi->points[i].dclk);
302
303 return dclk;
304}
305
306struct intel_sa_info {
307 u16 displayrtids;
308 u8 deburst, deprogbwlimit, derating;
309};
310
311static const struct intel_sa_info icl_sa_info = {
312 .deburst = 8,
313 .deprogbwlimit = 25, /* GB/s */
314 .displayrtids = 128,
315 .derating = 10,
316};
317
318static const struct intel_sa_info tgl_sa_info = {
319 .deburst = 16,
320 .deprogbwlimit = 34, /* GB/s */
321 .displayrtids = 256,
322 .derating = 10,
323};
324
325static const struct intel_sa_info rkl_sa_info = {
326 .deburst = 8,
327 .deprogbwlimit = 20, /* GB/s */
328 .displayrtids = 128,
329 .derating = 10,
330};
331
332static const struct intel_sa_info adls_sa_info = {
333 .deburst = 16,
334 .deprogbwlimit = 38, /* GB/s */
335 .displayrtids = 256,
336 .derating = 10,
337};
338
339static const struct intel_sa_info adlp_sa_info = {
340 .deburst = 16,
341 .deprogbwlimit = 38, /* GB/s */
342 .displayrtids = 256,
343 .derating = 20,
344};
345
346static const struct intel_sa_info mtl_sa_info = {
347 .deburst = 32,
348 .deprogbwlimit = 38, /* GB/s */
349 .displayrtids = 256,
350 .derating = 20,
351};
352
353static int icl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
354{
355 struct intel_qgv_info qi = {};
356 bool is_y_tile = true; /* assume y tile may be used */
357 int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
358 int ipqdepth, ipqdepthpch = 16;
359 int dclk_max;
360 int maxdebw;
361 int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
362 int i, ret;
363
364 ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
365 if (ret) {
366 drm_dbg_kms(&dev_priv->drm,
367 "Failed to get memory subsystem information, ignoring bandwidth limits");
368 return ret;
369 }
370
371 dclk_max = icl_sagv_max_dclk(&qi);
372 maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
373 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
374 qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
375
376 for (i = 0; i < num_groups; i++) {
377 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
378 int clpchgroup;
379 int j;
380
381 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
382 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
383
384 bi->num_qgv_points = qi.num_points;
385 bi->num_psf_gv_points = qi.num_psf_points;
386
387 for (j = 0; j < qi.num_points; j++) {
388 const struct intel_qgv_point *sp = &qi.points[j];
389 int ct, bw;
390
391 /*
392 * Max row cycle time
393 *
394 * FIXME what is the logic behind the
395 * assumed burst length?
396 */
397 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
398 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
399 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
400
401 bi->deratedbw[j] = min(maxdebw,
402 bw * (100 - sa->derating) / 100);
403
404 drm_dbg_kms(&dev_priv->drm,
405 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
406 i, j, bi->num_planes, bi->deratedbw[j]);
407 }
408 }
409 /*
410 * In case if SAGV is disabled in BIOS, we always get 1
411 * SAGV point, but we can't send PCode commands to restrict it
412 * as it will fail and pointless anyway.
413 */
414 if (qi.num_points == 1)
415 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
416 else
417 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
418
419 return 0;
420}
421
422static int tgl_get_bw_info(struct drm_i915_private *dev_priv, const struct intel_sa_info *sa)
423{
424 struct intel_qgv_info qi = {};
425 const struct dram_info *dram_info = &dev_priv->dram_info;
426 bool is_y_tile = true; /* assume y tile may be used */
427 int num_channels = max_t(u8, 1, dev_priv->dram_info.num_channels);
428 int ipqdepth, ipqdepthpch = 16;
429 int dclk_max;
430 int maxdebw, peakbw;
431 int clperchgroup;
432 int num_groups = ARRAY_SIZE(dev_priv->display.bw.max);
433 int i, ret;
434
435 ret = icl_get_qgv_points(dev_priv, &qi, is_y_tile);
436 if (ret) {
437 drm_dbg_kms(&dev_priv->drm,
438 "Failed to get memory subsystem information, ignoring bandwidth limits");
439 return ret;
440 }
441
442 if (DISPLAY_VER(dev_priv) < 14 &&
443 (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
444 num_channels *= 2;
445
446 qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
447
448 if (num_channels < qi.max_numchannels && DISPLAY_VER(dev_priv) >= 12)
449 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
450
451 if (DISPLAY_VER(dev_priv) > 11 && num_channels > qi.max_numchannels)
452 drm_warn(&dev_priv->drm, "Number of channels exceeds max number of channels.");
453 if (qi.max_numchannels != 0)
454 num_channels = min_t(u8, num_channels, qi.max_numchannels);
455
456 dclk_max = icl_sagv_max_dclk(&qi);
457
458 peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
459 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * 6 / 10); /* 60% */
460
461 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
462 /*
463 * clperchgroup = 4kpagespermempage * clperchperblock,
464 * clperchperblock = 8 / num_channels * interleave
465 */
466 clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
467
468 for (i = 0; i < num_groups; i++) {
469 struct intel_bw_info *bi = &dev_priv->display.bw.max[i];
470 struct intel_bw_info *bi_next;
471 int clpchgroup;
472 int j;
473
474 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
475
476 if (i < num_groups - 1) {
477 bi_next = &dev_priv->display.bw.max[i + 1];
478
479 if (clpchgroup < clperchgroup)
480 bi_next->num_planes = (ipqdepth - clpchgroup) /
481 clpchgroup + 1;
482 else
483 bi_next->num_planes = 0;
484 }
485
486 bi->num_qgv_points = qi.num_points;
487 bi->num_psf_gv_points = qi.num_psf_points;
488
489 for (j = 0; j < qi.num_points; j++) {
490 const struct intel_qgv_point *sp = &qi.points[j];
491 int ct, bw;
492
493 /*
494 * Max row cycle time
495 *
496 * FIXME what is the logic behind the
497 * assumed burst length?
498 */
499 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
500 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
501 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
502
503 bi->deratedbw[j] = min(maxdebw,
504 bw * (100 - sa->derating) / 100);
505
506 drm_dbg_kms(&dev_priv->drm,
507 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
508 i, j, bi->num_planes, bi->deratedbw[j]);
509 }
510
511 for (j = 0; j < qi.num_psf_points; j++) {
512 const struct intel_psf_gv_point *sp = &qi.psf_points[j];
513
514 bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
515
516 drm_dbg_kms(&dev_priv->drm,
517 "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
518 i, j, bi->num_planes, bi->psf_bw[j]);
519 }
520 }
521
522 /*
523 * In case if SAGV is disabled in BIOS, we always get 1
524 * SAGV point, but we can't send PCode commands to restrict it
525 * as it will fail and pointless anyway.
526 */
527 if (qi.num_points == 1)
528 dev_priv->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
529 else
530 dev_priv->display.sagv.status = I915_SAGV_ENABLED;
531
532 return 0;
533}
534
535static void dg2_get_bw_info(struct drm_i915_private *i915)
536{
537 unsigned int deratedbw = IS_DG2_G11(i915) ? 38000 : 50000;
538 int num_groups = ARRAY_SIZE(i915->display.bw.max);
539 int i;
540
541 /*
542 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
543 * that doesn't depend on the number of planes enabled. So fill all the
544 * plane group with constant bw information for uniformity with other
545 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
546 * whereas DG2-G11 platforms have 38 GB/s.
547 */
548 for (i = 0; i < num_groups; i++) {
549 struct intel_bw_info *bi = &i915->display.bw.max[i];
550
551 bi->num_planes = 1;
552 /* Need only one dummy QGV point per group */
553 bi->num_qgv_points = 1;
554 bi->deratedbw[0] = deratedbw;
555 }
556
557 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
558}
559
560static unsigned int icl_max_bw(struct drm_i915_private *dev_priv,
561 int num_planes, int qgv_point)
562{
563 int i;
564
565 /*
566 * Let's return max bw for 0 planes
567 */
568 num_planes = max(1, num_planes);
569
570 for (i = 0; i < ARRAY_SIZE(dev_priv->display.bw.max); i++) {
571 const struct intel_bw_info *bi =
572 &dev_priv->display.bw.max[i];
573
574 /*
575 * Pcode will not expose all QGV points when
576 * SAGV is forced to off/min/med/max.
577 */
578 if (qgv_point >= bi->num_qgv_points)
579 return UINT_MAX;
580
581 if (num_planes >= bi->num_planes)
582 return bi->deratedbw[qgv_point];
583 }
584
585 return 0;
586}
587
588static unsigned int tgl_max_bw(struct drm_i915_private *dev_priv,
589 int num_planes, int qgv_point)
590{
591 int i;
592
593 /*
594 * Let's return max bw for 0 planes
595 */
596 num_planes = max(1, num_planes);
597
598 for (i = ARRAY_SIZE(dev_priv->display.bw.max) - 1; i >= 0; i--) {
599 const struct intel_bw_info *bi =
600 &dev_priv->display.bw.max[i];
601
602 /*
603 * Pcode will not expose all QGV points when
604 * SAGV is forced to off/min/med/max.
605 */
606 if (qgv_point >= bi->num_qgv_points)
607 return UINT_MAX;
608
609 if (num_planes <= bi->num_planes)
610 return bi->deratedbw[qgv_point];
611 }
612
613 return dev_priv->display.bw.max[0].deratedbw[qgv_point];
614}
615
616static unsigned int adl_psf_bw(struct drm_i915_private *dev_priv,
617 int psf_gv_point)
618{
619 const struct intel_bw_info *bi =
620 &dev_priv->display.bw.max[0];
621
622 return bi->psf_bw[psf_gv_point];
623}
624
625void intel_bw_init_hw(struct drm_i915_private *dev_priv)
626{
627 if (!HAS_DISPLAY(dev_priv))
628 return;
629
630 if (DISPLAY_VER(dev_priv) >= 14)
631 tgl_get_bw_info(dev_priv, &mtl_sa_info);
632 else if (IS_DG2(dev_priv))
633 dg2_get_bw_info(dev_priv);
634 else if (IS_ALDERLAKE_P(dev_priv))
635 tgl_get_bw_info(dev_priv, &adlp_sa_info);
636 else if (IS_ALDERLAKE_S(dev_priv))
637 tgl_get_bw_info(dev_priv, &adls_sa_info);
638 else if (IS_ROCKETLAKE(dev_priv))
639 tgl_get_bw_info(dev_priv, &rkl_sa_info);
640 else if (DISPLAY_VER(dev_priv) == 12)
641 tgl_get_bw_info(dev_priv, &tgl_sa_info);
642 else if (DISPLAY_VER(dev_priv) == 11)
643 icl_get_bw_info(dev_priv, &icl_sa_info);
644}
645
646static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
647{
648 /*
649 * We assume cursors are small enough
650 * to not not cause bandwidth problems.
651 */
652 return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
653}
654
655static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
656{
657 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
658 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
659 unsigned int data_rate = 0;
660 enum plane_id plane_id;
661
662 for_each_plane_id_on_crtc(crtc, plane_id) {
663 /*
664 * We assume cursors are small enough
665 * to not not cause bandwidth problems.
666 */
667 if (plane_id == PLANE_CURSOR)
668 continue;
669
670 data_rate += crtc_state->data_rate[plane_id];
671
672 if (DISPLAY_VER(i915) < 11)
673 data_rate += crtc_state->data_rate_y[plane_id];
674 }
675
676 return data_rate;
677}
678
679/* "Maximum Pipe Read Bandwidth" */
680static int intel_bw_crtc_min_cdclk(const struct intel_crtc_state *crtc_state)
681{
682 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
683 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
684
685 if (DISPLAY_VER(i915) < 12)
686 return 0;
687
688 return DIV_ROUND_UP_ULL(mul_u32_u32(intel_bw_crtc_data_rate(crtc_state), 10), 512);
689}
690
691void intel_bw_crtc_update(struct intel_bw_state *bw_state,
692 const struct intel_crtc_state *crtc_state)
693{
694 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
695 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
696
697 bw_state->data_rate[crtc->pipe] =
698 intel_bw_crtc_data_rate(crtc_state);
699 bw_state->num_active_planes[crtc->pipe] =
700 intel_bw_crtc_num_active_planes(crtc_state);
701
702 drm_dbg_kms(&i915->drm, "pipe %c data rate %u num active planes %u\n",
703 pipe_name(crtc->pipe),
704 bw_state->data_rate[crtc->pipe],
705 bw_state->num_active_planes[crtc->pipe]);
706}
707
708static unsigned int intel_bw_num_active_planes(struct drm_i915_private *dev_priv,
709 const struct intel_bw_state *bw_state)
710{
711 unsigned int num_active_planes = 0;
712 enum pipe pipe;
713
714 for_each_pipe(dev_priv, pipe)
715 num_active_planes += bw_state->num_active_planes[pipe];
716
717 return num_active_planes;
718}
719
720static unsigned int intel_bw_data_rate(struct drm_i915_private *dev_priv,
721 const struct intel_bw_state *bw_state)
722{
723 unsigned int data_rate = 0;
724 enum pipe pipe;
725
726 for_each_pipe(dev_priv, pipe)
727 data_rate += bw_state->data_rate[pipe];
728
729 if (DISPLAY_VER(dev_priv) >= 13 && i915_vtd_active(dev_priv))
730 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
731
732 return data_rate;
733}
734
735struct intel_bw_state *
736intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
737{
738 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
739 struct intel_global_state *bw_state;
740
741 bw_state = intel_atomic_get_old_global_obj_state(state, &dev_priv->display.bw.obj);
742
743 return to_intel_bw_state(bw_state);
744}
745
746struct intel_bw_state *
747intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
748{
749 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
750 struct intel_global_state *bw_state;
751
752 bw_state = intel_atomic_get_new_global_obj_state(state, &dev_priv->display.bw.obj);
753
754 return to_intel_bw_state(bw_state);
755}
756
757struct intel_bw_state *
758intel_atomic_get_bw_state(struct intel_atomic_state *state)
759{
760 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
761 struct intel_global_state *bw_state;
762
763 bw_state = intel_atomic_get_global_obj_state(state, &dev_priv->display.bw.obj);
764 if (IS_ERR(bw_state))
765 return ERR_CAST(bw_state);
766
767 return to_intel_bw_state(bw_state);
768}
769
770static bool intel_bw_state_changed(struct drm_i915_private *i915,
771 const struct intel_bw_state *old_bw_state,
772 const struct intel_bw_state *new_bw_state)
773{
774 enum pipe pipe;
775
776 for_each_pipe(i915, pipe) {
777 const struct intel_dbuf_bw *old_crtc_bw =
778 &old_bw_state->dbuf_bw[pipe];
779 const struct intel_dbuf_bw *new_crtc_bw =
780 &new_bw_state->dbuf_bw[pipe];
781 enum dbuf_slice slice;
782
783 for_each_dbuf_slice(i915, slice) {
784 if (old_crtc_bw->max_bw[slice] != new_crtc_bw->max_bw[slice] ||
785 old_crtc_bw->active_planes[slice] != new_crtc_bw->active_planes[slice])
786 return true;
787 }
788
789 if (old_bw_state->min_cdclk[pipe] != new_bw_state->min_cdclk[pipe])
790 return true;
791 }
792
793 return false;
794}
795
796static void skl_plane_calc_dbuf_bw(struct intel_bw_state *bw_state,
797 struct intel_crtc *crtc,
798 enum plane_id plane_id,
799 const struct skl_ddb_entry *ddb,
800 unsigned int data_rate)
801{
802 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
803 struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
804 unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(i915, ddb);
805 enum dbuf_slice slice;
806
807 /*
808 * The arbiter can only really guarantee an
809 * equal share of the total bw to each plane.
810 */
811 for_each_dbuf_slice_in_mask(i915, slice, dbuf_mask) {
812 crtc_bw->max_bw[slice] = max(crtc_bw->max_bw[slice], data_rate);
813 crtc_bw->active_planes[slice] |= BIT(plane_id);
814 }
815}
816
817static void skl_crtc_calc_dbuf_bw(struct intel_bw_state *bw_state,
818 const struct intel_crtc_state *crtc_state)
819{
820 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
821 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
822 struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[crtc->pipe];
823 enum plane_id plane_id;
824
825 memset(crtc_bw, 0, sizeof(*crtc_bw));
826
827 if (!crtc_state->hw.active)
828 return;
829
830 for_each_plane_id_on_crtc(crtc, plane_id) {
831 /*
832 * We assume cursors are small enough
833 * to not cause bandwidth problems.
834 */
835 if (plane_id == PLANE_CURSOR)
836 continue;
837
838 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
839 &crtc_state->wm.skl.plane_ddb[plane_id],
840 crtc_state->data_rate[plane_id]);
841
842 if (DISPLAY_VER(i915) < 11)
843 skl_plane_calc_dbuf_bw(bw_state, crtc, plane_id,
844 &crtc_state->wm.skl.plane_ddb_y[plane_id],
845 crtc_state->data_rate[plane_id]);
846 }
847}
848
849/* "Maximum Data Buffer Bandwidth" */
850static int
851intel_bw_dbuf_min_cdclk(struct drm_i915_private *i915,
852 const struct intel_bw_state *bw_state)
853{
854 unsigned int total_max_bw = 0;
855 enum dbuf_slice slice;
856
857 for_each_dbuf_slice(i915, slice) {
858 int num_active_planes = 0;
859 unsigned int max_bw = 0;
860 enum pipe pipe;
861
862 /*
863 * The arbiter can only really guarantee an
864 * equal share of the total bw to each plane.
865 */
866 for_each_pipe(i915, pipe) {
867 const struct intel_dbuf_bw *crtc_bw = &bw_state->dbuf_bw[pipe];
868
869 max_bw = max(crtc_bw->max_bw[slice], max_bw);
870 num_active_planes += hweight8(crtc_bw->active_planes[slice]);
871 }
872 max_bw *= num_active_planes;
873
874 total_max_bw = max(total_max_bw, max_bw);
875 }
876
877 return DIV_ROUND_UP(total_max_bw, 64);
878}
879
880int intel_bw_min_cdclk(struct drm_i915_private *i915,
881 const struct intel_bw_state *bw_state)
882{
883 enum pipe pipe;
884 int min_cdclk;
885
886 min_cdclk = intel_bw_dbuf_min_cdclk(i915, bw_state);
887
888 for_each_pipe(i915, pipe)
889 min_cdclk = max(bw_state->min_cdclk[pipe], min_cdclk);
890
891 return min_cdclk;
892}
893
894int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
895 bool *need_cdclk_calc)
896{
897 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
898 struct intel_bw_state *new_bw_state = NULL;
899 const struct intel_bw_state *old_bw_state = NULL;
900 const struct intel_cdclk_state *cdclk_state;
901 const struct intel_crtc_state *crtc_state;
902 int old_min_cdclk, new_min_cdclk;
903 struct intel_crtc *crtc;
904 int i;
905
906 if (DISPLAY_VER(dev_priv) < 9)
907 return 0;
908
909 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
910 new_bw_state = intel_atomic_get_bw_state(state);
911 if (IS_ERR(new_bw_state))
912 return PTR_ERR(new_bw_state);
913
914 old_bw_state = intel_atomic_get_old_bw_state(state);
915
916 skl_crtc_calc_dbuf_bw(new_bw_state, crtc_state);
917
918 new_bw_state->min_cdclk[crtc->pipe] =
919 intel_bw_crtc_min_cdclk(crtc_state);
920 }
921
922 if (!old_bw_state)
923 return 0;
924
925 if (intel_bw_state_changed(dev_priv, old_bw_state, new_bw_state)) {
926 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
927 if (ret)
928 return ret;
929 }
930
931 old_min_cdclk = intel_bw_min_cdclk(dev_priv, old_bw_state);
932 new_min_cdclk = intel_bw_min_cdclk(dev_priv, new_bw_state);
933
934 /*
935 * No need to check against the cdclk state if
936 * the min cdclk doesn't increase.
937 *
938 * Ie. we only ever increase the cdclk due to bandwidth
939 * requirements. This can reduce back and forth
940 * display blinking due to constant cdclk changes.
941 */
942 if (new_min_cdclk <= old_min_cdclk)
943 return 0;
944
945 cdclk_state = intel_atomic_get_cdclk_state(state);
946 if (IS_ERR(cdclk_state))
947 return PTR_ERR(cdclk_state);
948
949 /*
950 * No need to recalculate the cdclk state if
951 * the min cdclk doesn't increase.
952 *
953 * Ie. we only ever increase the cdclk due to bandwidth
954 * requirements. This can reduce back and forth
955 * display blinking due to constant cdclk changes.
956 */
957 if (new_min_cdclk <= cdclk_state->bw_min_cdclk)
958 return 0;
959
960 drm_dbg_kms(&dev_priv->drm,
961 "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
962 new_min_cdclk, cdclk_state->bw_min_cdclk);
963 *need_cdclk_calc = true;
964
965 return 0;
966}
967
968static u16 icl_qgv_points_mask(struct drm_i915_private *i915)
969{
970 unsigned int num_psf_gv_points = i915->display.bw.max[0].num_psf_gv_points;
971 unsigned int num_qgv_points = i915->display.bw.max[0].num_qgv_points;
972 u16 qgv_points = 0, psf_points = 0;
973
974 /*
975 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
976 * it with failure if we try masking any unadvertised points.
977 * So need to operate only with those returned from PCode.
978 */
979 if (num_qgv_points > 0)
980 qgv_points = GENMASK(num_qgv_points - 1, 0);
981
982 if (num_psf_gv_points > 0)
983 psf_points = GENMASK(num_psf_gv_points - 1, 0);
984
985 return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
986}
987
988static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
989{
990 struct drm_i915_private *i915 = to_i915(state->base.dev);
991 const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
992 struct intel_crtc *crtc;
993 int i;
994
995 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
996 new_crtc_state, i) {
997 unsigned int old_data_rate =
998 intel_bw_crtc_data_rate(old_crtc_state);
999 unsigned int new_data_rate =
1000 intel_bw_crtc_data_rate(new_crtc_state);
1001 unsigned int old_active_planes =
1002 intel_bw_crtc_num_active_planes(old_crtc_state);
1003 unsigned int new_active_planes =
1004 intel_bw_crtc_num_active_planes(new_crtc_state);
1005 struct intel_bw_state *new_bw_state;
1006
1007 /*
1008 * Avoid locking the bw state when
1009 * nothing significant has changed.
1010 */
1011 if (old_data_rate == new_data_rate &&
1012 old_active_planes == new_active_planes)
1013 continue;
1014
1015 new_bw_state = intel_atomic_get_bw_state(state);
1016 if (IS_ERR(new_bw_state))
1017 return PTR_ERR(new_bw_state);
1018
1019 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1020 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1021
1022 *changed = true;
1023
1024 drm_dbg_kms(&i915->drm,
1025 "[CRTC:%d:%s] data rate %u num active planes %u\n",
1026 crtc->base.base.id, crtc->base.name,
1027 new_bw_state->data_rate[crtc->pipe],
1028 new_bw_state->num_active_planes[crtc->pipe]);
1029 }
1030
1031 return 0;
1032}
1033
1034int intel_bw_atomic_check(struct intel_atomic_state *state)
1035{
1036 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
1037 const struct intel_bw_state *old_bw_state;
1038 struct intel_bw_state *new_bw_state;
1039 unsigned int data_rate;
1040 unsigned int num_active_planes;
1041 int i, ret;
1042 u16 qgv_points = 0, psf_points = 0;
1043 unsigned int max_bw_point = 0, max_bw = 0;
1044 unsigned int num_qgv_points = dev_priv->display.bw.max[0].num_qgv_points;
1045 unsigned int num_psf_gv_points = dev_priv->display.bw.max[0].num_psf_gv_points;
1046 bool changed = false;
1047
1048 /* FIXME earlier gens need some checks too */
1049 if (DISPLAY_VER(dev_priv) < 11)
1050 return 0;
1051
1052 ret = intel_bw_check_data_rate(state, &changed);
1053 if (ret)
1054 return ret;
1055
1056 old_bw_state = intel_atomic_get_old_bw_state(state);
1057 new_bw_state = intel_atomic_get_new_bw_state(state);
1058
1059 if (new_bw_state &&
1060 intel_can_enable_sagv(dev_priv, old_bw_state) !=
1061 intel_can_enable_sagv(dev_priv, new_bw_state))
1062 changed = true;
1063
1064 /*
1065 * If none of our inputs (data rates, number of active
1066 * planes, SAGV yes/no) changed then nothing to do here.
1067 */
1068 if (!changed)
1069 return 0;
1070
1071 ret = intel_atomic_lock_global_state(&new_bw_state->base);
1072 if (ret)
1073 return ret;
1074
1075 data_rate = intel_bw_data_rate(dev_priv, new_bw_state);
1076 data_rate = DIV_ROUND_UP(data_rate, 1000);
1077
1078 num_active_planes = intel_bw_num_active_planes(dev_priv, new_bw_state);
1079
1080 for (i = 0; i < num_qgv_points; i++) {
1081 unsigned int max_data_rate;
1082
1083 if (DISPLAY_VER(dev_priv) > 11)
1084 max_data_rate = tgl_max_bw(dev_priv, num_active_planes, i);
1085 else
1086 max_data_rate = icl_max_bw(dev_priv, num_active_planes, i);
1087 /*
1088 * We need to know which qgv point gives us
1089 * maximum bandwidth in order to disable SAGV
1090 * if we find that we exceed SAGV block time
1091 * with watermarks. By that moment we already
1092 * have those, as it is calculated earlier in
1093 * intel_atomic_check,
1094 */
1095 if (max_data_rate > max_bw) {
1096 max_bw_point = i;
1097 max_bw = max_data_rate;
1098 }
1099 if (max_data_rate >= data_rate)
1100 qgv_points |= BIT(i);
1101
1102 drm_dbg_kms(&dev_priv->drm, "QGV point %d: max bw %d required %d\n",
1103 i, max_data_rate, data_rate);
1104 }
1105
1106 for (i = 0; i < num_psf_gv_points; i++) {
1107 unsigned int max_data_rate = adl_psf_bw(dev_priv, i);
1108
1109 if (max_data_rate >= data_rate)
1110 psf_points |= BIT(i);
1111
1112 drm_dbg_kms(&dev_priv->drm, "PSF GV point %d: max bw %d"
1113 " required %d\n",
1114 i, max_data_rate, data_rate);
1115 }
1116
1117 /*
1118 * BSpec states that we always should have at least one allowed point
1119 * left, so if we couldn't - simply reject the configuration for obvious
1120 * reasons.
1121 */
1122 if (qgv_points == 0) {
1123 drm_dbg_kms(&dev_priv->drm, "No QGV points provide sufficient memory"
1124 " bandwidth %d for display configuration(%d active planes).\n",
1125 data_rate, num_active_planes);
1126 return -EINVAL;
1127 }
1128
1129 if (num_psf_gv_points > 0 && psf_points == 0) {
1130 drm_dbg_kms(&dev_priv->drm, "No PSF GV points provide sufficient memory"
1131 " bandwidth %d for display configuration(%d active planes).\n",
1132 data_rate, num_active_planes);
1133 return -EINVAL;
1134 }
1135
1136 /*
1137 * Leave only single point with highest bandwidth, if
1138 * we can't enable SAGV due to the increased memory latency it may
1139 * cause.
1140 */
1141 if (!intel_can_enable_sagv(dev_priv, new_bw_state)) {
1142 qgv_points = BIT(max_bw_point);
1143 drm_dbg_kms(&dev_priv->drm, "No SAGV, using single QGV point %d\n",
1144 max_bw_point);
1145 }
1146
1147 /*
1148 * We store the ones which need to be masked as that is what PCode
1149 * actually accepts as a parameter.
1150 */
1151 new_bw_state->qgv_points_mask =
1152 ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
1153 ADLS_PCODE_REQ_PSF_PT(psf_points)) &
1154 icl_qgv_points_mask(dev_priv);
1155
1156 /*
1157 * If the actual mask had changed we need to make sure that
1158 * the commits are serialized(in case this is a nomodeset, nonblocking)
1159 */
1160 if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1161 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1162 if (ret)
1163 return ret;
1164 }
1165
1166 return 0;
1167}
1168
1169static struct intel_global_state *
1170intel_bw_duplicate_state(struct intel_global_obj *obj)
1171{
1172 struct intel_bw_state *state;
1173
1174 state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1175 if (!state)
1176 return NULL;
1177
1178 return &state->base;
1179}
1180
1181static void intel_bw_destroy_state(struct intel_global_obj *obj,
1182 struct intel_global_state *state)
1183{
1184 kfree(state);
1185}
1186
1187static const struct intel_global_state_funcs intel_bw_funcs = {
1188 .atomic_duplicate_state = intel_bw_duplicate_state,
1189 .atomic_destroy_state = intel_bw_destroy_state,
1190};
1191
1192int intel_bw_init(struct drm_i915_private *dev_priv)
1193{
1194 struct intel_bw_state *state;
1195
1196 state = kzalloc(sizeof(*state), GFP_KERNEL);
1197 if (!state)
1198 return -ENOMEM;
1199
1200 intel_atomic_global_obj_init(dev_priv, &dev_priv->display.bw.obj,
1201 &state->base, &intel_bw_funcs);
1202
1203 return 0;
1204}