Loading...
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5#include <linux/kernel.h>
6#include <linux/slab.h>
7
8#include <drm/drm_atomic_helper.h>
9#include <drm/drm_fourcc.h>
10#include <drm/drm_plane.h>
11#include <drm/drm_plane_helper.h>
12
13#include "i915_trace.h"
14#include "i915_vgpu.h"
15
16#include "intel_atomic.h"
17#include "intel_atomic_plane.h"
18#include "intel_color.h"
19#include "intel_crtc.h"
20#include "intel_cursor.h"
21#include "intel_display_debugfs.h"
22#include "intel_display_types.h"
23#include "intel_dsi.h"
24#include "intel_pipe_crc.h"
25#include "intel_psr.h"
26#include "intel_sprite.h"
27#include "intel_vrr.h"
28#include "i9xx_plane.h"
29#include "skl_universal_plane.h"
30
31static void assert_vblank_disabled(struct drm_crtc *crtc)
32{
33 if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
34 drm_crtc_vblank_put(crtc);
35}
36
37u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
38{
39 struct drm_device *dev = crtc->base.dev;
40 struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
41
42 if (!crtc->active)
43 return 0;
44
45 if (!vblank->max_vblank_count)
46 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
47
48 return crtc->base.funcs->get_vblank_counter(&crtc->base);
49}
50
51u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
52{
53 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
54
55 /*
56 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
57 * have updated at the beginning of TE, if we want to use
58 * the hw counter, then we would find it updated in only
59 * the next TE, hence switching to sw counter.
60 */
61 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
62 I915_MODE_FLAG_DSI_USE_TE1))
63 return 0;
64
65 /*
66 * On i965gm the hardware frame counter reads
67 * zero when the TV encoder is enabled :(
68 */
69 if (IS_I965GM(dev_priv) &&
70 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
71 return 0;
72
73 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
74 return 0xffffffff; /* full 32 bit counter */
75 else if (DISPLAY_VER(dev_priv) >= 3)
76 return 0xffffff; /* only 24 bits of frame count */
77 else
78 return 0; /* Gen2 doesn't have a hardware frame counter */
79}
80
81void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
82{
83 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
84
85 assert_vblank_disabled(&crtc->base);
86 drm_crtc_set_max_vblank_count(&crtc->base,
87 intel_crtc_max_vblank_count(crtc_state));
88 drm_crtc_vblank_on(&crtc->base);
89
90 /*
91 * Should really happen exactly when we enable the pipe
92 * but we want the frame counters in the trace, and that
93 * requires vblank support on some platforms/outputs.
94 */
95 trace_intel_pipe_enable(crtc);
96}
97
98void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
99{
100 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
101
102 /*
103 * Should really happen exactly when we disable the pipe
104 * but we want the frame counters in the trace, and that
105 * requires vblank support on some platforms/outputs.
106 */
107 trace_intel_pipe_disable(crtc);
108
109 drm_crtc_vblank_off(&crtc->base);
110 assert_vblank_disabled(&crtc->base);
111}
112
113struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
114{
115 struct intel_crtc_state *crtc_state;
116
117 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
118
119 if (crtc_state)
120 intel_crtc_state_reset(crtc_state, crtc);
121
122 return crtc_state;
123}
124
125void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
126 struct intel_crtc *crtc)
127{
128 memset(crtc_state, 0, sizeof(*crtc_state));
129
130 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
131
132 crtc_state->cpu_transcoder = INVALID_TRANSCODER;
133 crtc_state->master_transcoder = INVALID_TRANSCODER;
134 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
135 crtc_state->scaler_state.scaler_id = -1;
136 crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
137}
138
139static struct intel_crtc *intel_crtc_alloc(void)
140{
141 struct intel_crtc_state *crtc_state;
142 struct intel_crtc *crtc;
143
144 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
145 if (!crtc)
146 return ERR_PTR(-ENOMEM);
147
148 crtc_state = intel_crtc_state_alloc(crtc);
149 if (!crtc_state) {
150 kfree(crtc);
151 return ERR_PTR(-ENOMEM);
152 }
153
154 crtc->base.state = &crtc_state->uapi;
155 crtc->config = crtc_state;
156
157 return crtc;
158}
159
160static void intel_crtc_free(struct intel_crtc *crtc)
161{
162 intel_crtc_destroy_state(&crtc->base, crtc->base.state);
163 kfree(crtc);
164}
165
166static void intel_crtc_destroy(struct drm_crtc *crtc)
167{
168 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
169
170 drm_crtc_cleanup(crtc);
171 kfree(intel_crtc);
172}
173
174static int intel_crtc_late_register(struct drm_crtc *crtc)
175{
176 intel_crtc_debugfs_add(crtc);
177 return 0;
178}
179
180#define INTEL_CRTC_FUNCS \
181 .set_config = drm_atomic_helper_set_config, \
182 .destroy = intel_crtc_destroy, \
183 .page_flip = drm_atomic_helper_page_flip, \
184 .atomic_duplicate_state = intel_crtc_duplicate_state, \
185 .atomic_destroy_state = intel_crtc_destroy_state, \
186 .set_crc_source = intel_crtc_set_crc_source, \
187 .verify_crc_source = intel_crtc_verify_crc_source, \
188 .get_crc_sources = intel_crtc_get_crc_sources, \
189 .late_register = intel_crtc_late_register
190
191static const struct drm_crtc_funcs bdw_crtc_funcs = {
192 INTEL_CRTC_FUNCS,
193
194 .get_vblank_counter = g4x_get_vblank_counter,
195 .enable_vblank = bdw_enable_vblank,
196 .disable_vblank = bdw_disable_vblank,
197 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
198};
199
200static const struct drm_crtc_funcs ilk_crtc_funcs = {
201 INTEL_CRTC_FUNCS,
202
203 .get_vblank_counter = g4x_get_vblank_counter,
204 .enable_vblank = ilk_enable_vblank,
205 .disable_vblank = ilk_disable_vblank,
206 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
207};
208
209static const struct drm_crtc_funcs g4x_crtc_funcs = {
210 INTEL_CRTC_FUNCS,
211
212 .get_vblank_counter = g4x_get_vblank_counter,
213 .enable_vblank = i965_enable_vblank,
214 .disable_vblank = i965_disable_vblank,
215 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
216};
217
218static const struct drm_crtc_funcs i965_crtc_funcs = {
219 INTEL_CRTC_FUNCS,
220
221 .get_vblank_counter = i915_get_vblank_counter,
222 .enable_vblank = i965_enable_vblank,
223 .disable_vblank = i965_disable_vblank,
224 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
225};
226
227static const struct drm_crtc_funcs i915gm_crtc_funcs = {
228 INTEL_CRTC_FUNCS,
229
230 .get_vblank_counter = i915_get_vblank_counter,
231 .enable_vblank = i915gm_enable_vblank,
232 .disable_vblank = i915gm_disable_vblank,
233 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
234};
235
236static const struct drm_crtc_funcs i915_crtc_funcs = {
237 INTEL_CRTC_FUNCS,
238
239 .get_vblank_counter = i915_get_vblank_counter,
240 .enable_vblank = i8xx_enable_vblank,
241 .disable_vblank = i8xx_disable_vblank,
242 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
243};
244
245static const struct drm_crtc_funcs i8xx_crtc_funcs = {
246 INTEL_CRTC_FUNCS,
247
248 /* no hw vblank counter */
249 .enable_vblank = i8xx_enable_vblank,
250 .disable_vblank = i8xx_disable_vblank,
251 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
252};
253
254int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
255{
256 struct intel_plane *primary, *cursor;
257 const struct drm_crtc_funcs *funcs;
258 struct intel_crtc *crtc;
259 int sprite, ret;
260
261 crtc = intel_crtc_alloc();
262 if (IS_ERR(crtc))
263 return PTR_ERR(crtc);
264
265 crtc->pipe = pipe;
266 crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
267
268 if (DISPLAY_VER(dev_priv) >= 9)
269 primary = skl_universal_plane_create(dev_priv, pipe,
270 PLANE_PRIMARY);
271 else
272 primary = intel_primary_plane_create(dev_priv, pipe);
273 if (IS_ERR(primary)) {
274 ret = PTR_ERR(primary);
275 goto fail;
276 }
277 crtc->plane_ids_mask |= BIT(primary->id);
278
279 for_each_sprite(dev_priv, pipe, sprite) {
280 struct intel_plane *plane;
281
282 if (DISPLAY_VER(dev_priv) >= 9)
283 plane = skl_universal_plane_create(dev_priv, pipe,
284 PLANE_SPRITE0 + sprite);
285 else
286 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
287 if (IS_ERR(plane)) {
288 ret = PTR_ERR(plane);
289 goto fail;
290 }
291 crtc->plane_ids_mask |= BIT(plane->id);
292 }
293
294 cursor = intel_cursor_plane_create(dev_priv, pipe);
295 if (IS_ERR(cursor)) {
296 ret = PTR_ERR(cursor);
297 goto fail;
298 }
299 crtc->plane_ids_mask |= BIT(cursor->id);
300
301 if (HAS_GMCH(dev_priv)) {
302 if (IS_CHERRYVIEW(dev_priv) ||
303 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
304 funcs = &g4x_crtc_funcs;
305 else if (DISPLAY_VER(dev_priv) == 4)
306 funcs = &i965_crtc_funcs;
307 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
308 funcs = &i915gm_crtc_funcs;
309 else if (DISPLAY_VER(dev_priv) == 3)
310 funcs = &i915_crtc_funcs;
311 else
312 funcs = &i8xx_crtc_funcs;
313 } else {
314 if (DISPLAY_VER(dev_priv) >= 8)
315 funcs = &bdw_crtc_funcs;
316 else
317 funcs = &ilk_crtc_funcs;
318 }
319
320 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
321 &primary->base, &cursor->base,
322 funcs, "pipe %c", pipe_name(pipe));
323 if (ret)
324 goto fail;
325
326 BUG_ON(pipe >= ARRAY_SIZE(dev_priv->pipe_to_crtc_mapping) ||
327 dev_priv->pipe_to_crtc_mapping[pipe] != NULL);
328 dev_priv->pipe_to_crtc_mapping[pipe] = crtc;
329
330 if (DISPLAY_VER(dev_priv) < 9) {
331 enum i9xx_plane_id i9xx_plane = primary->i9xx_plane;
332
333 BUG_ON(i9xx_plane >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
334 dev_priv->plane_to_crtc_mapping[i9xx_plane] != NULL);
335 dev_priv->plane_to_crtc_mapping[i9xx_plane] = crtc;
336 }
337
338 if (DISPLAY_VER(dev_priv) >= 11 || IS_CANNONLAKE(dev_priv))
339 drm_crtc_create_scaling_filter_property(&crtc->base,
340 BIT(DRM_SCALING_FILTER_DEFAULT) |
341 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
342
343 intel_color_init(crtc);
344
345 intel_crtc_crc_init(crtc);
346
347 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
348
349 return 0;
350
351fail:
352 intel_crtc_free(crtc);
353
354 return ret;
355}
356
357int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
358 int usecs)
359{
360 /* paranoia */
361 if (!adjusted_mode->crtc_htotal)
362 return 1;
363
364 return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
365 1000 * adjusted_mode->crtc_htotal);
366}
367
368static int intel_mode_vblank_start(const struct drm_display_mode *mode)
369{
370 int vblank_start = mode->crtc_vblank_start;
371
372 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
373 vblank_start = DIV_ROUND_UP(vblank_start, 2);
374
375 return vblank_start;
376}
377
378/**
379 * intel_pipe_update_start() - start update of a set of display registers
380 * @new_crtc_state: the new crtc state
381 *
382 * Mark the start of an update to pipe registers that should be updated
383 * atomically regarding vblank. If the next vblank will happens within
384 * the next 100 us, this function waits until the vblank passes.
385 *
386 * After a successful call to this function, interrupts will be disabled
387 * until a subsequent call to intel_pipe_update_end(). That is done to
388 * avoid random delays.
389 */
390void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state)
391{
392 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
393 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
394 const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode;
395 long timeout = msecs_to_jiffies_timeout(1);
396 int scanline, min, max, vblank_start;
397 wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
398 bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
399 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
400 DEFINE_WAIT(wait);
401
402 if (new_crtc_state->uapi.async_flip)
403 return;
404
405 if (new_crtc_state->vrr.enable)
406 vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state);
407 else
408 vblank_start = intel_mode_vblank_start(adjusted_mode);
409
410 /* FIXME needs to be calibrated sensibly */
411 min = vblank_start - intel_usecs_to_scanlines(adjusted_mode,
412 VBLANK_EVASION_TIME_US);
413 max = vblank_start - 1;
414
415 if (min <= 0 || max <= 0)
416 goto irq_disable;
417
418 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
419 goto irq_disable;
420
421 /*
422 * Wait for psr to idle out after enabling the VBL interrupts
423 * VBL interrupts will start the PSR exit and prevent a PSR
424 * re-entry as well.
425 */
426 intel_psr_wait_for_idle(new_crtc_state);
427
428 local_irq_disable();
429
430 crtc->debug.min_vbl = min;
431 crtc->debug.max_vbl = max;
432 trace_intel_pipe_update_start(crtc);
433
434 for (;;) {
435 /*
436 * prepare_to_wait() has a memory barrier, which guarantees
437 * other CPUs can see the task state update by the time we
438 * read the scanline.
439 */
440 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
441
442 scanline = intel_get_crtc_scanline(crtc);
443 if (scanline < min || scanline > max)
444 break;
445
446 if (!timeout) {
447 drm_err(&dev_priv->drm,
448 "Potential atomic update failure on pipe %c\n",
449 pipe_name(crtc->pipe));
450 break;
451 }
452
453 local_irq_enable();
454
455 timeout = schedule_timeout(timeout);
456
457 local_irq_disable();
458 }
459
460 finish_wait(wq, &wait);
461
462 drm_crtc_vblank_put(&crtc->base);
463
464 /*
465 * On VLV/CHV DSI the scanline counter would appear to
466 * increment approx. 1/3 of a scanline before start of vblank.
467 * The registers still get latched at start of vblank however.
468 * This means we must not write any registers on the first
469 * line of vblank (since not the whole line is actually in
470 * vblank). And unfortunately we can't use the interrupt to
471 * wait here since it will fire too soon. We could use the
472 * frame start interrupt instead since it will fire after the
473 * critical scanline, but that would require more changes
474 * in the interrupt code. So for now we'll just do the nasty
475 * thing and poll for the bad scanline to pass us by.
476 *
477 * FIXME figure out if BXT+ DSI suffers from this as well
478 */
479 while (need_vlv_dsi_wa && scanline == vblank_start)
480 scanline = intel_get_crtc_scanline(crtc);
481
482 crtc->debug.scanline_start = scanline;
483 crtc->debug.start_vbl_time = ktime_get();
484 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
485
486 trace_intel_pipe_update_vblank_evaded(crtc);
487 return;
488
489irq_disable:
490 local_irq_disable();
491}
492
493#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
494static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
495{
496 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
497 unsigned int h;
498
499 h = ilog2(delta >> 9);
500 if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
501 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
502 crtc->debug.vbl.times[h]++;
503
504 crtc->debug.vbl.sum += delta;
505 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
506 crtc->debug.vbl.min = delta;
507 if (delta > crtc->debug.vbl.max)
508 crtc->debug.vbl.max = delta;
509
510 if (delta > 1000 * VBLANK_EVASION_TIME_US) {
511 drm_dbg_kms(crtc->base.dev,
512 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
513 pipe_name(crtc->pipe),
514 div_u64(delta, 1000),
515 VBLANK_EVASION_TIME_US);
516 crtc->debug.vbl.over++;
517 }
518}
519#else
520static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
521#endif
522
523/**
524 * intel_pipe_update_end() - end update of a set of display registers
525 * @new_crtc_state: the new crtc state
526 *
527 * Mark the end of an update started with intel_pipe_update_start(). This
528 * re-enables interrupts and verifies the update was actually completed
529 * before a vblank.
530 */
531void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state)
532{
533 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
534 enum pipe pipe = crtc->pipe;
535 int scanline_end = intel_get_crtc_scanline(crtc);
536 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
537 ktime_t end_vbl_time = ktime_get();
538 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
539
540 if (new_crtc_state->uapi.async_flip)
541 return;
542
543 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
544
545 /*
546 * Incase of mipi dsi command mode, we need to set frame update
547 * request for every commit.
548 */
549 if (DISPLAY_VER(dev_priv) >= 11 &&
550 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
551 icl_dsi_frame_update(new_crtc_state);
552
553 /* We're still in the vblank-evade critical section, this can't race.
554 * Would be slightly nice to just grab the vblank count and arm the
555 * event outside of the critical section - the spinlock might spin for a
556 * while ... */
557 if (new_crtc_state->uapi.event) {
558 drm_WARN_ON(&dev_priv->drm,
559 drm_crtc_vblank_get(&crtc->base) != 0);
560
561 spin_lock(&crtc->base.dev->event_lock);
562 drm_crtc_arm_vblank_event(&crtc->base,
563 new_crtc_state->uapi.event);
564 spin_unlock(&crtc->base.dev->event_lock);
565
566 new_crtc_state->uapi.event = NULL;
567 }
568
569 local_irq_enable();
570
571 /* Send VRR Push to terminate Vblank */
572 intel_vrr_send_push(new_crtc_state);
573
574 if (intel_vgpu_active(dev_priv))
575 return;
576
577 if (crtc->debug.start_vbl_count &&
578 crtc->debug.start_vbl_count != end_vbl_count) {
579 drm_err(&dev_priv->drm,
580 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
581 pipe_name(pipe), crtc->debug.start_vbl_count,
582 end_vbl_count,
583 ktime_us_delta(end_vbl_time,
584 crtc->debug.start_vbl_time),
585 crtc->debug.min_vbl, crtc->debug.max_vbl,
586 crtc->debug.scanline_start, scanline_end);
587 }
588
589 dbg_vblank_evade(crtc, end_vbl_time);
590}
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5#include <linux/kernel.h>
6#include <linux/pm_qos.h>
7#include <linux/slab.h>
8
9#include <drm/drm_atomic_helper.h>
10#include <drm/drm_fourcc.h>
11#include <drm/drm_plane.h>
12#include <drm/drm_vblank_work.h>
13
14#include "i915_irq.h"
15#include "i915_vgpu.h"
16#include "i9xx_plane.h"
17#include "icl_dsi.h"
18#include "intel_atomic.h"
19#include "intel_atomic_plane.h"
20#include "intel_color.h"
21#include "intel_crtc.h"
22#include "intel_cursor.h"
23#include "intel_display_debugfs.h"
24#include "intel_display_trace.h"
25#include "intel_display_types.h"
26#include "intel_drrs.h"
27#include "intel_dsi.h"
28#include "intel_pipe_crc.h"
29#include "intel_psr.h"
30#include "intel_sprite.h"
31#include "intel_vrr.h"
32#include "skl_universal_plane.h"
33
34static void assert_vblank_disabled(struct drm_crtc *crtc)
35{
36 if (I915_STATE_WARN_ON(drm_crtc_vblank_get(crtc) == 0))
37 drm_crtc_vblank_put(crtc);
38}
39
40struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
41{
42 return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
43}
44
45struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
46 enum pipe pipe)
47{
48 struct intel_crtc *crtc;
49
50 for_each_intel_crtc(&i915->drm, crtc) {
51 if (crtc->pipe == pipe)
52 return crtc;
53 }
54
55 return NULL;
56}
57
58void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
59{
60 drm_crtc_wait_one_vblank(&crtc->base);
61}
62
63void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
64 enum pipe pipe)
65{
66 struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
67
68 if (crtc->active)
69 intel_crtc_wait_for_next_vblank(crtc);
70}
71
72u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
73{
74 struct drm_device *dev = crtc->base.dev;
75 struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
76
77 if (!crtc->active)
78 return 0;
79
80 if (!vblank->max_vblank_count)
81 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
82
83 return crtc->base.funcs->get_vblank_counter(&crtc->base);
84}
85
86u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
87{
88 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
89
90 /*
91 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
92 * have updated at the beginning of TE, if we want to use
93 * the hw counter, then we would find it updated in only
94 * the next TE, hence switching to sw counter.
95 */
96 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
97 I915_MODE_FLAG_DSI_USE_TE1))
98 return 0;
99
100 /*
101 * On i965gm the hardware frame counter reads
102 * zero when the TV encoder is enabled :(
103 */
104 if (IS_I965GM(dev_priv) &&
105 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
106 return 0;
107
108 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
109 return 0xffffffff; /* full 32 bit counter */
110 else if (DISPLAY_VER(dev_priv) >= 3)
111 return 0xffffff; /* only 24 bits of frame count */
112 else
113 return 0; /* Gen2 doesn't have a hardware frame counter */
114}
115
116void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
117{
118 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
119
120 assert_vblank_disabled(&crtc->base);
121 drm_crtc_set_max_vblank_count(&crtc->base,
122 intel_crtc_max_vblank_count(crtc_state));
123 drm_crtc_vblank_on(&crtc->base);
124
125 /*
126 * Should really happen exactly when we enable the pipe
127 * but we want the frame counters in the trace, and that
128 * requires vblank support on some platforms/outputs.
129 */
130 trace_intel_pipe_enable(crtc);
131}
132
133void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
134{
135 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
136
137 /*
138 * Should really happen exactly when we disable the pipe
139 * but we want the frame counters in the trace, and that
140 * requires vblank support on some platforms/outputs.
141 */
142 trace_intel_pipe_disable(crtc);
143
144 drm_crtc_vblank_off(&crtc->base);
145 assert_vblank_disabled(&crtc->base);
146}
147
148struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
149{
150 struct intel_crtc_state *crtc_state;
151
152 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
153
154 if (crtc_state)
155 intel_crtc_state_reset(crtc_state, crtc);
156
157 return crtc_state;
158}
159
160void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
161 struct intel_crtc *crtc)
162{
163 memset(crtc_state, 0, sizeof(*crtc_state));
164
165 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
166
167 crtc_state->cpu_transcoder = INVALID_TRANSCODER;
168 crtc_state->master_transcoder = INVALID_TRANSCODER;
169 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
170 crtc_state->scaler_state.scaler_id = -1;
171 crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
172}
173
174static struct intel_crtc *intel_crtc_alloc(void)
175{
176 struct intel_crtc_state *crtc_state;
177 struct intel_crtc *crtc;
178
179 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
180 if (!crtc)
181 return ERR_PTR(-ENOMEM);
182
183 crtc_state = intel_crtc_state_alloc(crtc);
184 if (!crtc_state) {
185 kfree(crtc);
186 return ERR_PTR(-ENOMEM);
187 }
188
189 crtc->base.state = &crtc_state->uapi;
190 crtc->config = crtc_state;
191
192 return crtc;
193}
194
195static void intel_crtc_free(struct intel_crtc *crtc)
196{
197 intel_crtc_destroy_state(&crtc->base, crtc->base.state);
198 kfree(crtc);
199}
200
201static void intel_crtc_destroy(struct drm_crtc *_crtc)
202{
203 struct intel_crtc *crtc = to_intel_crtc(_crtc);
204
205 cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
206
207 drm_crtc_cleanup(&crtc->base);
208 kfree(crtc);
209}
210
211static int intel_crtc_late_register(struct drm_crtc *crtc)
212{
213 intel_crtc_debugfs_add(crtc);
214 return 0;
215}
216
217#define INTEL_CRTC_FUNCS \
218 .set_config = drm_atomic_helper_set_config, \
219 .destroy = intel_crtc_destroy, \
220 .page_flip = drm_atomic_helper_page_flip, \
221 .atomic_duplicate_state = intel_crtc_duplicate_state, \
222 .atomic_destroy_state = intel_crtc_destroy_state, \
223 .set_crc_source = intel_crtc_set_crc_source, \
224 .verify_crc_source = intel_crtc_verify_crc_source, \
225 .get_crc_sources = intel_crtc_get_crc_sources, \
226 .late_register = intel_crtc_late_register
227
228static const struct drm_crtc_funcs bdw_crtc_funcs = {
229 INTEL_CRTC_FUNCS,
230
231 .get_vblank_counter = g4x_get_vblank_counter,
232 .enable_vblank = bdw_enable_vblank,
233 .disable_vblank = bdw_disable_vblank,
234 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
235};
236
237static const struct drm_crtc_funcs ilk_crtc_funcs = {
238 INTEL_CRTC_FUNCS,
239
240 .get_vblank_counter = g4x_get_vblank_counter,
241 .enable_vblank = ilk_enable_vblank,
242 .disable_vblank = ilk_disable_vblank,
243 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
244};
245
246static const struct drm_crtc_funcs g4x_crtc_funcs = {
247 INTEL_CRTC_FUNCS,
248
249 .get_vblank_counter = g4x_get_vblank_counter,
250 .enable_vblank = i965_enable_vblank,
251 .disable_vblank = i965_disable_vblank,
252 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
253};
254
255static const struct drm_crtc_funcs i965_crtc_funcs = {
256 INTEL_CRTC_FUNCS,
257
258 .get_vblank_counter = i915_get_vblank_counter,
259 .enable_vblank = i965_enable_vblank,
260 .disable_vblank = i965_disable_vblank,
261 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
262};
263
264static const struct drm_crtc_funcs i915gm_crtc_funcs = {
265 INTEL_CRTC_FUNCS,
266
267 .get_vblank_counter = i915_get_vblank_counter,
268 .enable_vblank = i915gm_enable_vblank,
269 .disable_vblank = i915gm_disable_vblank,
270 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
271};
272
273static const struct drm_crtc_funcs i915_crtc_funcs = {
274 INTEL_CRTC_FUNCS,
275
276 .get_vblank_counter = i915_get_vblank_counter,
277 .enable_vblank = i8xx_enable_vblank,
278 .disable_vblank = i8xx_disable_vblank,
279 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
280};
281
282static const struct drm_crtc_funcs i8xx_crtc_funcs = {
283 INTEL_CRTC_FUNCS,
284
285 /* no hw vblank counter */
286 .enable_vblank = i8xx_enable_vblank,
287 .disable_vblank = i8xx_disable_vblank,
288 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
289};
290
291int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
292{
293 struct intel_plane *primary, *cursor;
294 const struct drm_crtc_funcs *funcs;
295 struct intel_crtc *crtc;
296 int sprite, ret;
297
298 crtc = intel_crtc_alloc();
299 if (IS_ERR(crtc))
300 return PTR_ERR(crtc);
301
302 crtc->pipe = pipe;
303 crtc->num_scalers = RUNTIME_INFO(dev_priv)->num_scalers[pipe];
304
305 if (DISPLAY_VER(dev_priv) >= 9)
306 primary = skl_universal_plane_create(dev_priv, pipe,
307 PLANE_PRIMARY);
308 else
309 primary = intel_primary_plane_create(dev_priv, pipe);
310 if (IS_ERR(primary)) {
311 ret = PTR_ERR(primary);
312 goto fail;
313 }
314 crtc->plane_ids_mask |= BIT(primary->id);
315
316 for_each_sprite(dev_priv, pipe, sprite) {
317 struct intel_plane *plane;
318
319 if (DISPLAY_VER(dev_priv) >= 9)
320 plane = skl_universal_plane_create(dev_priv, pipe,
321 PLANE_SPRITE0 + sprite);
322 else
323 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
324 if (IS_ERR(plane)) {
325 ret = PTR_ERR(plane);
326 goto fail;
327 }
328 crtc->plane_ids_mask |= BIT(plane->id);
329 }
330
331 cursor = intel_cursor_plane_create(dev_priv, pipe);
332 if (IS_ERR(cursor)) {
333 ret = PTR_ERR(cursor);
334 goto fail;
335 }
336 crtc->plane_ids_mask |= BIT(cursor->id);
337
338 if (HAS_GMCH(dev_priv)) {
339 if (IS_CHERRYVIEW(dev_priv) ||
340 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
341 funcs = &g4x_crtc_funcs;
342 else if (DISPLAY_VER(dev_priv) == 4)
343 funcs = &i965_crtc_funcs;
344 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
345 funcs = &i915gm_crtc_funcs;
346 else if (DISPLAY_VER(dev_priv) == 3)
347 funcs = &i915_crtc_funcs;
348 else
349 funcs = &i8xx_crtc_funcs;
350 } else {
351 if (DISPLAY_VER(dev_priv) >= 8)
352 funcs = &bdw_crtc_funcs;
353 else
354 funcs = &ilk_crtc_funcs;
355 }
356
357 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
358 &primary->base, &cursor->base,
359 funcs, "pipe %c", pipe_name(pipe));
360 if (ret)
361 goto fail;
362
363 if (DISPLAY_VER(dev_priv) >= 11)
364 drm_crtc_create_scaling_filter_property(&crtc->base,
365 BIT(DRM_SCALING_FILTER_DEFAULT) |
366 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
367
368 intel_color_crtc_init(crtc);
369 intel_drrs_crtc_init(crtc);
370 intel_crtc_crc_init(crtc);
371
372 cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
373
374 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
375
376 return 0;
377
378fail:
379 intel_crtc_free(crtc);
380
381 return ret;
382}
383
384static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
385{
386 return crtc_state->hw.active &&
387 !intel_crtc_needs_modeset(crtc_state) &&
388 !crtc_state->preload_luts &&
389 intel_crtc_needs_color_update(crtc_state);
390}
391
392static void intel_crtc_vblank_work(struct kthread_work *base)
393{
394 struct drm_vblank_work *work = to_drm_vblank_work(base);
395 struct intel_crtc_state *crtc_state =
396 container_of(work, typeof(*crtc_state), vblank_work);
397 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
398
399 trace_intel_crtc_vblank_work_start(crtc);
400
401 intel_color_load_luts(crtc_state);
402
403 if (crtc_state->uapi.event) {
404 spin_lock_irq(&crtc->base.dev->event_lock);
405 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
406 crtc_state->uapi.event = NULL;
407 spin_unlock_irq(&crtc->base.dev->event_lock);
408 }
409
410 trace_intel_crtc_vblank_work_end(crtc);
411}
412
413static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
414{
415 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
416
417 drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
418 intel_crtc_vblank_work);
419 /*
420 * Interrupt latency is critical for getting the vblank
421 * work executed as early as possible during the vblank.
422 */
423 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
424}
425
426void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
427{
428 struct intel_crtc_state *crtc_state;
429 struct intel_crtc *crtc;
430 int i;
431
432 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
433 if (!intel_crtc_needs_vblank_work(crtc_state))
434 continue;
435
436 drm_vblank_work_flush(&crtc_state->vblank_work);
437 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
438 PM_QOS_DEFAULT_VALUE);
439 }
440}
441
442int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
443 int usecs)
444{
445 /* paranoia */
446 if (!adjusted_mode->crtc_htotal)
447 return 1;
448
449 return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
450 1000 * adjusted_mode->crtc_htotal);
451}
452
453static int intel_mode_vblank_start(const struct drm_display_mode *mode)
454{
455 int vblank_start = mode->crtc_vblank_start;
456
457 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
458 vblank_start = DIV_ROUND_UP(vblank_start, 2);
459
460 return vblank_start;
461}
462
463/**
464 * intel_pipe_update_start() - start update of a set of display registers
465 * @new_crtc_state: the new crtc state
466 *
467 * Mark the start of an update to pipe registers that should be updated
468 * atomically regarding vblank. If the next vblank will happens within
469 * the next 100 us, this function waits until the vblank passes.
470 *
471 * After a successful call to this function, interrupts will be disabled
472 * until a subsequent call to intel_pipe_update_end(). That is done to
473 * avoid random delays.
474 */
475void intel_pipe_update_start(struct intel_crtc_state *new_crtc_state)
476{
477 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
478 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
479 const struct drm_display_mode *adjusted_mode = &new_crtc_state->hw.adjusted_mode;
480 long timeout = msecs_to_jiffies_timeout(1);
481 int scanline, min, max, vblank_start;
482 wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
483 bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
484 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
485 DEFINE_WAIT(wait);
486
487 intel_psr_lock(new_crtc_state);
488
489 if (new_crtc_state->do_async_flip)
490 return;
491
492 if (intel_crtc_needs_vblank_work(new_crtc_state))
493 intel_crtc_vblank_work_init(new_crtc_state);
494
495 if (new_crtc_state->vrr.enable) {
496 if (intel_vrr_is_push_sent(new_crtc_state))
497 vblank_start = intel_vrr_vmin_vblank_start(new_crtc_state);
498 else
499 vblank_start = intel_vrr_vmax_vblank_start(new_crtc_state);
500 } else {
501 vblank_start = intel_mode_vblank_start(adjusted_mode);
502 }
503
504 /* FIXME needs to be calibrated sensibly */
505 min = vblank_start - intel_usecs_to_scanlines(adjusted_mode,
506 VBLANK_EVASION_TIME_US);
507 max = vblank_start - 1;
508
509 if (min <= 0 || max <= 0)
510 goto irq_disable;
511
512 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
513 goto irq_disable;
514
515 /*
516 * Wait for psr to idle out after enabling the VBL interrupts
517 * VBL interrupts will start the PSR exit and prevent a PSR
518 * re-entry as well.
519 */
520 intel_psr_wait_for_idle_locked(new_crtc_state);
521
522 local_irq_disable();
523
524 crtc->debug.min_vbl = min;
525 crtc->debug.max_vbl = max;
526 trace_intel_pipe_update_start(crtc);
527
528 for (;;) {
529 /*
530 * prepare_to_wait() has a memory barrier, which guarantees
531 * other CPUs can see the task state update by the time we
532 * read the scanline.
533 */
534 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
535
536 scanline = intel_get_crtc_scanline(crtc);
537 if (scanline < min || scanline > max)
538 break;
539
540 if (!timeout) {
541 drm_err(&dev_priv->drm,
542 "Potential atomic update failure on pipe %c\n",
543 pipe_name(crtc->pipe));
544 break;
545 }
546
547 local_irq_enable();
548
549 timeout = schedule_timeout(timeout);
550
551 local_irq_disable();
552 }
553
554 finish_wait(wq, &wait);
555
556 drm_crtc_vblank_put(&crtc->base);
557
558 /*
559 * On VLV/CHV DSI the scanline counter would appear to
560 * increment approx. 1/3 of a scanline before start of vblank.
561 * The registers still get latched at start of vblank however.
562 * This means we must not write any registers on the first
563 * line of vblank (since not the whole line is actually in
564 * vblank). And unfortunately we can't use the interrupt to
565 * wait here since it will fire too soon. We could use the
566 * frame start interrupt instead since it will fire after the
567 * critical scanline, but that would require more changes
568 * in the interrupt code. So for now we'll just do the nasty
569 * thing and poll for the bad scanline to pass us by.
570 *
571 * FIXME figure out if BXT+ DSI suffers from this as well
572 */
573 while (need_vlv_dsi_wa && scanline == vblank_start)
574 scanline = intel_get_crtc_scanline(crtc);
575
576 crtc->debug.scanline_start = scanline;
577 crtc->debug.start_vbl_time = ktime_get();
578 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
579
580 trace_intel_pipe_update_vblank_evaded(crtc);
581 return;
582
583irq_disable:
584 local_irq_disable();
585}
586
587#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
588static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
589{
590 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
591 unsigned int h;
592
593 h = ilog2(delta >> 9);
594 if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
595 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
596 crtc->debug.vbl.times[h]++;
597
598 crtc->debug.vbl.sum += delta;
599 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
600 crtc->debug.vbl.min = delta;
601 if (delta > crtc->debug.vbl.max)
602 crtc->debug.vbl.max = delta;
603
604 if (delta > 1000 * VBLANK_EVASION_TIME_US) {
605 drm_dbg_kms(crtc->base.dev,
606 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
607 pipe_name(crtc->pipe),
608 div_u64(delta, 1000),
609 VBLANK_EVASION_TIME_US);
610 crtc->debug.vbl.over++;
611 }
612}
613#else
614static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
615#endif
616
617/**
618 * intel_pipe_update_end() - end update of a set of display registers
619 * @new_crtc_state: the new crtc state
620 *
621 * Mark the end of an update started with intel_pipe_update_start(). This
622 * re-enables interrupts and verifies the update was actually completed
623 * before a vblank.
624 */
625void intel_pipe_update_end(struct intel_crtc_state *new_crtc_state)
626{
627 struct intel_crtc *crtc = to_intel_crtc(new_crtc_state->uapi.crtc);
628 enum pipe pipe = crtc->pipe;
629 int scanline_end = intel_get_crtc_scanline(crtc);
630 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
631 ktime_t end_vbl_time = ktime_get();
632 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
633
634 intel_psr_unlock(new_crtc_state);
635
636 if (new_crtc_state->do_async_flip)
637 return;
638
639 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
640
641 /*
642 * Incase of mipi dsi command mode, we need to set frame update
643 * request for every commit.
644 */
645 if (DISPLAY_VER(dev_priv) >= 11 &&
646 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
647 icl_dsi_frame_update(new_crtc_state);
648
649 /* We're still in the vblank-evade critical section, this can't race.
650 * Would be slightly nice to just grab the vblank count and arm the
651 * event outside of the critical section - the spinlock might spin for a
652 * while ... */
653 if (intel_crtc_needs_vblank_work(new_crtc_state)) {
654 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
655 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
656 false);
657 } else if (new_crtc_state->uapi.event) {
658 drm_WARN_ON(&dev_priv->drm,
659 drm_crtc_vblank_get(&crtc->base) != 0);
660
661 spin_lock(&crtc->base.dev->event_lock);
662 drm_crtc_arm_vblank_event(&crtc->base,
663 new_crtc_state->uapi.event);
664 spin_unlock(&crtc->base.dev->event_lock);
665
666 new_crtc_state->uapi.event = NULL;
667 }
668
669 /*
670 * Send VRR Push to terminate Vblank. If we are already in vblank
671 * this has to be done _after_ sampling the frame counter, as
672 * otherwise the push would immediately terminate the vblank and
673 * the sampled frame counter would correspond to the next frame
674 * instead of the current frame.
675 *
676 * There is a tiny race here (iff vblank evasion failed us) where
677 * we might sample the frame counter just before vmax vblank start
678 * but the push would be sent just after it. That would cause the
679 * push to affect the next frame instead of the current frame,
680 * which would cause the next frame to terminate already at vmin
681 * vblank start instead of vmax vblank start.
682 */
683 intel_vrr_send_push(new_crtc_state);
684
685 local_irq_enable();
686
687 if (intel_vgpu_active(dev_priv))
688 return;
689
690 if (crtc->debug.start_vbl_count &&
691 crtc->debug.start_vbl_count != end_vbl_count) {
692 drm_err(&dev_priv->drm,
693 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
694 pipe_name(pipe), crtc->debug.start_vbl_count,
695 end_vbl_count,
696 ktime_us_delta(end_vbl_time,
697 crtc->debug.start_vbl_time),
698 crtc->debug.min_vbl, crtc->debug.max_vbl,
699 crtc->debug.scanline_start, scanline_end);
700 }
701
702 dbg_vblank_evade(crtc, end_vbl_time);
703}