Loading...
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5#include <linux/kernel.h>
6#include <linux/pm_qos.h>
7#include <linux/slab.h>
8
9#include <drm/drm_atomic_helper.h>
10#include <drm/drm_fourcc.h>
11#include <drm/drm_plane.h>
12#include <drm/drm_vblank.h>
13#include <drm/drm_vblank_work.h>
14
15#include "i915_vgpu.h"
16#include "i9xx_plane.h"
17#include "icl_dsi.h"
18#include "intel_atomic.h"
19#include "intel_atomic_plane.h"
20#include "intel_color.h"
21#include "intel_crtc.h"
22#include "intel_cursor.h"
23#include "intel_display_debugfs.h"
24#include "intel_display_irq.h"
25#include "intel_display_trace.h"
26#include "intel_display_types.h"
27#include "intel_drrs.h"
28#include "intel_dsi.h"
29#include "intel_fifo_underrun.h"
30#include "intel_pipe_crc.h"
31#include "intel_psr.h"
32#include "intel_sprite.h"
33#include "intel_vblank.h"
34#include "intel_vrr.h"
35#include "skl_universal_plane.h"
36
37static void assert_vblank_disabled(struct drm_crtc *crtc)
38{
39 struct intel_display *display = to_intel_display(crtc->dev);
40
41 if (INTEL_DISPLAY_STATE_WARN(display, drm_crtc_vblank_get(crtc) == 0,
42 "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
43 crtc->base.id, crtc->name))
44 drm_crtc_vblank_put(crtc);
45}
46
47struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
48{
49 return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
50}
51
52struct intel_crtc *intel_crtc_for_pipe(struct intel_display *display,
53 enum pipe pipe)
54{
55 struct intel_crtc *crtc;
56
57 for_each_intel_crtc(display->drm, crtc) {
58 if (crtc->pipe == pipe)
59 return crtc;
60 }
61
62 return NULL;
63}
64
65void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
66{
67 drm_crtc_wait_one_vblank(&crtc->base);
68}
69
70void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
71 enum pipe pipe)
72{
73 struct intel_display *display = &i915->display;
74 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
75
76 if (crtc->active)
77 intel_crtc_wait_for_next_vblank(crtc);
78}
79
80u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
81{
82 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
83
84 if (!crtc->active)
85 return 0;
86
87 if (!vblank->max_vblank_count)
88 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
89
90 return crtc->base.funcs->get_vblank_counter(&crtc->base);
91}
92
93u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
94{
95 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
96
97 /*
98 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
99 * have updated at the beginning of TE, if we want to use
100 * the hw counter, then we would find it updated in only
101 * the next TE, hence switching to sw counter.
102 */
103 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
104 I915_MODE_FLAG_DSI_USE_TE1))
105 return 0;
106
107 /*
108 * On i965gm the hardware frame counter reads
109 * zero when the TV encoder is enabled :(
110 */
111 if (IS_I965GM(dev_priv) &&
112 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
113 return 0;
114
115 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
116 return 0xffffffff; /* full 32 bit counter */
117 else if (DISPLAY_VER(dev_priv) >= 3)
118 return 0xffffff; /* only 24 bits of frame count */
119 else
120 return 0; /* Gen2 doesn't have a hardware frame counter */
121}
122
123void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
124{
125 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
126
127 crtc->block_dc_for_vblank = intel_psr_needs_block_dc_vblank(crtc_state);
128
129 assert_vblank_disabled(&crtc->base);
130 drm_crtc_set_max_vblank_count(&crtc->base,
131 intel_crtc_max_vblank_count(crtc_state));
132 drm_crtc_vblank_on(&crtc->base);
133
134 /*
135 * Should really happen exactly when we enable the pipe
136 * but we want the frame counters in the trace, and that
137 * requires vblank support on some platforms/outputs.
138 */
139 trace_intel_pipe_enable(crtc);
140}
141
142void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
143{
144 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
145 struct intel_display *display = to_intel_display(crtc);
146
147 /*
148 * Should really happen exactly when we disable the pipe
149 * but we want the frame counters in the trace, and that
150 * requires vblank support on some platforms/outputs.
151 */
152 trace_intel_pipe_disable(crtc);
153
154 drm_crtc_vblank_off(&crtc->base);
155 assert_vblank_disabled(&crtc->base);
156
157 crtc->block_dc_for_vblank = false;
158
159 flush_work(&display->irq.vblank_dc_work);
160}
161
162struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
163{
164 struct intel_crtc_state *crtc_state;
165
166 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
167
168 if (crtc_state)
169 intel_crtc_state_reset(crtc_state, crtc);
170
171 return crtc_state;
172}
173
174void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
175 struct intel_crtc *crtc)
176{
177 memset(crtc_state, 0, sizeof(*crtc_state));
178
179 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
180
181 crtc_state->cpu_transcoder = INVALID_TRANSCODER;
182 crtc_state->master_transcoder = INVALID_TRANSCODER;
183 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
184 crtc_state->scaler_state.scaler_id = -1;
185 crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
186 crtc_state->max_link_bpp_x16 = INT_MAX;
187}
188
189static struct intel_crtc *intel_crtc_alloc(void)
190{
191 struct intel_crtc_state *crtc_state;
192 struct intel_crtc *crtc;
193
194 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
195 if (!crtc)
196 return ERR_PTR(-ENOMEM);
197
198 crtc_state = intel_crtc_state_alloc(crtc);
199 if (!crtc_state) {
200 kfree(crtc);
201 return ERR_PTR(-ENOMEM);
202 }
203
204 crtc->base.state = &crtc_state->uapi;
205 crtc->config = crtc_state;
206
207 return crtc;
208}
209
210static void intel_crtc_free(struct intel_crtc *crtc)
211{
212 intel_crtc_destroy_state(&crtc->base, crtc->base.state);
213 kfree(crtc);
214}
215
216static void intel_crtc_destroy(struct drm_crtc *_crtc)
217{
218 struct intel_crtc *crtc = to_intel_crtc(_crtc);
219
220 cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
221
222 drm_crtc_cleanup(&crtc->base);
223 kfree(crtc);
224}
225
226static int intel_crtc_late_register(struct drm_crtc *crtc)
227{
228 intel_crtc_debugfs_add(to_intel_crtc(crtc));
229 return 0;
230}
231
232#define INTEL_CRTC_FUNCS \
233 .set_config = drm_atomic_helper_set_config, \
234 .destroy = intel_crtc_destroy, \
235 .page_flip = drm_atomic_helper_page_flip, \
236 .atomic_duplicate_state = intel_crtc_duplicate_state, \
237 .atomic_destroy_state = intel_crtc_destroy_state, \
238 .set_crc_source = intel_crtc_set_crc_source, \
239 .verify_crc_source = intel_crtc_verify_crc_source, \
240 .get_crc_sources = intel_crtc_get_crc_sources, \
241 .late_register = intel_crtc_late_register
242
243static const struct drm_crtc_funcs bdw_crtc_funcs = {
244 INTEL_CRTC_FUNCS,
245
246 .get_vblank_counter = g4x_get_vblank_counter,
247 .enable_vblank = bdw_enable_vblank,
248 .disable_vblank = bdw_disable_vblank,
249 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
250};
251
252static const struct drm_crtc_funcs ilk_crtc_funcs = {
253 INTEL_CRTC_FUNCS,
254
255 .get_vblank_counter = g4x_get_vblank_counter,
256 .enable_vblank = ilk_enable_vblank,
257 .disable_vblank = ilk_disable_vblank,
258 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
259};
260
261static const struct drm_crtc_funcs g4x_crtc_funcs = {
262 INTEL_CRTC_FUNCS,
263
264 .get_vblank_counter = g4x_get_vblank_counter,
265 .enable_vblank = i965_enable_vblank,
266 .disable_vblank = i965_disable_vblank,
267 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
268};
269
270static const struct drm_crtc_funcs i965_crtc_funcs = {
271 INTEL_CRTC_FUNCS,
272
273 .get_vblank_counter = i915_get_vblank_counter,
274 .enable_vblank = i965_enable_vblank,
275 .disable_vblank = i965_disable_vblank,
276 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
277};
278
279static const struct drm_crtc_funcs i915gm_crtc_funcs = {
280 INTEL_CRTC_FUNCS,
281
282 .get_vblank_counter = i915_get_vblank_counter,
283 .enable_vblank = i915gm_enable_vblank,
284 .disable_vblank = i915gm_disable_vblank,
285 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
286};
287
288static const struct drm_crtc_funcs i915_crtc_funcs = {
289 INTEL_CRTC_FUNCS,
290
291 .get_vblank_counter = i915_get_vblank_counter,
292 .enable_vblank = i8xx_enable_vblank,
293 .disable_vblank = i8xx_disable_vblank,
294 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
295};
296
297static const struct drm_crtc_funcs i8xx_crtc_funcs = {
298 INTEL_CRTC_FUNCS,
299
300 /* no hw vblank counter */
301 .enable_vblank = i8xx_enable_vblank,
302 .disable_vblank = i8xx_disable_vblank,
303 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
304};
305
306int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
307{
308 struct intel_plane *primary, *cursor;
309 const struct drm_crtc_funcs *funcs;
310 struct intel_crtc *crtc;
311 int sprite, ret;
312
313 crtc = intel_crtc_alloc();
314 if (IS_ERR(crtc))
315 return PTR_ERR(crtc);
316
317 crtc->pipe = pipe;
318 crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe];
319
320 if (DISPLAY_VER(dev_priv) >= 9)
321 primary = skl_universal_plane_create(dev_priv, pipe, PLANE_1);
322 else
323 primary = intel_primary_plane_create(dev_priv, pipe);
324 if (IS_ERR(primary)) {
325 ret = PTR_ERR(primary);
326 goto fail;
327 }
328 crtc->plane_ids_mask |= BIT(primary->id);
329
330 intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
331
332 for_each_sprite(dev_priv, pipe, sprite) {
333 struct intel_plane *plane;
334
335 if (DISPLAY_VER(dev_priv) >= 9)
336 plane = skl_universal_plane_create(dev_priv, pipe, PLANE_2 + sprite);
337 else
338 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
339 if (IS_ERR(plane)) {
340 ret = PTR_ERR(plane);
341 goto fail;
342 }
343 crtc->plane_ids_mask |= BIT(plane->id);
344 }
345
346 cursor = intel_cursor_plane_create(dev_priv, pipe);
347 if (IS_ERR(cursor)) {
348 ret = PTR_ERR(cursor);
349 goto fail;
350 }
351 crtc->plane_ids_mask |= BIT(cursor->id);
352
353 if (HAS_GMCH(dev_priv)) {
354 if (IS_CHERRYVIEW(dev_priv) ||
355 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
356 funcs = &g4x_crtc_funcs;
357 else if (DISPLAY_VER(dev_priv) == 4)
358 funcs = &i965_crtc_funcs;
359 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
360 funcs = &i915gm_crtc_funcs;
361 else if (DISPLAY_VER(dev_priv) == 3)
362 funcs = &i915_crtc_funcs;
363 else
364 funcs = &i8xx_crtc_funcs;
365 } else {
366 if (DISPLAY_VER(dev_priv) >= 8)
367 funcs = &bdw_crtc_funcs;
368 else
369 funcs = &ilk_crtc_funcs;
370 }
371
372 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
373 &primary->base, &cursor->base,
374 funcs, "pipe %c", pipe_name(pipe));
375 if (ret)
376 goto fail;
377
378 if (DISPLAY_VER(dev_priv) >= 11)
379 drm_crtc_create_scaling_filter_property(&crtc->base,
380 BIT(DRM_SCALING_FILTER_DEFAULT) |
381 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
382
383 intel_color_crtc_init(crtc);
384 intel_drrs_crtc_init(crtc);
385 intel_crtc_crc_init(crtc);
386
387 cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
388
389 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
390
391 return 0;
392
393fail:
394 intel_crtc_free(crtc);
395
396 return ret;
397}
398
399int intel_crtc_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
400 struct drm_file *file)
401{
402 struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
403 struct drm_crtc *drm_crtc;
404 struct intel_crtc *crtc;
405
406 drm_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
407 if (!drm_crtc)
408 return -ENOENT;
409
410 crtc = to_intel_crtc(drm_crtc);
411 pipe_from_crtc_id->pipe = crtc->pipe;
412
413 return 0;
414}
415
416static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
417{
418 return crtc_state->hw.active &&
419 !crtc_state->preload_luts &&
420 !intel_crtc_needs_modeset(crtc_state) &&
421 intel_crtc_needs_color_update(crtc_state) &&
422 !intel_color_uses_dsb(crtc_state) &&
423 !crtc_state->use_dsb;
424}
425
426static void intel_crtc_vblank_work(struct kthread_work *base)
427{
428 struct drm_vblank_work *work = to_drm_vblank_work(base);
429 struct intel_crtc_state *crtc_state =
430 container_of(work, typeof(*crtc_state), vblank_work);
431 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
432
433 trace_intel_crtc_vblank_work_start(crtc);
434
435 intel_color_load_luts(crtc_state);
436
437 if (crtc_state->uapi.event) {
438 spin_lock_irq(&crtc->base.dev->event_lock);
439 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
440 spin_unlock_irq(&crtc->base.dev->event_lock);
441 crtc_state->uapi.event = NULL;
442 }
443
444 trace_intel_crtc_vblank_work_end(crtc);
445}
446
447static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
448{
449 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
450
451 drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
452 intel_crtc_vblank_work);
453 /*
454 * Interrupt latency is critical for getting the vblank
455 * work executed as early as possible during the vblank.
456 */
457 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
458}
459
460void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
461{
462 struct intel_crtc_state *crtc_state;
463 struct intel_crtc *crtc;
464 int i;
465
466 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
467 if (!intel_crtc_needs_vblank_work(crtc_state))
468 continue;
469
470 drm_vblank_work_flush(&crtc_state->vblank_work);
471 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
472 PM_QOS_DEFAULT_VALUE);
473 }
474}
475
476int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
477 int usecs)
478{
479 /* paranoia */
480 if (!adjusted_mode->crtc_htotal)
481 return 1;
482
483 return DIV_ROUND_UP_ULL(mul_u32_u32(usecs, adjusted_mode->crtc_clock),
484 1000 * adjusted_mode->crtc_htotal);
485}
486
487int intel_scanlines_to_usecs(const struct drm_display_mode *adjusted_mode,
488 int scanlines)
489{
490 /* paranoia */
491 if (!adjusted_mode->crtc_clock)
492 return 1;
493
494 return DIV_ROUND_UP_ULL(mul_u32_u32(scanlines, adjusted_mode->crtc_htotal * 1000),
495 adjusted_mode->crtc_clock);
496}
497
498/**
499 * intel_pipe_update_start() - start update of a set of display registers
500 * @state: the atomic state
501 * @crtc: the crtc
502 *
503 * Mark the start of an update to pipe registers that should be updated
504 * atomically regarding vblank. If the next vblank will happens within
505 * the next 100 us, this function waits until the vblank passes.
506 *
507 * After a successful call to this function, interrupts will be disabled
508 * until a subsequent call to intel_pipe_update_end(). That is done to
509 * avoid random delays.
510 */
511void intel_pipe_update_start(struct intel_atomic_state *state,
512 struct intel_crtc *crtc)
513{
514 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
515 const struct intel_crtc_state *old_crtc_state =
516 intel_atomic_get_old_crtc_state(state, crtc);
517 struct intel_crtc_state *new_crtc_state =
518 intel_atomic_get_new_crtc_state(state, crtc);
519 struct intel_vblank_evade_ctx evade;
520 int scanline;
521
522 intel_psr_lock(new_crtc_state);
523
524 if (new_crtc_state->do_async_flip) {
525 intel_crtc_prepare_vblank_event(new_crtc_state,
526 &crtc->flip_done_event);
527 return;
528 }
529
530 if (intel_crtc_needs_vblank_work(new_crtc_state))
531 intel_crtc_vblank_work_init(new_crtc_state);
532
533 if (state->base.legacy_cursor_update) {
534 struct intel_plane *plane;
535 struct intel_plane_state *old_plane_state, *new_plane_state;
536 int i;
537
538 for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
539 new_plane_state, i) {
540 if (old_plane_state->uapi.crtc == &crtc->base)
541 intel_plane_init_cursor_vblank_work(old_plane_state,
542 new_plane_state);
543 }
544 }
545
546 intel_vblank_evade_init(old_crtc_state, new_crtc_state, &evade);
547
548 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
549 goto irq_disable;
550
551 /*
552 * Wait for psr to idle out after enabling the VBL interrupts
553 * VBL interrupts will start the PSR exit and prevent a PSR
554 * re-entry as well.
555 */
556 intel_psr_wait_for_idle_locked(new_crtc_state);
557
558 local_irq_disable();
559
560 crtc->debug.min_vbl = evade.min;
561 crtc->debug.max_vbl = evade.max;
562 trace_intel_pipe_update_start(crtc);
563
564 scanline = intel_vblank_evade(&evade);
565
566 drm_crtc_vblank_put(&crtc->base);
567
568 crtc->debug.scanline_start = scanline;
569 crtc->debug.start_vbl_time = ktime_get();
570 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
571
572 trace_intel_pipe_update_vblank_evaded(crtc);
573 return;
574
575irq_disable:
576 local_irq_disable();
577}
578
579#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
580static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
581{
582 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
583 unsigned int h;
584
585 h = ilog2(delta >> 9);
586 if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
587 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
588 crtc->debug.vbl.times[h]++;
589
590 crtc->debug.vbl.sum += delta;
591 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
592 crtc->debug.vbl.min = delta;
593 if (delta > crtc->debug.vbl.max)
594 crtc->debug.vbl.max = delta;
595
596 if (delta > 1000 * VBLANK_EVASION_TIME_US) {
597 drm_dbg_kms(crtc->base.dev,
598 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
599 pipe_name(crtc->pipe),
600 div_u64(delta, 1000),
601 VBLANK_EVASION_TIME_US);
602 crtc->debug.vbl.over++;
603 }
604}
605#else
606static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
607#endif
608
609void intel_crtc_arm_vblank_event(struct intel_crtc_state *crtc_state)
610{
611 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
612 unsigned long irqflags;
613
614 if (!crtc_state->uapi.event)
615 return;
616
617 drm_WARN_ON(crtc->base.dev, drm_crtc_vblank_get(&crtc->base) != 0);
618
619 spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
620 drm_crtc_arm_vblank_event(&crtc->base, crtc_state->uapi.event);
621 spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
622
623 crtc_state->uapi.event = NULL;
624}
625
626void intel_crtc_prepare_vblank_event(struct intel_crtc_state *crtc_state,
627 struct drm_pending_vblank_event **event)
628{
629 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
630 unsigned long irqflags;
631
632 spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
633 *event = crtc_state->uapi.event;
634 spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
635
636 crtc_state->uapi.event = NULL;
637}
638
639/**
640 * intel_pipe_update_end() - end update of a set of display registers
641 * @state: the atomic state
642 * @crtc: the crtc
643 *
644 * Mark the end of an update started with intel_pipe_update_start(). This
645 * re-enables interrupts and verifies the update was actually completed
646 * before a vblank.
647 */
648void intel_pipe_update_end(struct intel_atomic_state *state,
649 struct intel_crtc *crtc)
650{
651 struct intel_crtc_state *new_crtc_state =
652 intel_atomic_get_new_crtc_state(state, crtc);
653 enum pipe pipe = crtc->pipe;
654 int scanline_end = intel_get_crtc_scanline(crtc);
655 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
656 ktime_t end_vbl_time = ktime_get();
657 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
658
659 if (new_crtc_state->do_async_flip)
660 goto out;
661
662 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
663
664 /*
665 * Incase of mipi dsi command mode, we need to set frame update
666 * request for every commit.
667 */
668 if (DISPLAY_VER(dev_priv) >= 11 &&
669 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
670 icl_dsi_frame_update(new_crtc_state);
671
672 /* We're still in the vblank-evade critical section, this can't race.
673 * Would be slightly nice to just grab the vblank count and arm the
674 * event outside of the critical section - the spinlock might spin for a
675 * while ... */
676 if (intel_crtc_needs_vblank_work(new_crtc_state)) {
677 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
678 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
679 false);
680 } else {
681 intel_crtc_arm_vblank_event(new_crtc_state);
682 }
683
684 if (state->base.legacy_cursor_update) {
685 struct intel_plane *plane;
686 struct intel_plane_state *old_plane_state;
687 int i;
688
689 for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
690 if (old_plane_state->uapi.crtc == &crtc->base &&
691 old_plane_state->unpin_work.vblank) {
692 drm_vblank_work_schedule(&old_plane_state->unpin_work,
693 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
694 false);
695
696 /* Remove plane from atomic state, cleanup/free is done from vblank worker. */
697 memset(&state->base.planes[i], 0, sizeof(state->base.planes[i]));
698 }
699 }
700 }
701
702 /*
703 * Send VRR Push to terminate Vblank. If we are already in vblank
704 * this has to be done _after_ sampling the frame counter, as
705 * otherwise the push would immediately terminate the vblank and
706 * the sampled frame counter would correspond to the next frame
707 * instead of the current frame.
708 *
709 * There is a tiny race here (iff vblank evasion failed us) where
710 * we might sample the frame counter just before vmax vblank start
711 * but the push would be sent just after it. That would cause the
712 * push to affect the next frame instead of the current frame,
713 * which would cause the next frame to terminate already at vmin
714 * vblank start instead of vmax vblank start.
715 */
716 intel_vrr_send_push(new_crtc_state);
717
718 local_irq_enable();
719
720 if (intel_vgpu_active(dev_priv))
721 goto out;
722
723 if (crtc->debug.start_vbl_count &&
724 crtc->debug.start_vbl_count != end_vbl_count) {
725 drm_err(&dev_priv->drm,
726 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
727 pipe_name(pipe), crtc->debug.start_vbl_count,
728 end_vbl_count,
729 ktime_us_delta(end_vbl_time,
730 crtc->debug.start_vbl_time),
731 crtc->debug.min_vbl, crtc->debug.max_vbl,
732 crtc->debug.scanline_start, scanline_end);
733 }
734
735 dbg_vblank_evade(crtc, end_vbl_time);
736
737out:
738 intel_psr_unlock(new_crtc_state);
739}
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5#include <linux/kernel.h>
6#include <linux/pm_qos.h>
7#include <linux/slab.h>
8
9#include <drm/drm_atomic_helper.h>
10#include <drm/drm_fourcc.h>
11#include <drm/drm_plane.h>
12#include <drm/drm_vblank_work.h>
13
14#include "i915_vgpu.h"
15#include "i9xx_plane.h"
16#include "icl_dsi.h"
17#include "intel_atomic.h"
18#include "intel_atomic_plane.h"
19#include "intel_color.h"
20#include "intel_crtc.h"
21#include "intel_cursor.h"
22#include "intel_display_debugfs.h"
23#include "intel_display_irq.h"
24#include "intel_display_trace.h"
25#include "intel_display_types.h"
26#include "intel_drrs.h"
27#include "intel_dsb.h"
28#include "intel_dsi.h"
29#include "intel_fifo_underrun.h"
30#include "intel_pipe_crc.h"
31#include "intel_psr.h"
32#include "intel_sprite.h"
33#include "intel_vblank.h"
34#include "intel_vrr.h"
35#include "skl_universal_plane.h"
36
37static void assert_vblank_disabled(struct drm_crtc *crtc)
38{
39 struct drm_i915_private *i915 = to_i915(crtc->dev);
40
41 if (I915_STATE_WARN(i915, drm_crtc_vblank_get(crtc) == 0,
42 "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
43 crtc->base.id, crtc->name))
44 drm_crtc_vblank_put(crtc);
45}
46
47struct intel_crtc *intel_first_crtc(struct drm_i915_private *i915)
48{
49 return to_intel_crtc(drm_crtc_from_index(&i915->drm, 0));
50}
51
52struct intel_crtc *intel_crtc_for_pipe(struct drm_i915_private *i915,
53 enum pipe pipe)
54{
55 struct intel_crtc *crtc;
56
57 for_each_intel_crtc(&i915->drm, crtc) {
58 if (crtc->pipe == pipe)
59 return crtc;
60 }
61
62 return NULL;
63}
64
65void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
66{
67 drm_crtc_wait_one_vblank(&crtc->base);
68}
69
70void intel_wait_for_vblank_if_active(struct drm_i915_private *i915,
71 enum pipe pipe)
72{
73 struct intel_crtc *crtc = intel_crtc_for_pipe(i915, pipe);
74
75 if (crtc->active)
76 intel_crtc_wait_for_next_vblank(crtc);
77}
78
79u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
80{
81 struct drm_device *dev = crtc->base.dev;
82 struct drm_vblank_crtc *vblank = &dev->vblank[drm_crtc_index(&crtc->base)];
83
84 if (!crtc->active)
85 return 0;
86
87 if (!vblank->max_vblank_count)
88 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
89
90 return crtc->base.funcs->get_vblank_counter(&crtc->base);
91}
92
93u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
94{
95 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
96
97 /*
98 * From Gen 11, In case of dsi cmd mode, frame counter wouldnt
99 * have updated at the beginning of TE, if we want to use
100 * the hw counter, then we would find it updated in only
101 * the next TE, hence switching to sw counter.
102 */
103 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
104 I915_MODE_FLAG_DSI_USE_TE1))
105 return 0;
106
107 /*
108 * On i965gm the hardware frame counter reads
109 * zero when the TV encoder is enabled :(
110 */
111 if (IS_I965GM(dev_priv) &&
112 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
113 return 0;
114
115 if (DISPLAY_VER(dev_priv) >= 5 || IS_G4X(dev_priv))
116 return 0xffffffff; /* full 32 bit counter */
117 else if (DISPLAY_VER(dev_priv) >= 3)
118 return 0xffffff; /* only 24 bits of frame count */
119 else
120 return 0; /* Gen2 doesn't have a hardware frame counter */
121}
122
123void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
124{
125 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
126
127 assert_vblank_disabled(&crtc->base);
128 drm_crtc_set_max_vblank_count(&crtc->base,
129 intel_crtc_max_vblank_count(crtc_state));
130 drm_crtc_vblank_on(&crtc->base);
131
132 /*
133 * Should really happen exactly when we enable the pipe
134 * but we want the frame counters in the trace, and that
135 * requires vblank support on some platforms/outputs.
136 */
137 trace_intel_pipe_enable(crtc);
138}
139
140void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
141{
142 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
143
144 /*
145 * Should really happen exactly when we disable the pipe
146 * but we want the frame counters in the trace, and that
147 * requires vblank support on some platforms/outputs.
148 */
149 trace_intel_pipe_disable(crtc);
150
151 drm_crtc_vblank_off(&crtc->base);
152 assert_vblank_disabled(&crtc->base);
153}
154
155struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
156{
157 struct intel_crtc_state *crtc_state;
158
159 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
160
161 if (crtc_state)
162 intel_crtc_state_reset(crtc_state, crtc);
163
164 return crtc_state;
165}
166
167void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
168 struct intel_crtc *crtc)
169{
170 memset(crtc_state, 0, sizeof(*crtc_state));
171
172 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
173
174 crtc_state->cpu_transcoder = INVALID_TRANSCODER;
175 crtc_state->master_transcoder = INVALID_TRANSCODER;
176 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
177 crtc_state->scaler_state.scaler_id = -1;
178 crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
179 crtc_state->max_link_bpp_x16 = INT_MAX;
180}
181
182static struct intel_crtc *intel_crtc_alloc(void)
183{
184 struct intel_crtc_state *crtc_state;
185 struct intel_crtc *crtc;
186
187 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
188 if (!crtc)
189 return ERR_PTR(-ENOMEM);
190
191 crtc_state = intel_crtc_state_alloc(crtc);
192 if (!crtc_state) {
193 kfree(crtc);
194 return ERR_PTR(-ENOMEM);
195 }
196
197 crtc->base.state = &crtc_state->uapi;
198 crtc->config = crtc_state;
199
200 return crtc;
201}
202
203static void intel_crtc_free(struct intel_crtc *crtc)
204{
205 intel_crtc_destroy_state(&crtc->base, crtc->base.state);
206 kfree(crtc);
207}
208
209static void intel_crtc_destroy(struct drm_crtc *_crtc)
210{
211 struct intel_crtc *crtc = to_intel_crtc(_crtc);
212
213 cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
214
215 drm_crtc_cleanup(&crtc->base);
216 kfree(crtc);
217}
218
219static int intel_crtc_late_register(struct drm_crtc *crtc)
220{
221 intel_crtc_debugfs_add(to_intel_crtc(crtc));
222 return 0;
223}
224
225#define INTEL_CRTC_FUNCS \
226 .set_config = drm_atomic_helper_set_config, \
227 .destroy = intel_crtc_destroy, \
228 .page_flip = drm_atomic_helper_page_flip, \
229 .atomic_duplicate_state = intel_crtc_duplicate_state, \
230 .atomic_destroy_state = intel_crtc_destroy_state, \
231 .set_crc_source = intel_crtc_set_crc_source, \
232 .verify_crc_source = intel_crtc_verify_crc_source, \
233 .get_crc_sources = intel_crtc_get_crc_sources, \
234 .late_register = intel_crtc_late_register
235
236static const struct drm_crtc_funcs bdw_crtc_funcs = {
237 INTEL_CRTC_FUNCS,
238
239 .get_vblank_counter = g4x_get_vblank_counter,
240 .enable_vblank = bdw_enable_vblank,
241 .disable_vblank = bdw_disable_vblank,
242 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
243};
244
245static const struct drm_crtc_funcs ilk_crtc_funcs = {
246 INTEL_CRTC_FUNCS,
247
248 .get_vblank_counter = g4x_get_vblank_counter,
249 .enable_vblank = ilk_enable_vblank,
250 .disable_vblank = ilk_disable_vblank,
251 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
252};
253
254static const struct drm_crtc_funcs g4x_crtc_funcs = {
255 INTEL_CRTC_FUNCS,
256
257 .get_vblank_counter = g4x_get_vblank_counter,
258 .enable_vblank = i965_enable_vblank,
259 .disable_vblank = i965_disable_vblank,
260 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
261};
262
263static const struct drm_crtc_funcs i965_crtc_funcs = {
264 INTEL_CRTC_FUNCS,
265
266 .get_vblank_counter = i915_get_vblank_counter,
267 .enable_vblank = i965_enable_vblank,
268 .disable_vblank = i965_disable_vblank,
269 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
270};
271
272static const struct drm_crtc_funcs i915gm_crtc_funcs = {
273 INTEL_CRTC_FUNCS,
274
275 .get_vblank_counter = i915_get_vblank_counter,
276 .enable_vblank = i915gm_enable_vblank,
277 .disable_vblank = i915gm_disable_vblank,
278 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
279};
280
281static const struct drm_crtc_funcs i915_crtc_funcs = {
282 INTEL_CRTC_FUNCS,
283
284 .get_vblank_counter = i915_get_vblank_counter,
285 .enable_vblank = i8xx_enable_vblank,
286 .disable_vblank = i8xx_disable_vblank,
287 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
288};
289
290static const struct drm_crtc_funcs i8xx_crtc_funcs = {
291 INTEL_CRTC_FUNCS,
292
293 /* no hw vblank counter */
294 .enable_vblank = i8xx_enable_vblank,
295 .disable_vblank = i8xx_disable_vblank,
296 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
297};
298
299int intel_crtc_init(struct drm_i915_private *dev_priv, enum pipe pipe)
300{
301 struct intel_plane *primary, *cursor;
302 const struct drm_crtc_funcs *funcs;
303 struct intel_crtc *crtc;
304 int sprite, ret;
305
306 crtc = intel_crtc_alloc();
307 if (IS_ERR(crtc))
308 return PTR_ERR(crtc);
309
310 crtc->pipe = pipe;
311 crtc->num_scalers = DISPLAY_RUNTIME_INFO(dev_priv)->num_scalers[pipe];
312
313 if (DISPLAY_VER(dev_priv) >= 9)
314 primary = skl_universal_plane_create(dev_priv, pipe,
315 PLANE_PRIMARY);
316 else
317 primary = intel_primary_plane_create(dev_priv, pipe);
318 if (IS_ERR(primary)) {
319 ret = PTR_ERR(primary);
320 goto fail;
321 }
322 crtc->plane_ids_mask |= BIT(primary->id);
323
324 intel_init_fifo_underrun_reporting(dev_priv, crtc, false);
325
326 for_each_sprite(dev_priv, pipe, sprite) {
327 struct intel_plane *plane;
328
329 if (DISPLAY_VER(dev_priv) >= 9)
330 plane = skl_universal_plane_create(dev_priv, pipe,
331 PLANE_SPRITE0 + sprite);
332 else
333 plane = intel_sprite_plane_create(dev_priv, pipe, sprite);
334 if (IS_ERR(plane)) {
335 ret = PTR_ERR(plane);
336 goto fail;
337 }
338 crtc->plane_ids_mask |= BIT(plane->id);
339 }
340
341 cursor = intel_cursor_plane_create(dev_priv, pipe);
342 if (IS_ERR(cursor)) {
343 ret = PTR_ERR(cursor);
344 goto fail;
345 }
346 crtc->plane_ids_mask |= BIT(cursor->id);
347
348 if (HAS_GMCH(dev_priv)) {
349 if (IS_CHERRYVIEW(dev_priv) ||
350 IS_VALLEYVIEW(dev_priv) || IS_G4X(dev_priv))
351 funcs = &g4x_crtc_funcs;
352 else if (DISPLAY_VER(dev_priv) == 4)
353 funcs = &i965_crtc_funcs;
354 else if (IS_I945GM(dev_priv) || IS_I915GM(dev_priv))
355 funcs = &i915gm_crtc_funcs;
356 else if (DISPLAY_VER(dev_priv) == 3)
357 funcs = &i915_crtc_funcs;
358 else
359 funcs = &i8xx_crtc_funcs;
360 } else {
361 if (DISPLAY_VER(dev_priv) >= 8)
362 funcs = &bdw_crtc_funcs;
363 else
364 funcs = &ilk_crtc_funcs;
365 }
366
367 ret = drm_crtc_init_with_planes(&dev_priv->drm, &crtc->base,
368 &primary->base, &cursor->base,
369 funcs, "pipe %c", pipe_name(pipe));
370 if (ret)
371 goto fail;
372
373 if (DISPLAY_VER(dev_priv) >= 11)
374 drm_crtc_create_scaling_filter_property(&crtc->base,
375 BIT(DRM_SCALING_FILTER_DEFAULT) |
376 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
377
378 intel_color_crtc_init(crtc);
379 intel_drrs_crtc_init(crtc);
380 intel_crtc_crc_init(crtc);
381
382 cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
383
384 drm_WARN_ON(&dev_priv->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
385
386 return 0;
387
388fail:
389 intel_crtc_free(crtc);
390
391 return ret;
392}
393
394static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
395{
396 return crtc_state->hw.active &&
397 !intel_crtc_needs_modeset(crtc_state) &&
398 !crtc_state->preload_luts &&
399 intel_crtc_needs_color_update(crtc_state) &&
400 !intel_color_uses_dsb(crtc_state);
401}
402
403static void intel_crtc_vblank_work(struct kthread_work *base)
404{
405 struct drm_vblank_work *work = to_drm_vblank_work(base);
406 struct intel_crtc_state *crtc_state =
407 container_of(work, typeof(*crtc_state), vblank_work);
408 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
409
410 trace_intel_crtc_vblank_work_start(crtc);
411
412 intel_color_load_luts(crtc_state);
413
414 if (crtc_state->uapi.event) {
415 spin_lock_irq(&crtc->base.dev->event_lock);
416 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
417 crtc_state->uapi.event = NULL;
418 spin_unlock_irq(&crtc->base.dev->event_lock);
419 }
420
421 trace_intel_crtc_vblank_work_end(crtc);
422}
423
424static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
425{
426 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
427
428 drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
429 intel_crtc_vblank_work);
430 /*
431 * Interrupt latency is critical for getting the vblank
432 * work executed as early as possible during the vblank.
433 */
434 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
435}
436
437void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
438{
439 struct intel_crtc_state *crtc_state;
440 struct intel_crtc *crtc;
441 int i;
442
443 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
444 if (!intel_crtc_needs_vblank_work(crtc_state))
445 continue;
446
447 drm_vblank_work_flush(&crtc_state->vblank_work);
448 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
449 PM_QOS_DEFAULT_VALUE);
450 }
451}
452
453int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
454 int usecs)
455{
456 /* paranoia */
457 if (!adjusted_mode->crtc_htotal)
458 return 1;
459
460 return DIV_ROUND_UP(usecs * adjusted_mode->crtc_clock,
461 1000 * adjusted_mode->crtc_htotal);
462}
463
464static int intel_mode_vblank_start(const struct drm_display_mode *mode)
465{
466 int vblank_start = mode->crtc_vblank_start;
467
468 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
469 vblank_start = DIV_ROUND_UP(vblank_start, 2);
470
471 return vblank_start;
472}
473
474static void intel_crtc_vblank_evade_scanlines(struct intel_atomic_state *state,
475 struct intel_crtc *crtc,
476 int *min, int *max, int *vblank_start)
477{
478 const struct intel_crtc_state *old_crtc_state =
479 intel_atomic_get_old_crtc_state(state, crtc);
480 const struct intel_crtc_state *new_crtc_state =
481 intel_atomic_get_new_crtc_state(state, crtc);
482 const struct intel_crtc_state *crtc_state;
483 const struct drm_display_mode *adjusted_mode;
484
485 /*
486 * During fastsets/etc. the transcoder is still
487 * running with the old timings at this point.
488 *
489 * TODO: maybe just use the active timings here?
490 */
491 if (intel_crtc_needs_modeset(new_crtc_state))
492 crtc_state = new_crtc_state;
493 else
494 crtc_state = old_crtc_state;
495
496 adjusted_mode = &crtc_state->hw.adjusted_mode;
497
498 if (crtc->mode_flags & I915_MODE_FLAG_VRR) {
499 /* timing changes should happen with VRR disabled */
500 drm_WARN_ON(state->base.dev, intel_crtc_needs_modeset(new_crtc_state) ||
501 new_crtc_state->update_m_n || new_crtc_state->update_lrr);
502
503 if (intel_vrr_is_push_sent(crtc_state))
504 *vblank_start = intel_vrr_vmin_vblank_start(crtc_state);
505 else
506 *vblank_start = intel_vrr_vmax_vblank_start(crtc_state);
507 } else {
508 *vblank_start = intel_mode_vblank_start(adjusted_mode);
509 }
510
511 /* FIXME needs to be calibrated sensibly */
512 *min = *vblank_start - intel_usecs_to_scanlines(adjusted_mode,
513 VBLANK_EVASION_TIME_US);
514 *max = *vblank_start - 1;
515
516 /*
517 * M/N and TRANS_VTOTAL are double buffered on the transcoder's
518 * undelayed vblank, so with seamless M/N and LRR we must evade
519 * both vblanks.
520 *
521 * DSB execution waits for the transcoder's undelayed vblank,
522 * hence we must kick off the commit before that.
523 */
524 if (new_crtc_state->dsb || new_crtc_state->update_m_n || new_crtc_state->update_lrr)
525 *min -= adjusted_mode->crtc_vblank_start - adjusted_mode->crtc_vdisplay;
526}
527
528/**
529 * intel_pipe_update_start() - start update of a set of display registers
530 * @state: the atomic state
531 * @crtc: the crtc
532 *
533 * Mark the start of an update to pipe registers that should be updated
534 * atomically regarding vblank. If the next vblank will happens within
535 * the next 100 us, this function waits until the vblank passes.
536 *
537 * After a successful call to this function, interrupts will be disabled
538 * until a subsequent call to intel_pipe_update_end(). That is done to
539 * avoid random delays.
540 */
541void intel_pipe_update_start(struct intel_atomic_state *state,
542 struct intel_crtc *crtc)
543{
544 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
545 struct intel_crtc_state *new_crtc_state =
546 intel_atomic_get_new_crtc_state(state, crtc);
547 long timeout = msecs_to_jiffies_timeout(1);
548 int scanline, min, max, vblank_start;
549 wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
550 bool need_vlv_dsi_wa = (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) &&
551 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI);
552 DEFINE_WAIT(wait);
553
554 intel_psr_lock(new_crtc_state);
555
556 if (new_crtc_state->do_async_flip) {
557 spin_lock_irq(&crtc->base.dev->event_lock);
558 /* arm the event for the flip done irq handler */
559 crtc->flip_done_event = new_crtc_state->uapi.event;
560 spin_unlock_irq(&crtc->base.dev->event_lock);
561
562 new_crtc_state->uapi.event = NULL;
563 return;
564 }
565
566 if (intel_crtc_needs_vblank_work(new_crtc_state))
567 intel_crtc_vblank_work_init(new_crtc_state);
568
569 intel_crtc_vblank_evade_scanlines(state, crtc, &min, &max, &vblank_start);
570 if (min <= 0 || max <= 0)
571 goto irq_disable;
572
573 if (drm_WARN_ON(&dev_priv->drm, drm_crtc_vblank_get(&crtc->base)))
574 goto irq_disable;
575
576 /*
577 * Wait for psr to idle out after enabling the VBL interrupts
578 * VBL interrupts will start the PSR exit and prevent a PSR
579 * re-entry as well.
580 */
581 intel_psr_wait_for_idle_locked(new_crtc_state);
582
583 local_irq_disable();
584
585 crtc->debug.min_vbl = min;
586 crtc->debug.max_vbl = max;
587 trace_intel_pipe_update_start(crtc);
588
589 for (;;) {
590 /*
591 * prepare_to_wait() has a memory barrier, which guarantees
592 * other CPUs can see the task state update by the time we
593 * read the scanline.
594 */
595 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
596
597 scanline = intel_get_crtc_scanline(crtc);
598 if (scanline < min || scanline > max)
599 break;
600
601 if (!timeout) {
602 drm_err(&dev_priv->drm,
603 "Potential atomic update failure on pipe %c\n",
604 pipe_name(crtc->pipe));
605 break;
606 }
607
608 local_irq_enable();
609
610 timeout = schedule_timeout(timeout);
611
612 local_irq_disable();
613 }
614
615 finish_wait(wq, &wait);
616
617 drm_crtc_vblank_put(&crtc->base);
618
619 /*
620 * On VLV/CHV DSI the scanline counter would appear to
621 * increment approx. 1/3 of a scanline before start of vblank.
622 * The registers still get latched at start of vblank however.
623 * This means we must not write any registers on the first
624 * line of vblank (since not the whole line is actually in
625 * vblank). And unfortunately we can't use the interrupt to
626 * wait here since it will fire too soon. We could use the
627 * frame start interrupt instead since it will fire after the
628 * critical scanline, but that would require more changes
629 * in the interrupt code. So for now we'll just do the nasty
630 * thing and poll for the bad scanline to pass us by.
631 *
632 * FIXME figure out if BXT+ DSI suffers from this as well
633 */
634 while (need_vlv_dsi_wa && scanline == vblank_start)
635 scanline = intel_get_crtc_scanline(crtc);
636
637 crtc->debug.scanline_start = scanline;
638 crtc->debug.start_vbl_time = ktime_get();
639 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
640
641 trace_intel_pipe_update_vblank_evaded(crtc);
642 return;
643
644irq_disable:
645 local_irq_disable();
646}
647
648#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
649static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
650{
651 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
652 unsigned int h;
653
654 h = ilog2(delta >> 9);
655 if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
656 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
657 crtc->debug.vbl.times[h]++;
658
659 crtc->debug.vbl.sum += delta;
660 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
661 crtc->debug.vbl.min = delta;
662 if (delta > crtc->debug.vbl.max)
663 crtc->debug.vbl.max = delta;
664
665 if (delta > 1000 * VBLANK_EVASION_TIME_US) {
666 drm_dbg_kms(crtc->base.dev,
667 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
668 pipe_name(crtc->pipe),
669 div_u64(delta, 1000),
670 VBLANK_EVASION_TIME_US);
671 crtc->debug.vbl.over++;
672 }
673}
674#else
675static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
676#endif
677
678/**
679 * intel_pipe_update_end() - end update of a set of display registers
680 * @state: the atomic state
681 * @crtc: the crtc
682 *
683 * Mark the end of an update started with intel_pipe_update_start(). This
684 * re-enables interrupts and verifies the update was actually completed
685 * before a vblank.
686 */
687void intel_pipe_update_end(struct intel_atomic_state *state,
688 struct intel_crtc *crtc)
689{
690 struct intel_crtc_state *new_crtc_state =
691 intel_atomic_get_new_crtc_state(state, crtc);
692 enum pipe pipe = crtc->pipe;
693 int scanline_end = intel_get_crtc_scanline(crtc);
694 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
695 ktime_t end_vbl_time = ktime_get();
696 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
697
698 if (new_crtc_state->do_async_flip)
699 goto out;
700
701 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
702
703 /*
704 * Incase of mipi dsi command mode, we need to set frame update
705 * request for every commit.
706 */
707 if (DISPLAY_VER(dev_priv) >= 11 &&
708 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
709 icl_dsi_frame_update(new_crtc_state);
710
711 /* We're still in the vblank-evade critical section, this can't race.
712 * Would be slightly nice to just grab the vblank count and arm the
713 * event outside of the critical section - the spinlock might spin for a
714 * while ... */
715 if (intel_crtc_needs_vblank_work(new_crtc_state)) {
716 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
717 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
718 false);
719 } else if (new_crtc_state->uapi.event) {
720 drm_WARN_ON(&dev_priv->drm,
721 drm_crtc_vblank_get(&crtc->base) != 0);
722
723 spin_lock(&crtc->base.dev->event_lock);
724 drm_crtc_arm_vblank_event(&crtc->base,
725 new_crtc_state->uapi.event);
726 spin_unlock(&crtc->base.dev->event_lock);
727
728 new_crtc_state->uapi.event = NULL;
729 }
730
731 /*
732 * Send VRR Push to terminate Vblank. If we are already in vblank
733 * this has to be done _after_ sampling the frame counter, as
734 * otherwise the push would immediately terminate the vblank and
735 * the sampled frame counter would correspond to the next frame
736 * instead of the current frame.
737 *
738 * There is a tiny race here (iff vblank evasion failed us) where
739 * we might sample the frame counter just before vmax vblank start
740 * but the push would be sent just after it. That would cause the
741 * push to affect the next frame instead of the current frame,
742 * which would cause the next frame to terminate already at vmin
743 * vblank start instead of vmax vblank start.
744 */
745 intel_vrr_send_push(new_crtc_state);
746
747 local_irq_enable();
748
749 if (intel_vgpu_active(dev_priv))
750 goto out;
751
752 if (crtc->debug.start_vbl_count &&
753 crtc->debug.start_vbl_count != end_vbl_count) {
754 drm_err(&dev_priv->drm,
755 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
756 pipe_name(pipe), crtc->debug.start_vbl_count,
757 end_vbl_count,
758 ktime_us_delta(end_vbl_time,
759 crtc->debug.start_vbl_time),
760 crtc->debug.min_vbl, crtc->debug.max_vbl,
761 crtc->debug.scanline_start, scanline_end);
762 }
763
764 dbg_vblank_evade(crtc, end_vbl_time);
765
766out:
767 intel_psr_unlock(new_crtc_state);
768}