Loading...
Note: File does not exist in v3.1.
1/*
2 * drm_irq.c IRQ and vblank support
3 *
4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
5 * \author Gareth Hughes <gareth@valinux.com>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 * OTHER DEALINGS IN THE SOFTWARE.
25 */
26
27#include <linux/export.h>
28#include <linux/moduleparam.h>
29
30#include <drm/drm_crtc.h>
31#include <drm/drm_drv.h>
32#include <drm/drm_framebuffer.h>
33#include <drm/drm_print.h>
34#include <drm/drm_vblank.h>
35
36#include "drm_internal.h"
37#include "drm_trace.h"
38
39/**
40 * DOC: vblank handling
41 *
42 * Vertical blanking plays a major role in graphics rendering. To achieve
43 * tear-free display, users must synchronize page flips and/or rendering to
44 * vertical blanking. The DRM API offers ioctls to perform page flips
45 * synchronized to vertical blanking and wait for vertical blanking.
46 *
47 * The DRM core handles most of the vertical blanking management logic, which
48 * involves filtering out spurious interrupts, keeping race-free blanking
49 * counters, coping with counter wrap-around and resets and keeping use counts.
50 * It relies on the driver to generate vertical blanking interrupts and
51 * optionally provide a hardware vertical blanking counter.
52 *
53 * Drivers must initialize the vertical blanking handling core with a call to
54 * drm_vblank_init(). Minimally, a driver needs to implement
55 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
56 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
57 * support.
58 *
59 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
60 * themselves (for instance to handle page flipping operations). The DRM core
61 * maintains a vertical blanking use count to ensure that the interrupts are not
62 * disabled while a user still needs them. To increment the use count, drivers
63 * call drm_crtc_vblank_get() and release the vblank reference again with
64 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
65 * guaranteed to be enabled.
66 *
67 * On many hardware disabling the vblank interrupt cannot be done in a race-free
68 * manner, see &drm_driver.vblank_disable_immediate and
69 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
70 * vblanks after a timer has expired, which can be configured through the
71 * ``vblankoffdelay`` module parameter.
72 */
73
74/* Retry timestamp calculation up to 3 times to satisfy
75 * drm_timestamp_precision before giving up.
76 */
77#define DRM_TIMESTAMP_MAXRETRIES 3
78
79/* Threshold in nanoseconds for detection of redundant
80 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
81 */
82#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
83
84static bool
85drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
86 ktime_t *tvblank, bool in_vblank_irq);
87
88static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
89
90static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
91
92module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
93module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
94MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
95MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
96
97static void store_vblank(struct drm_device *dev, unsigned int pipe,
98 u32 vblank_count_inc,
99 ktime_t t_vblank, u32 last)
100{
101 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
102
103 assert_spin_locked(&dev->vblank_time_lock);
104
105 vblank->last = last;
106
107 write_seqlock(&vblank->seqlock);
108 vblank->time = t_vblank;
109 vblank->count += vblank_count_inc;
110 write_sequnlock(&vblank->seqlock);
111}
112
113static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
114{
115 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
116
117 return vblank->max_vblank_count ?: dev->max_vblank_count;
118}
119
120/*
121 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
122 * if there is no useable hardware frame counter available.
123 */
124static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
125{
126 WARN_ON_ONCE(drm_max_vblank_count(dev, pipe) != 0);
127 return 0;
128}
129
130static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
131{
132 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
133 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
134
135 if (WARN_ON(!crtc))
136 return 0;
137
138 if (crtc->funcs->get_vblank_counter)
139 return crtc->funcs->get_vblank_counter(crtc);
140 }
141
142 if (dev->driver->get_vblank_counter)
143 return dev->driver->get_vblank_counter(dev, pipe);
144
145 return drm_vblank_no_hw_counter(dev, pipe);
146}
147
148/*
149 * Reset the stored timestamp for the current vblank count to correspond
150 * to the last vblank occurred.
151 *
152 * Only to be called from drm_crtc_vblank_on().
153 *
154 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
155 * device vblank fields.
156 */
157static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
158{
159 u32 cur_vblank;
160 bool rc;
161 ktime_t t_vblank;
162 int count = DRM_TIMESTAMP_MAXRETRIES;
163
164 spin_lock(&dev->vblank_time_lock);
165
166 /*
167 * sample the current counter to avoid random jumps
168 * when drm_vblank_enable() applies the diff
169 */
170 do {
171 cur_vblank = __get_vblank_counter(dev, pipe);
172 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
173 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
174
175 /*
176 * Only reinitialize corresponding vblank timestamp if high-precision query
177 * available and didn't fail. Otherwise reinitialize delayed at next vblank
178 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
179 */
180 if (!rc)
181 t_vblank = 0;
182
183 /*
184 * +1 to make sure user will never see the same
185 * vblank counter value before and after a modeset
186 */
187 store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
188
189 spin_unlock(&dev->vblank_time_lock);
190}
191
192/*
193 * Call back into the driver to update the appropriate vblank counter
194 * (specified by @pipe). Deal with wraparound, if it occurred, and
195 * update the last read value so we can deal with wraparound on the next
196 * call if necessary.
197 *
198 * Only necessary when going from off->on, to account for frames we
199 * didn't get an interrupt for.
200 *
201 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
202 * device vblank fields.
203 */
204static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
205 bool in_vblank_irq)
206{
207 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
208 u32 cur_vblank, diff;
209 bool rc;
210 ktime_t t_vblank;
211 int count = DRM_TIMESTAMP_MAXRETRIES;
212 int framedur_ns = vblank->framedur_ns;
213 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
214
215 /*
216 * Interrupts were disabled prior to this call, so deal with counter
217 * wrap if needed.
218 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
219 * here if the register is small or we had vblank interrupts off for
220 * a long time.
221 *
222 * We repeat the hardware vblank counter & timestamp query until
223 * we get consistent results. This to prevent races between gpu
224 * updating its hardware counter while we are retrieving the
225 * corresponding vblank timestamp.
226 */
227 do {
228 cur_vblank = __get_vblank_counter(dev, pipe);
229 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
230 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
231
232 if (max_vblank_count) {
233 /* trust the hw counter when it's around */
234 diff = (cur_vblank - vblank->last) & max_vblank_count;
235 } else if (rc && framedur_ns) {
236 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
237
238 /*
239 * Figure out how many vblanks we've missed based
240 * on the difference in the timestamps and the
241 * frame/field duration.
242 */
243
244 DRM_DEBUG_VBL("crtc %u: Calculating number of vblanks."
245 " diff_ns = %lld, framedur_ns = %d)\n",
246 pipe, (long long) diff_ns, framedur_ns);
247
248 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
249
250 if (diff == 0 && in_vblank_irq)
251 DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored\n",
252 pipe);
253 } else {
254 /* some kind of default for drivers w/o accurate vbl timestamping */
255 diff = in_vblank_irq ? 1 : 0;
256 }
257
258 /*
259 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
260 * interval? If so then vblank irqs keep running and it will likely
261 * happen that the hardware vblank counter is not trustworthy as it
262 * might reset at some point in that interval and vblank timestamps
263 * are not trustworthy either in that interval. Iow. this can result
264 * in a bogus diff >> 1 which must be avoided as it would cause
265 * random large forward jumps of the software vblank counter.
266 */
267 if (diff > 1 && (vblank->inmodeset & 0x2)) {
268 DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
269 " due to pre-modeset.\n", pipe, diff);
270 diff = 1;
271 }
272
273 DRM_DEBUG_VBL("updating vblank count on crtc %u:"
274 " current=%llu, diff=%u, hw=%u hw_last=%u\n",
275 pipe, vblank->count, diff, cur_vblank, vblank->last);
276
277 if (diff == 0) {
278 WARN_ON_ONCE(cur_vblank != vblank->last);
279 return;
280 }
281
282 /*
283 * Only reinitialize corresponding vblank timestamp if high-precision query
284 * available and didn't fail, or we were called from the vblank interrupt.
285 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
286 * for now, to mark the vblanktimestamp as invalid.
287 */
288 if (!rc && !in_vblank_irq)
289 t_vblank = 0;
290
291 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
292}
293
294static u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
295{
296 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
297
298 if (WARN_ON(pipe >= dev->num_crtcs))
299 return 0;
300
301 return vblank->count;
302}
303
304/**
305 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
306 * @crtc: which counter to retrieve
307 *
308 * This function is similar to drm_crtc_vblank_count() but this function
309 * interpolates to handle a race with vblank interrupts using the high precision
310 * timestamping support.
311 *
312 * This is mostly useful for hardware that can obtain the scanout position, but
313 * doesn't have a hardware frame counter.
314 */
315u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
316{
317 struct drm_device *dev = crtc->dev;
318 unsigned int pipe = drm_crtc_index(crtc);
319 u64 vblank;
320 unsigned long flags;
321
322 WARN_ONCE(drm_debug & DRM_UT_VBL && !dev->driver->get_vblank_timestamp,
323 "This function requires support for accurate vblank timestamps.");
324
325 spin_lock_irqsave(&dev->vblank_time_lock, flags);
326
327 drm_update_vblank_count(dev, pipe, false);
328 vblank = drm_vblank_count(dev, pipe);
329
330 spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
331
332 return vblank;
333}
334EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
335
336static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
337{
338 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
339 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
340
341 if (WARN_ON(!crtc))
342 return;
343
344 if (crtc->funcs->disable_vblank) {
345 crtc->funcs->disable_vblank(crtc);
346 return;
347 }
348 }
349
350 dev->driver->disable_vblank(dev, pipe);
351}
352
353/*
354 * Disable vblank irq's on crtc, make sure that last vblank count
355 * of hardware and corresponding consistent software vblank counter
356 * are preserved, even if there are any spurious vblank irq's after
357 * disable.
358 */
359void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
360{
361 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
362 unsigned long irqflags;
363
364 assert_spin_locked(&dev->vbl_lock);
365
366 /* Prevent vblank irq processing while disabling vblank irqs,
367 * so no updates of timestamps or count can happen after we've
368 * disabled. Needed to prevent races in case of delayed irq's.
369 */
370 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
371
372 /*
373 * Update vblank count and disable vblank interrupts only if the
374 * interrupts were enabled. This avoids calling the ->disable_vblank()
375 * operation in atomic context with the hardware potentially runtime
376 * suspended.
377 */
378 if (!vblank->enabled)
379 goto out;
380
381 /*
382 * Update the count and timestamp to maintain the
383 * appearance that the counter has been ticking all along until
384 * this time. This makes the count account for the entire time
385 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
386 */
387 drm_update_vblank_count(dev, pipe, false);
388 __disable_vblank(dev, pipe);
389 vblank->enabled = false;
390
391out:
392 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
393}
394
395static void vblank_disable_fn(struct timer_list *t)
396{
397 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
398 struct drm_device *dev = vblank->dev;
399 unsigned int pipe = vblank->pipe;
400 unsigned long irqflags;
401
402 spin_lock_irqsave(&dev->vbl_lock, irqflags);
403 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
404 DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
405 drm_vblank_disable_and_save(dev, pipe);
406 }
407 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
408}
409
410void drm_vblank_cleanup(struct drm_device *dev)
411{
412 unsigned int pipe;
413
414 /* Bail if the driver didn't call drm_vblank_init() */
415 if (dev->num_crtcs == 0)
416 return;
417
418 for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
419 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
420
421 WARN_ON(READ_ONCE(vblank->enabled) &&
422 drm_core_check_feature(dev, DRIVER_MODESET));
423
424 del_timer_sync(&vblank->disable_timer);
425 }
426
427 kfree(dev->vblank);
428
429 dev->num_crtcs = 0;
430}
431
432/**
433 * drm_vblank_init - initialize vblank support
434 * @dev: DRM device
435 * @num_crtcs: number of CRTCs supported by @dev
436 *
437 * This function initializes vblank support for @num_crtcs display pipelines.
438 * Cleanup is handled by the DRM core, or through calling drm_dev_fini() for
439 * drivers with a &drm_driver.release callback.
440 *
441 * Returns:
442 * Zero on success or a negative error code on failure.
443 */
444int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
445{
446 int ret = -ENOMEM;
447 unsigned int i;
448
449 spin_lock_init(&dev->vbl_lock);
450 spin_lock_init(&dev->vblank_time_lock);
451
452 dev->num_crtcs = num_crtcs;
453
454 dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
455 if (!dev->vblank)
456 goto err;
457
458 for (i = 0; i < num_crtcs; i++) {
459 struct drm_vblank_crtc *vblank = &dev->vblank[i];
460
461 vblank->dev = dev;
462 vblank->pipe = i;
463 init_waitqueue_head(&vblank->queue);
464 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
465 seqlock_init(&vblank->seqlock);
466 }
467
468 DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
469
470 /* Driver specific high-precision vblank timestamping supported? */
471 if (dev->driver->get_vblank_timestamp)
472 DRM_INFO("Driver supports precise vblank timestamp query.\n");
473 else
474 DRM_INFO("No driver support for vblank timestamp query.\n");
475
476 /* Must have precise timestamping for reliable vblank instant disable */
477 if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
478 dev->vblank_disable_immediate = false;
479 DRM_INFO("Setting vblank_disable_immediate to false because "
480 "get_vblank_timestamp == NULL\n");
481 }
482
483 return 0;
484
485err:
486 dev->num_crtcs = 0;
487 return ret;
488}
489EXPORT_SYMBOL(drm_vblank_init);
490
491/**
492 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
493 * @crtc: which CRTC's vblank waitqueue to retrieve
494 *
495 * This function returns a pointer to the vblank waitqueue for the CRTC.
496 * Drivers can use this to implement vblank waits using wait_event() and related
497 * functions.
498 */
499wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
500{
501 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
502}
503EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
504
505
506/**
507 * drm_calc_timestamping_constants - calculate vblank timestamp constants
508 * @crtc: drm_crtc whose timestamp constants should be updated.
509 * @mode: display mode containing the scanout timings
510 *
511 * Calculate and store various constants which are later needed by vblank and
512 * swap-completion timestamping, e.g, by
513 * drm_calc_vbltimestamp_from_scanoutpos(). They are derived from CRTC's true
514 * scanout timing, so they take things like panel scaling or other adjustments
515 * into account.
516 */
517void drm_calc_timestamping_constants(struct drm_crtc *crtc,
518 const struct drm_display_mode *mode)
519{
520 struct drm_device *dev = crtc->dev;
521 unsigned int pipe = drm_crtc_index(crtc);
522 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
523 int linedur_ns = 0, framedur_ns = 0;
524 int dotclock = mode->crtc_clock;
525
526 if (!dev->num_crtcs)
527 return;
528
529 if (WARN_ON(pipe >= dev->num_crtcs))
530 return;
531
532 /* Valid dotclock? */
533 if (dotclock > 0) {
534 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
535
536 /*
537 * Convert scanline length in pixels and video
538 * dot clock to line duration and frame duration
539 * in nanoseconds:
540 */
541 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
542 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
543
544 /*
545 * Fields of interlaced scanout modes are only half a frame duration.
546 */
547 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
548 framedur_ns /= 2;
549 } else
550 DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
551 crtc->base.id);
552
553 vblank->linedur_ns = linedur_ns;
554 vblank->framedur_ns = framedur_ns;
555 vblank->hwmode = *mode;
556
557 DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
558 crtc->base.id, mode->crtc_htotal,
559 mode->crtc_vtotal, mode->crtc_vdisplay);
560 DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
561 crtc->base.id, dotclock, framedur_ns, linedur_ns);
562}
563EXPORT_SYMBOL(drm_calc_timestamping_constants);
564
565/**
566 * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper
567 * @dev: DRM device
568 * @pipe: index of CRTC whose vblank timestamp to retrieve
569 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
570 * On return contains true maximum error of timestamp
571 * @vblank_time: Pointer to time which should receive the timestamp
572 * @in_vblank_irq:
573 * True when called from drm_crtc_handle_vblank(). Some drivers
574 * need to apply some workarounds for gpu-specific vblank irq quirks
575 * if flag is set.
576 *
577 * Implements calculation of exact vblank timestamps from given drm_display_mode
578 * timings and current video scanout position of a CRTC. This can be directly
579 * used as the &drm_driver.get_vblank_timestamp implementation of a kms driver
580 * if &drm_driver.get_scanout_position is implemented.
581 *
582 * The current implementation only handles standard video modes. For double scan
583 * and interlaced modes the driver is supposed to adjust the hardware mode
584 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
585 * match the scanout position reported.
586 *
587 * Note that atomic drivers must call drm_calc_timestamping_constants() before
588 * enabling a CRTC. The atomic helpers already take care of that in
589 * drm_atomic_helper_update_legacy_modeset_state().
590 *
591 * Returns:
592 *
593 * Returns true on success, and false on failure, i.e. when no accurate
594 * timestamp could be acquired.
595 */
596bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
597 unsigned int pipe,
598 int *max_error,
599 ktime_t *vblank_time,
600 bool in_vblank_irq)
601{
602 struct timespec64 ts_etime, ts_vblank_time;
603 ktime_t stime, etime;
604 bool vbl_status;
605 struct drm_crtc *crtc;
606 const struct drm_display_mode *mode;
607 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
608 int vpos, hpos, i;
609 int delta_ns, duration_ns;
610
611 if (!drm_core_check_feature(dev, DRIVER_MODESET))
612 return false;
613
614 crtc = drm_crtc_from_index(dev, pipe);
615
616 if (pipe >= dev->num_crtcs || !crtc) {
617 DRM_ERROR("Invalid crtc %u\n", pipe);
618 return false;
619 }
620
621 /* Scanout position query not supported? Should not happen. */
622 if (!dev->driver->get_scanout_position) {
623 DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
624 return false;
625 }
626
627 if (drm_drv_uses_atomic_modeset(dev))
628 mode = &vblank->hwmode;
629 else
630 mode = &crtc->hwmode;
631
632 /* If mode timing undefined, just return as no-op:
633 * Happens during initial modesetting of a crtc.
634 */
635 if (mode->crtc_clock == 0) {
636 DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
637 WARN_ON_ONCE(drm_drv_uses_atomic_modeset(dev));
638
639 return false;
640 }
641
642 /* Get current scanout position with system timestamp.
643 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
644 * if single query takes longer than max_error nanoseconds.
645 *
646 * This guarantees a tight bound on maximum error if
647 * code gets preempted or delayed for some reason.
648 */
649 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
650 /*
651 * Get vertical and horizontal scanout position vpos, hpos,
652 * and bounding timestamps stime, etime, pre/post query.
653 */
654 vbl_status = dev->driver->get_scanout_position(dev, pipe,
655 in_vblank_irq,
656 &vpos, &hpos,
657 &stime, &etime,
658 mode);
659
660 /* Return as no-op if scanout query unsupported or failed. */
661 if (!vbl_status) {
662 DRM_DEBUG("crtc %u : scanoutpos query failed.\n",
663 pipe);
664 return false;
665 }
666
667 /* Compute uncertainty in timestamp of scanout position query. */
668 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
669
670 /* Accept result with < max_error nsecs timing uncertainty. */
671 if (duration_ns <= *max_error)
672 break;
673 }
674
675 /* Noisy system timing? */
676 if (i == DRM_TIMESTAMP_MAXRETRIES) {
677 DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
678 pipe, duration_ns/1000, *max_error/1000, i);
679 }
680
681 /* Return upper bound of timestamp precision error. */
682 *max_error = duration_ns;
683
684 /* Convert scanout position into elapsed time at raw_time query
685 * since start of scanout at first display scanline. delta_ns
686 * can be negative if start of scanout hasn't happened yet.
687 */
688 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
689 mode->crtc_clock);
690
691 /* Subtract time delta from raw timestamp to get final
692 * vblank_time timestamp for end of vblank.
693 */
694 *vblank_time = ktime_sub_ns(etime, delta_ns);
695
696 if ((drm_debug & DRM_UT_VBL) == 0)
697 return true;
698
699 ts_etime = ktime_to_timespec64(etime);
700 ts_vblank_time = ktime_to_timespec64(*vblank_time);
701
702 DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
703 pipe, hpos, vpos,
704 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
705 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
706 duration_ns / 1000, i);
707
708 return true;
709}
710EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
711
712/**
713 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
714 * vblank interval
715 * @dev: DRM device
716 * @pipe: index of CRTC whose vblank timestamp to retrieve
717 * @tvblank: Pointer to target time which should receive the timestamp
718 * @in_vblank_irq:
719 * True when called from drm_crtc_handle_vblank(). Some drivers
720 * need to apply some workarounds for gpu-specific vblank irq quirks
721 * if flag is set.
722 *
723 * Fetches the system timestamp corresponding to the time of the most recent
724 * vblank interval on specified CRTC. May call into kms-driver to
725 * compute the timestamp with a high-precision GPU specific method.
726 *
727 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
728 * call, i.e., it isn't very precisely locked to the true vblank.
729 *
730 * Returns:
731 * True if timestamp is considered to be very precise, false otherwise.
732 */
733static bool
734drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
735 ktime_t *tvblank, bool in_vblank_irq)
736{
737 bool ret = false;
738
739 /* Define requested maximum error on timestamps (nanoseconds). */
740 int max_error = (int) drm_timestamp_precision * 1000;
741
742 /* Query driver if possible and precision timestamping enabled. */
743 if (dev->driver->get_vblank_timestamp && (max_error > 0))
744 ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
745 tvblank, in_vblank_irq);
746
747 /* GPU high precision timestamp query unsupported or failed.
748 * Return current monotonic/gettimeofday timestamp as best estimate.
749 */
750 if (!ret)
751 *tvblank = ktime_get();
752
753 return ret;
754}
755
756/**
757 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
758 * @crtc: which counter to retrieve
759 *
760 * Fetches the "cooked" vblank count value that represents the number of
761 * vblank events since the system was booted, including lost events due to
762 * modesetting activity. Note that this timer isn't correct against a racing
763 * vblank interrupt (since it only reports the software vblank counter), see
764 * drm_crtc_accurate_vblank_count() for such use-cases.
765 *
766 * Returns:
767 * The software vblank counter.
768 */
769u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
770{
771 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
772}
773EXPORT_SYMBOL(drm_crtc_vblank_count);
774
775/**
776 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
777 * system timestamp corresponding to that vblank counter value.
778 * @dev: DRM device
779 * @pipe: index of CRTC whose counter to retrieve
780 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
781 *
782 * Fetches the "cooked" vblank count value that represents the number of
783 * vblank events since the system was booted, including lost events due to
784 * modesetting activity. Returns corresponding system timestamp of the time
785 * of the vblank interval that corresponds to the current vblank counter value.
786 *
787 * This is the legacy version of drm_crtc_vblank_count_and_time().
788 */
789static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
790 ktime_t *vblanktime)
791{
792 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
793 u64 vblank_count;
794 unsigned int seq;
795
796 if (WARN_ON(pipe >= dev->num_crtcs)) {
797 *vblanktime = 0;
798 return 0;
799 }
800
801 do {
802 seq = read_seqbegin(&vblank->seqlock);
803 vblank_count = vblank->count;
804 *vblanktime = vblank->time;
805 } while (read_seqretry(&vblank->seqlock, seq));
806
807 return vblank_count;
808}
809
810/**
811 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
812 * and the system timestamp corresponding to that vblank counter value
813 * @crtc: which counter to retrieve
814 * @vblanktime: Pointer to time to receive the vblank timestamp.
815 *
816 * Fetches the "cooked" vblank count value that represents the number of
817 * vblank events since the system was booted, including lost events due to
818 * modesetting activity. Returns corresponding system timestamp of the time
819 * of the vblank interval that corresponds to the current vblank counter value.
820 */
821u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
822 ktime_t *vblanktime)
823{
824 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
825 vblanktime);
826}
827EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
828
829static void send_vblank_event(struct drm_device *dev,
830 struct drm_pending_vblank_event *e,
831 u64 seq, ktime_t now)
832{
833 struct timespec64 tv;
834
835 switch (e->event.base.type) {
836 case DRM_EVENT_VBLANK:
837 case DRM_EVENT_FLIP_COMPLETE:
838 tv = ktime_to_timespec64(now);
839 e->event.vbl.sequence = seq;
840 /*
841 * e->event is a user space structure, with hardcoded unsigned
842 * 32-bit seconds/microseconds. This is safe as we always use
843 * monotonic timestamps since linux-4.15
844 */
845 e->event.vbl.tv_sec = tv.tv_sec;
846 e->event.vbl.tv_usec = tv.tv_nsec / 1000;
847 break;
848 case DRM_EVENT_CRTC_SEQUENCE:
849 if (seq)
850 e->event.seq.sequence = seq;
851 e->event.seq.time_ns = ktime_to_ns(now);
852 break;
853 }
854 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
855 drm_send_event_locked(dev, &e->base);
856}
857
858/**
859 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
860 * @crtc: the source CRTC of the vblank event
861 * @e: the event to send
862 *
863 * A lot of drivers need to generate vblank events for the very next vblank
864 * interrupt. For example when the page flip interrupt happens when the page
865 * flip gets armed, but not when it actually executes within the next vblank
866 * period. This helper function implements exactly the required vblank arming
867 * behaviour.
868 *
869 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
870 * atomic commit must ensure that the next vblank happens at exactly the same
871 * time as the atomic commit is committed to the hardware. This function itself
872 * does **not** protect against the next vblank interrupt racing with either this
873 * function call or the atomic commit operation. A possible sequence could be:
874 *
875 * 1. Driver commits new hardware state into vblank-synchronized registers.
876 * 2. A vblank happens, committing the hardware state. Also the corresponding
877 * vblank interrupt is fired off and fully processed by the interrupt
878 * handler.
879 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
880 * 4. The event is only send out for the next vblank, which is wrong.
881 *
882 * An equivalent race can happen when the driver calls
883 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
884 *
885 * The only way to make this work safely is to prevent the vblank from firing
886 * (and the hardware from committing anything else) until the entire atomic
887 * commit sequence has run to completion. If the hardware does not have such a
888 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
889 * Instead drivers need to manually send out the event from their interrupt
890 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
891 * possible race with the hardware committing the atomic update.
892 *
893 * Caller must hold a vblank reference for the event @e acquired by a
894 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
895 */
896void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
897 struct drm_pending_vblank_event *e)
898{
899 struct drm_device *dev = crtc->dev;
900 unsigned int pipe = drm_crtc_index(crtc);
901
902 assert_spin_locked(&dev->event_lock);
903
904 e->pipe = pipe;
905 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
906 list_add_tail(&e->base.link, &dev->vblank_event_list);
907}
908EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
909
910/**
911 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
912 * @crtc: the source CRTC of the vblank event
913 * @e: the event to send
914 *
915 * Updates sequence # and timestamp on event for the most recently processed
916 * vblank, and sends it to userspace. Caller must hold event lock.
917 *
918 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
919 * situation, especially to send out events for atomic commit operations.
920 */
921void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
922 struct drm_pending_vblank_event *e)
923{
924 struct drm_device *dev = crtc->dev;
925 u64 seq;
926 unsigned int pipe = drm_crtc_index(crtc);
927 ktime_t now;
928
929 if (dev->num_crtcs > 0) {
930 seq = drm_vblank_count_and_time(dev, pipe, &now);
931 } else {
932 seq = 0;
933
934 now = ktime_get();
935 }
936 e->pipe = pipe;
937 send_vblank_event(dev, e, seq, now);
938}
939EXPORT_SYMBOL(drm_crtc_send_vblank_event);
940
941static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
942{
943 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
944 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
945
946 if (WARN_ON(!crtc))
947 return 0;
948
949 if (crtc->funcs->enable_vblank)
950 return crtc->funcs->enable_vblank(crtc);
951 }
952
953 return dev->driver->enable_vblank(dev, pipe);
954}
955
956static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
957{
958 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
959 int ret = 0;
960
961 assert_spin_locked(&dev->vbl_lock);
962
963 spin_lock(&dev->vblank_time_lock);
964
965 if (!vblank->enabled) {
966 /*
967 * Enable vblank irqs under vblank_time_lock protection.
968 * All vblank count & timestamp updates are held off
969 * until we are done reinitializing master counter and
970 * timestamps. Filtercode in drm_handle_vblank() will
971 * prevent double-accounting of same vblank interval.
972 */
973 ret = __enable_vblank(dev, pipe);
974 DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
975 if (ret) {
976 atomic_dec(&vblank->refcount);
977 } else {
978 drm_update_vblank_count(dev, pipe, 0);
979 /* drm_update_vblank_count() includes a wmb so we just
980 * need to ensure that the compiler emits the write
981 * to mark the vblank as enabled after the call
982 * to drm_update_vblank_count().
983 */
984 WRITE_ONCE(vblank->enabled, true);
985 }
986 }
987
988 spin_unlock(&dev->vblank_time_lock);
989
990 return ret;
991}
992
993static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
994{
995 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
996 unsigned long irqflags;
997 int ret = 0;
998
999 if (!dev->num_crtcs)
1000 return -EINVAL;
1001
1002 if (WARN_ON(pipe >= dev->num_crtcs))
1003 return -EINVAL;
1004
1005 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1006 /* Going from 0->1 means we have to enable interrupts again */
1007 if (atomic_add_return(1, &vblank->refcount) == 1) {
1008 ret = drm_vblank_enable(dev, pipe);
1009 } else {
1010 if (!vblank->enabled) {
1011 atomic_dec(&vblank->refcount);
1012 ret = -EINVAL;
1013 }
1014 }
1015 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1016
1017 return ret;
1018}
1019
1020/**
1021 * drm_crtc_vblank_get - get a reference count on vblank events
1022 * @crtc: which CRTC to own
1023 *
1024 * Acquire a reference count on vblank events to avoid having them disabled
1025 * while in use.
1026 *
1027 * Returns:
1028 * Zero on success or a negative error code on failure.
1029 */
1030int drm_crtc_vblank_get(struct drm_crtc *crtc)
1031{
1032 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1033}
1034EXPORT_SYMBOL(drm_crtc_vblank_get);
1035
1036static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1037{
1038 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1039
1040 if (WARN_ON(pipe >= dev->num_crtcs))
1041 return;
1042
1043 if (WARN_ON(atomic_read(&vblank->refcount) == 0))
1044 return;
1045
1046 /* Last user schedules interrupt disable */
1047 if (atomic_dec_and_test(&vblank->refcount)) {
1048 if (drm_vblank_offdelay == 0)
1049 return;
1050 else if (drm_vblank_offdelay < 0)
1051 vblank_disable_fn(&vblank->disable_timer);
1052 else if (!dev->vblank_disable_immediate)
1053 mod_timer(&vblank->disable_timer,
1054 jiffies + ((drm_vblank_offdelay * HZ)/1000));
1055 }
1056}
1057
1058/**
1059 * drm_crtc_vblank_put - give up ownership of vblank events
1060 * @crtc: which counter to give up
1061 *
1062 * Release ownership of a given vblank counter, turning off interrupts
1063 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1064 */
1065void drm_crtc_vblank_put(struct drm_crtc *crtc)
1066{
1067 drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1068}
1069EXPORT_SYMBOL(drm_crtc_vblank_put);
1070
1071/**
1072 * drm_wait_one_vblank - wait for one vblank
1073 * @dev: DRM device
1074 * @pipe: CRTC index
1075 *
1076 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1077 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1078 * due to lack of driver support or because the crtc is off.
1079 *
1080 * This is the legacy version of drm_crtc_wait_one_vblank().
1081 */
1082void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1083{
1084 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1085 int ret;
1086 u64 last;
1087
1088 if (WARN_ON(pipe >= dev->num_crtcs))
1089 return;
1090
1091 ret = drm_vblank_get(dev, pipe);
1092 if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
1093 return;
1094
1095 last = drm_vblank_count(dev, pipe);
1096
1097 ret = wait_event_timeout(vblank->queue,
1098 last != drm_vblank_count(dev, pipe),
1099 msecs_to_jiffies(100));
1100
1101 WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1102
1103 drm_vblank_put(dev, pipe);
1104}
1105EXPORT_SYMBOL(drm_wait_one_vblank);
1106
1107/**
1108 * drm_crtc_wait_one_vblank - wait for one vblank
1109 * @crtc: DRM crtc
1110 *
1111 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1112 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1113 * due to lack of driver support or because the crtc is off.
1114 */
1115void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1116{
1117 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1118}
1119EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1120
1121/**
1122 * drm_crtc_vblank_off - disable vblank events on a CRTC
1123 * @crtc: CRTC in question
1124 *
1125 * Drivers can use this function to shut down the vblank interrupt handling when
1126 * disabling a crtc. This function ensures that the latest vblank frame count is
1127 * stored so that drm_vblank_on can restore it again.
1128 *
1129 * Drivers must use this function when the hardware vblank counter can get
1130 * reset, e.g. when suspending or disabling the @crtc in general.
1131 */
1132void drm_crtc_vblank_off(struct drm_crtc *crtc)
1133{
1134 struct drm_device *dev = crtc->dev;
1135 unsigned int pipe = drm_crtc_index(crtc);
1136 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1137 struct drm_pending_vblank_event *e, *t;
1138
1139 ktime_t now;
1140 unsigned long irqflags;
1141 u64 seq;
1142
1143 if (WARN_ON(pipe >= dev->num_crtcs))
1144 return;
1145
1146 spin_lock_irqsave(&dev->event_lock, irqflags);
1147
1148 spin_lock(&dev->vbl_lock);
1149 DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1150 pipe, vblank->enabled, vblank->inmodeset);
1151
1152 /* Avoid redundant vblank disables without previous
1153 * drm_crtc_vblank_on(). */
1154 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1155 drm_vblank_disable_and_save(dev, pipe);
1156
1157 wake_up(&vblank->queue);
1158
1159 /*
1160 * Prevent subsequent drm_vblank_get() from re-enabling
1161 * the vblank interrupt by bumping the refcount.
1162 */
1163 if (!vblank->inmodeset) {
1164 atomic_inc(&vblank->refcount);
1165 vblank->inmodeset = 1;
1166 }
1167 spin_unlock(&dev->vbl_lock);
1168
1169 /* Send any queued vblank events, lest the natives grow disquiet */
1170 seq = drm_vblank_count_and_time(dev, pipe, &now);
1171
1172 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1173 if (e->pipe != pipe)
1174 continue;
1175 DRM_DEBUG("Sending premature vblank event on disable: "
1176 "wanted %llu, current %llu\n",
1177 e->sequence, seq);
1178 list_del(&e->base.link);
1179 drm_vblank_put(dev, pipe);
1180 send_vblank_event(dev, e, seq, now);
1181 }
1182 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1183
1184 /* Will be reset by the modeset helpers when re-enabling the crtc by
1185 * calling drm_calc_timestamping_constants(). */
1186 vblank->hwmode.crtc_clock = 0;
1187}
1188EXPORT_SYMBOL(drm_crtc_vblank_off);
1189
1190/**
1191 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1192 * @crtc: CRTC in question
1193 *
1194 * Drivers can use this function to reset the vblank state to off at load time.
1195 * Drivers should use this together with the drm_crtc_vblank_off() and
1196 * drm_crtc_vblank_on() functions. The difference compared to
1197 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1198 * and hence doesn't need to call any driver hooks.
1199 *
1200 * This is useful for recovering driver state e.g. on driver load, or on resume.
1201 */
1202void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1203{
1204 struct drm_device *dev = crtc->dev;
1205 unsigned long irqflags;
1206 unsigned int pipe = drm_crtc_index(crtc);
1207 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1208
1209 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1210 /*
1211 * Prevent subsequent drm_vblank_get() from enabling the vblank
1212 * interrupt by bumping the refcount.
1213 */
1214 if (!vblank->inmodeset) {
1215 atomic_inc(&vblank->refcount);
1216 vblank->inmodeset = 1;
1217 }
1218 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1219
1220 WARN_ON(!list_empty(&dev->vblank_event_list));
1221}
1222EXPORT_SYMBOL(drm_crtc_vblank_reset);
1223
1224/**
1225 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1226 * @crtc: CRTC in question
1227 * @max_vblank_count: max hardware vblank counter value
1228 *
1229 * Update the maximum hardware vblank counter value for @crtc
1230 * at runtime. Useful for hardware where the operation of the
1231 * hardware vblank counter depends on the currently active
1232 * display configuration.
1233 *
1234 * For example, if the hardware vblank counter does not work
1235 * when a specific connector is active the maximum can be set
1236 * to zero. And when that specific connector isn't active the
1237 * maximum can again be set to the appropriate non-zero value.
1238 *
1239 * If used, must be called before drm_vblank_on().
1240 */
1241void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1242 u32 max_vblank_count)
1243{
1244 struct drm_device *dev = crtc->dev;
1245 unsigned int pipe = drm_crtc_index(crtc);
1246 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1247
1248 WARN_ON(dev->max_vblank_count);
1249 WARN_ON(!READ_ONCE(vblank->inmodeset));
1250
1251 vblank->max_vblank_count = max_vblank_count;
1252}
1253EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1254
1255/**
1256 * drm_crtc_vblank_on - enable vblank events on a CRTC
1257 * @crtc: CRTC in question
1258 *
1259 * This functions restores the vblank interrupt state captured with
1260 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1261 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1262 * unbalanced and so can also be unconditionally called in driver load code to
1263 * reflect the current hardware state of the crtc.
1264 */
1265void drm_crtc_vblank_on(struct drm_crtc *crtc)
1266{
1267 struct drm_device *dev = crtc->dev;
1268 unsigned int pipe = drm_crtc_index(crtc);
1269 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1270 unsigned long irqflags;
1271
1272 if (WARN_ON(pipe >= dev->num_crtcs))
1273 return;
1274
1275 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1276 DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1277 pipe, vblank->enabled, vblank->inmodeset);
1278
1279 /* Drop our private "prevent drm_vblank_get" refcount */
1280 if (vblank->inmodeset) {
1281 atomic_dec(&vblank->refcount);
1282 vblank->inmodeset = 0;
1283 }
1284
1285 drm_reset_vblank_timestamp(dev, pipe);
1286
1287 /*
1288 * re-enable interrupts if there are users left, or the
1289 * user wishes vblank interrupts to be enabled all the time.
1290 */
1291 if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1292 WARN_ON(drm_vblank_enable(dev, pipe));
1293 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1294}
1295EXPORT_SYMBOL(drm_crtc_vblank_on);
1296
1297/**
1298 * drm_vblank_restore - estimate missed vblanks and update vblank count.
1299 * @dev: DRM device
1300 * @pipe: CRTC index
1301 *
1302 * Power manamement features can cause frame counter resets between vblank
1303 * disable and enable. Drivers can use this function in their
1304 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1305 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1306 * vblank counter.
1307 *
1308 * This function is the legacy version of drm_crtc_vblank_restore().
1309 */
1310void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1311{
1312 ktime_t t_vblank;
1313 struct drm_vblank_crtc *vblank;
1314 int framedur_ns;
1315 u64 diff_ns;
1316 u32 cur_vblank, diff = 1;
1317 int count = DRM_TIMESTAMP_MAXRETRIES;
1318
1319 if (WARN_ON(pipe >= dev->num_crtcs))
1320 return;
1321
1322 assert_spin_locked(&dev->vbl_lock);
1323 assert_spin_locked(&dev->vblank_time_lock);
1324
1325 vblank = &dev->vblank[pipe];
1326 WARN_ONCE((drm_debug & DRM_UT_VBL) && !vblank->framedur_ns,
1327 "Cannot compute missed vblanks without frame duration\n");
1328 framedur_ns = vblank->framedur_ns;
1329
1330 do {
1331 cur_vblank = __get_vblank_counter(dev, pipe);
1332 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1333 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1334
1335 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1336 if (framedur_ns)
1337 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1338
1339
1340 DRM_DEBUG_VBL("missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1341 diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1342 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
1343}
1344EXPORT_SYMBOL(drm_vblank_restore);
1345
1346/**
1347 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1348 * @crtc: CRTC in question
1349 *
1350 * Power manamement features can cause frame counter resets between vblank
1351 * disable and enable. Drivers can use this function in their
1352 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1353 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1354 * vblank counter.
1355 */
1356void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1357{
1358 drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1359}
1360EXPORT_SYMBOL(drm_crtc_vblank_restore);
1361
1362static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1363 unsigned int pipe)
1364{
1365 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1366
1367 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1368 if (!dev->num_crtcs)
1369 return;
1370
1371 if (WARN_ON(pipe >= dev->num_crtcs))
1372 return;
1373
1374 /*
1375 * To avoid all the problems that might happen if interrupts
1376 * were enabled/disabled around or between these calls, we just
1377 * have the kernel take a reference on the CRTC (just once though
1378 * to avoid corrupting the count if multiple, mismatch calls occur),
1379 * so that interrupts remain enabled in the interim.
1380 */
1381 if (!vblank->inmodeset) {
1382 vblank->inmodeset = 0x1;
1383 if (drm_vblank_get(dev, pipe) == 0)
1384 vblank->inmodeset |= 0x2;
1385 }
1386}
1387
1388static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1389 unsigned int pipe)
1390{
1391 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1392 unsigned long irqflags;
1393
1394 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1395 if (!dev->num_crtcs)
1396 return;
1397
1398 if (WARN_ON(pipe >= dev->num_crtcs))
1399 return;
1400
1401 if (vblank->inmodeset) {
1402 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1403 drm_reset_vblank_timestamp(dev, pipe);
1404 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1405
1406 if (vblank->inmodeset & 0x2)
1407 drm_vblank_put(dev, pipe);
1408
1409 vblank->inmodeset = 0;
1410 }
1411}
1412
1413int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1414 struct drm_file *file_priv)
1415{
1416 struct drm_modeset_ctl *modeset = data;
1417 unsigned int pipe;
1418
1419 /* If drm_vblank_init() hasn't been called yet, just no-op */
1420 if (!dev->num_crtcs)
1421 return 0;
1422
1423 /* KMS drivers handle this internally */
1424 if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1425 return 0;
1426
1427 pipe = modeset->crtc;
1428 if (pipe >= dev->num_crtcs)
1429 return -EINVAL;
1430
1431 switch (modeset->cmd) {
1432 case _DRM_PRE_MODESET:
1433 drm_legacy_vblank_pre_modeset(dev, pipe);
1434 break;
1435 case _DRM_POST_MODESET:
1436 drm_legacy_vblank_post_modeset(dev, pipe);
1437 break;
1438 default:
1439 return -EINVAL;
1440 }
1441
1442 return 0;
1443}
1444
1445static inline bool vblank_passed(u64 seq, u64 ref)
1446{
1447 return (seq - ref) <= (1 << 23);
1448}
1449
1450static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1451 u64 req_seq,
1452 union drm_wait_vblank *vblwait,
1453 struct drm_file *file_priv)
1454{
1455 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1456 struct drm_pending_vblank_event *e;
1457 ktime_t now;
1458 unsigned long flags;
1459 u64 seq;
1460 int ret;
1461
1462 e = kzalloc(sizeof(*e), GFP_KERNEL);
1463 if (e == NULL) {
1464 ret = -ENOMEM;
1465 goto err_put;
1466 }
1467
1468 e->pipe = pipe;
1469 e->event.base.type = DRM_EVENT_VBLANK;
1470 e->event.base.length = sizeof(e->event.vbl);
1471 e->event.vbl.user_data = vblwait->request.signal;
1472 e->event.vbl.crtc_id = 0;
1473 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1474 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1475 if (crtc)
1476 e->event.vbl.crtc_id = crtc->base.id;
1477 }
1478
1479 spin_lock_irqsave(&dev->event_lock, flags);
1480
1481 /*
1482 * drm_crtc_vblank_off() might have been called after we called
1483 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1484 * vblank disable, so no need for further locking. The reference from
1485 * drm_vblank_get() protects against vblank disable from another source.
1486 */
1487 if (!READ_ONCE(vblank->enabled)) {
1488 ret = -EINVAL;
1489 goto err_unlock;
1490 }
1491
1492 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1493 &e->event.base);
1494
1495 if (ret)
1496 goto err_unlock;
1497
1498 seq = drm_vblank_count_and_time(dev, pipe, &now);
1499
1500 DRM_DEBUG("event on vblank count %llu, current %llu, crtc %u\n",
1501 req_seq, seq, pipe);
1502
1503 trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1504
1505 e->sequence = req_seq;
1506 if (vblank_passed(seq, req_seq)) {
1507 drm_vblank_put(dev, pipe);
1508 send_vblank_event(dev, e, seq, now);
1509 vblwait->reply.sequence = seq;
1510 } else {
1511 /* drm_handle_vblank_events will call drm_vblank_put */
1512 list_add_tail(&e->base.link, &dev->vblank_event_list);
1513 vblwait->reply.sequence = req_seq;
1514 }
1515
1516 spin_unlock_irqrestore(&dev->event_lock, flags);
1517
1518 return 0;
1519
1520err_unlock:
1521 spin_unlock_irqrestore(&dev->event_lock, flags);
1522 kfree(e);
1523err_put:
1524 drm_vblank_put(dev, pipe);
1525 return ret;
1526}
1527
1528static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1529{
1530 if (vblwait->request.sequence)
1531 return false;
1532
1533 return _DRM_VBLANK_RELATIVE ==
1534 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1535 _DRM_VBLANK_EVENT |
1536 _DRM_VBLANK_NEXTONMISS));
1537}
1538
1539/*
1540 * Widen a 32-bit param to 64-bits.
1541 *
1542 * \param narrow 32-bit value (missing upper 32 bits)
1543 * \param near 64-bit value that should be 'close' to near
1544 *
1545 * This function returns a 64-bit value using the lower 32-bits from
1546 * 'narrow' and constructing the upper 32-bits so that the result is
1547 * as close as possible to 'near'.
1548 */
1549
1550static u64 widen_32_to_64(u32 narrow, u64 near)
1551{
1552 return near + (s32) (narrow - near);
1553}
1554
1555static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1556 struct drm_wait_vblank_reply *reply)
1557{
1558 ktime_t now;
1559 struct timespec64 ts;
1560
1561 /*
1562 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1563 * to store the seconds. This is safe as we always use monotonic
1564 * timestamps since linux-4.15.
1565 */
1566 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1567 ts = ktime_to_timespec64(now);
1568 reply->tval_sec = (u32)ts.tv_sec;
1569 reply->tval_usec = ts.tv_nsec / 1000;
1570}
1571
1572int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1573 struct drm_file *file_priv)
1574{
1575 struct drm_crtc *crtc;
1576 struct drm_vblank_crtc *vblank;
1577 union drm_wait_vblank *vblwait = data;
1578 int ret;
1579 u64 req_seq, seq;
1580 unsigned int pipe_index;
1581 unsigned int flags, pipe, high_pipe;
1582
1583 if (!dev->irq_enabled)
1584 return -EINVAL;
1585
1586 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1587 return -EINVAL;
1588
1589 if (vblwait->request.type &
1590 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1591 _DRM_VBLANK_HIGH_CRTC_MASK)) {
1592 DRM_DEBUG("Unsupported type value 0x%x, supported mask 0x%x\n",
1593 vblwait->request.type,
1594 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1595 _DRM_VBLANK_HIGH_CRTC_MASK));
1596 return -EINVAL;
1597 }
1598
1599 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1600 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1601 if (high_pipe)
1602 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1603 else
1604 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1605
1606 /* Convert lease-relative crtc index into global crtc index */
1607 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1608 pipe = 0;
1609 drm_for_each_crtc(crtc, dev) {
1610 if (drm_lease_held(file_priv, crtc->base.id)) {
1611 if (pipe_index == 0)
1612 break;
1613 pipe_index--;
1614 }
1615 pipe++;
1616 }
1617 } else {
1618 pipe = pipe_index;
1619 }
1620
1621 if (pipe >= dev->num_crtcs)
1622 return -EINVAL;
1623
1624 vblank = &dev->vblank[pipe];
1625
1626 /* If the counter is currently enabled and accurate, short-circuit
1627 * queries to return the cached timestamp of the last vblank.
1628 */
1629 if (dev->vblank_disable_immediate &&
1630 drm_wait_vblank_is_query(vblwait) &&
1631 READ_ONCE(vblank->enabled)) {
1632 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1633 return 0;
1634 }
1635
1636 ret = drm_vblank_get(dev, pipe);
1637 if (ret) {
1638 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1639 return ret;
1640 }
1641 seq = drm_vblank_count(dev, pipe);
1642
1643 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1644 case _DRM_VBLANK_RELATIVE:
1645 req_seq = seq + vblwait->request.sequence;
1646 vblwait->request.sequence = req_seq;
1647 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1648 break;
1649 case _DRM_VBLANK_ABSOLUTE:
1650 req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1651 break;
1652 default:
1653 ret = -EINVAL;
1654 goto done;
1655 }
1656
1657 if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1658 vblank_passed(seq, req_seq)) {
1659 req_seq = seq + 1;
1660 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1661 vblwait->request.sequence = req_seq;
1662 }
1663
1664 if (flags & _DRM_VBLANK_EVENT) {
1665 /* must hold on to the vblank ref until the event fires
1666 * drm_vblank_put will be called asynchronously
1667 */
1668 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1669 }
1670
1671 if (req_seq != seq) {
1672 int wait;
1673
1674 DRM_DEBUG("waiting on vblank count %llu, crtc %u\n",
1675 req_seq, pipe);
1676 wait = wait_event_interruptible_timeout(vblank->queue,
1677 vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1678 !READ_ONCE(vblank->enabled),
1679 msecs_to_jiffies(3000));
1680
1681 switch (wait) {
1682 case 0:
1683 /* timeout */
1684 ret = -EBUSY;
1685 break;
1686 case -ERESTARTSYS:
1687 /* interrupted by signal */
1688 ret = -EINTR;
1689 break;
1690 default:
1691 ret = 0;
1692 break;
1693 }
1694 }
1695
1696 if (ret != -EINTR) {
1697 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1698
1699 DRM_DEBUG("crtc %d returning %u to client\n",
1700 pipe, vblwait->reply.sequence);
1701 } else {
1702 DRM_DEBUG("crtc %d vblank wait interrupted by signal\n", pipe);
1703 }
1704
1705done:
1706 drm_vblank_put(dev, pipe);
1707 return ret;
1708}
1709
1710static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1711{
1712 struct drm_pending_vblank_event *e, *t;
1713 ktime_t now;
1714 u64 seq;
1715
1716 assert_spin_locked(&dev->event_lock);
1717
1718 seq = drm_vblank_count_and_time(dev, pipe, &now);
1719
1720 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1721 if (e->pipe != pipe)
1722 continue;
1723 if (!vblank_passed(seq, e->sequence))
1724 continue;
1725
1726 DRM_DEBUG("vblank event on %llu, current %llu\n",
1727 e->sequence, seq);
1728
1729 list_del(&e->base.link);
1730 drm_vblank_put(dev, pipe);
1731 send_vblank_event(dev, e, seq, now);
1732 }
1733
1734 trace_drm_vblank_event(pipe, seq);
1735}
1736
1737/**
1738 * drm_handle_vblank - handle a vblank event
1739 * @dev: DRM device
1740 * @pipe: index of CRTC where this event occurred
1741 *
1742 * Drivers should call this routine in their vblank interrupt handlers to
1743 * update the vblank counter and send any signals that may be pending.
1744 *
1745 * This is the legacy version of drm_crtc_handle_vblank().
1746 */
1747bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1748{
1749 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1750 unsigned long irqflags;
1751 bool disable_irq;
1752
1753 if (WARN_ON_ONCE(!dev->num_crtcs))
1754 return false;
1755
1756 if (WARN_ON(pipe >= dev->num_crtcs))
1757 return false;
1758
1759 spin_lock_irqsave(&dev->event_lock, irqflags);
1760
1761 /* Need timestamp lock to prevent concurrent execution with
1762 * vblank enable/disable, as this would cause inconsistent
1763 * or corrupted timestamps and vblank counts.
1764 */
1765 spin_lock(&dev->vblank_time_lock);
1766
1767 /* Vblank irq handling disabled. Nothing to do. */
1768 if (!vblank->enabled) {
1769 spin_unlock(&dev->vblank_time_lock);
1770 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1771 return false;
1772 }
1773
1774 drm_update_vblank_count(dev, pipe, true);
1775
1776 spin_unlock(&dev->vblank_time_lock);
1777
1778 wake_up(&vblank->queue);
1779
1780 /* With instant-off, we defer disabling the interrupt until after
1781 * we finish processing the following vblank after all events have
1782 * been signaled. The disable has to be last (after
1783 * drm_handle_vblank_events) so that the timestamp is always accurate.
1784 */
1785 disable_irq = (dev->vblank_disable_immediate &&
1786 drm_vblank_offdelay > 0 &&
1787 !atomic_read(&vblank->refcount));
1788
1789 drm_handle_vblank_events(dev, pipe);
1790
1791 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1792
1793 if (disable_irq)
1794 vblank_disable_fn(&vblank->disable_timer);
1795
1796 return true;
1797}
1798EXPORT_SYMBOL(drm_handle_vblank);
1799
1800/**
1801 * drm_crtc_handle_vblank - handle a vblank event
1802 * @crtc: where this event occurred
1803 *
1804 * Drivers should call this routine in their vblank interrupt handlers to
1805 * update the vblank counter and send any signals that may be pending.
1806 *
1807 * This is the native KMS version of drm_handle_vblank().
1808 *
1809 * Returns:
1810 * True if the event was successfully handled, false on failure.
1811 */
1812bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1813{
1814 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1815}
1816EXPORT_SYMBOL(drm_crtc_handle_vblank);
1817
1818/*
1819 * Get crtc VBLANK count.
1820 *
1821 * \param dev DRM device
1822 * \param data user arguement, pointing to a drm_crtc_get_sequence structure.
1823 * \param file_priv drm file private for the user's open file descriptor
1824 */
1825
1826int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1827 struct drm_file *file_priv)
1828{
1829 struct drm_crtc *crtc;
1830 struct drm_vblank_crtc *vblank;
1831 int pipe;
1832 struct drm_crtc_get_sequence *get_seq = data;
1833 ktime_t now;
1834 bool vblank_enabled;
1835 int ret;
1836
1837 if (!drm_core_check_feature(dev, DRIVER_MODESET))
1838 return -EOPNOTSUPP;
1839
1840 if (!dev->irq_enabled)
1841 return -EINVAL;
1842
1843 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
1844 if (!crtc)
1845 return -ENOENT;
1846
1847 pipe = drm_crtc_index(crtc);
1848
1849 vblank = &dev->vblank[pipe];
1850 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
1851
1852 if (!vblank_enabled) {
1853 ret = drm_crtc_vblank_get(crtc);
1854 if (ret) {
1855 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1856 return ret;
1857 }
1858 }
1859 drm_modeset_lock(&crtc->mutex, NULL);
1860 if (crtc->state)
1861 get_seq->active = crtc->state->enable;
1862 else
1863 get_seq->active = crtc->enabled;
1864 drm_modeset_unlock(&crtc->mutex);
1865 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1866 get_seq->sequence_ns = ktime_to_ns(now);
1867 if (!vblank_enabled)
1868 drm_crtc_vblank_put(crtc);
1869 return 0;
1870}
1871
1872/*
1873 * Queue a event for VBLANK sequence
1874 *
1875 * \param dev DRM device
1876 * \param data user arguement, pointing to a drm_crtc_queue_sequence structure.
1877 * \param file_priv drm file private for the user's open file descriptor
1878 */
1879
1880int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
1881 struct drm_file *file_priv)
1882{
1883 struct drm_crtc *crtc;
1884 struct drm_vblank_crtc *vblank;
1885 int pipe;
1886 struct drm_crtc_queue_sequence *queue_seq = data;
1887 ktime_t now;
1888 struct drm_pending_vblank_event *e;
1889 u32 flags;
1890 u64 seq;
1891 u64 req_seq;
1892 int ret;
1893 unsigned long spin_flags;
1894
1895 if (!drm_core_check_feature(dev, DRIVER_MODESET))
1896 return -EOPNOTSUPP;
1897
1898 if (!dev->irq_enabled)
1899 return -EINVAL;
1900
1901 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
1902 if (!crtc)
1903 return -ENOENT;
1904
1905 flags = queue_seq->flags;
1906 /* Check valid flag bits */
1907 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
1908 DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
1909 return -EINVAL;
1910
1911 pipe = drm_crtc_index(crtc);
1912
1913 vblank = &dev->vblank[pipe];
1914
1915 e = kzalloc(sizeof(*e), GFP_KERNEL);
1916 if (e == NULL)
1917 return -ENOMEM;
1918
1919 ret = drm_crtc_vblank_get(crtc);
1920 if (ret) {
1921 DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1922 goto err_free;
1923 }
1924
1925 seq = drm_vblank_count_and_time(dev, pipe, &now);
1926 req_seq = queue_seq->sequence;
1927
1928 if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
1929 req_seq += seq;
1930
1931 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq))
1932 req_seq = seq + 1;
1933
1934 e->pipe = pipe;
1935 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
1936 e->event.base.length = sizeof(e->event.seq);
1937 e->event.seq.user_data = queue_seq->user_data;
1938
1939 spin_lock_irqsave(&dev->event_lock, spin_flags);
1940
1941 /*
1942 * drm_crtc_vblank_off() might have been called after we called
1943 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1944 * vblank disable, so no need for further locking. The reference from
1945 * drm_crtc_vblank_get() protects against vblank disable from another source.
1946 */
1947 if (!READ_ONCE(vblank->enabled)) {
1948 ret = -EINVAL;
1949 goto err_unlock;
1950 }
1951
1952 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1953 &e->event.base);
1954
1955 if (ret)
1956 goto err_unlock;
1957
1958 e->sequence = req_seq;
1959
1960 if (vblank_passed(seq, req_seq)) {
1961 drm_crtc_vblank_put(crtc);
1962 send_vblank_event(dev, e, seq, now);
1963 queue_seq->sequence = seq;
1964 } else {
1965 /* drm_handle_vblank_events will call drm_vblank_put */
1966 list_add_tail(&e->base.link, &dev->vblank_event_list);
1967 queue_seq->sequence = req_seq;
1968 }
1969
1970 spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1971 return 0;
1972
1973err_unlock:
1974 spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1975 drm_crtc_vblank_put(crtc);
1976err_free:
1977 kfree(e);
1978 return ret;
1979}