Loading...
Note: File does not exist in v3.1.
1/*
2 * drm_irq.c IRQ and vblank support
3 *
4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
5 * \author Gareth Hughes <gareth@valinux.com>
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the next
15 * paragraph) shall be included in all copies or substantial portions of the
16 * Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 * OTHER DEALINGS IN THE SOFTWARE.
25 */
26
27#include <linux/export.h>
28#include <linux/kthread.h>
29#include <linux/moduleparam.h>
30
31#include <drm/drm_crtc.h>
32#include <drm/drm_drv.h>
33#include <drm/drm_framebuffer.h>
34#include <drm/drm_managed.h>
35#include <drm/drm_modeset_helper_vtables.h>
36#include <drm/drm_print.h>
37#include <drm/drm_vblank.h>
38
39#include "drm_internal.h"
40#include "drm_trace.h"
41
42/**
43 * DOC: vblank handling
44 *
45 * From the computer's perspective, every time the monitor displays
46 * a new frame the scanout engine has "scanned out" the display image
47 * from top to bottom, one row of pixels at a time. The current row
48 * of pixels is referred to as the current scanline.
49 *
50 * In addition to the display's visible area, there's usually a couple of
51 * extra scanlines which aren't actually displayed on the screen.
52 * These extra scanlines don't contain image data and are occasionally used
53 * for features like audio and infoframes. The region made up of these
54 * scanlines is referred to as the vertical blanking region, or vblank for
55 * short.
56 *
57 * For historical reference, the vertical blanking period was designed to
58 * give the electron gun (on CRTs) enough time to move back to the top of
59 * the screen to start scanning out the next frame. Similar for horizontal
60 * blanking periods. They were designed to give the electron gun enough
61 * time to move back to the other side of the screen to start scanning the
62 * next scanline.
63 *
64 * ::
65 *
66 *
67 * physical → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
68 * top of | |
69 * display | |
70 * | New frame |
71 * | |
72 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
73 * |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
74 * |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓| updates the
75 * | | frame as it
76 * | | travels down
77 * | | ("scan out")
78 * | Old frame |
79 * | |
80 * | |
81 * | |
82 * | | physical
83 * | | bottom of
84 * vertical |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
85 * blanking ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
86 * region → ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
87 * ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
88 * start of → ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
89 * new frame
90 *
91 * "Physical top of display" is the reference point for the high-precision/
92 * corrected timestamp.
93 *
94 * On a lot of display hardware, programming needs to take effect during the
95 * vertical blanking period so that settings like gamma, the image buffer
96 * buffer to be scanned out, etc. can safely be changed without showing
97 * any visual artifacts on the screen. In some unforgiving hardware, some of
98 * this programming has to both start and end in the same vblank. To help
99 * with the timing of the hardware programming, an interrupt is usually
100 * available to notify the driver when it can start the updating of registers.
101 * The interrupt is in this context named the vblank interrupt.
102 *
103 * The vblank interrupt may be fired at different points depending on the
104 * hardware. Some hardware implementations will fire the interrupt when the
105 * new frame start, other implementations will fire the interrupt at different
106 * points in time.
107 *
108 * Vertical blanking plays a major role in graphics rendering. To achieve
109 * tear-free display, users must synchronize page flips and/or rendering to
110 * vertical blanking. The DRM API offers ioctls to perform page flips
111 * synchronized to vertical blanking and wait for vertical blanking.
112 *
113 * The DRM core handles most of the vertical blanking management logic, which
114 * involves filtering out spurious interrupts, keeping race-free blanking
115 * counters, coping with counter wrap-around and resets and keeping use counts.
116 * It relies on the driver to generate vertical blanking interrupts and
117 * optionally provide a hardware vertical blanking counter.
118 *
119 * Drivers must initialize the vertical blanking handling core with a call to
120 * drm_vblank_init(). Minimally, a driver needs to implement
121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
123 * support.
124 *
125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
126 * themselves (for instance to handle page flipping operations). The DRM core
127 * maintains a vertical blanking use count to ensure that the interrupts are not
128 * disabled while a user still needs them. To increment the use count, drivers
129 * call drm_crtc_vblank_get() and release the vblank reference again with
130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
131 * guaranteed to be enabled.
132 *
133 * On many hardware disabling the vblank interrupt cannot be done in a race-free
134 * manner, see &drm_driver.vblank_disable_immediate and
135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
136 * vblanks after a timer has expired, which can be configured through the
137 * ``vblankoffdelay`` module parameter.
138 *
139 * Drivers for hardware without support for vertical-blanking interrupts
140 * must not call drm_vblank_init(). For such drivers, atomic helpers will
141 * automatically generate fake vblank events as part of the display update.
142 * This functionality also can be controlled by the driver by enabling and
143 * disabling struct drm_crtc_state.no_vblank.
144 */
145
146/* Retry timestamp calculation up to 3 times to satisfy
147 * drm_timestamp_precision before giving up.
148 */
149#define DRM_TIMESTAMP_MAXRETRIES 3
150
151/* Threshold in nanoseconds for detection of redundant
152 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
153 */
154#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
155
156static bool
157drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
158 ktime_t *tvblank, bool in_vblank_irq);
159
160static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
161
162static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
163
164module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
165module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
166MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
167MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
168
169static void store_vblank(struct drm_device *dev, unsigned int pipe,
170 u32 vblank_count_inc,
171 ktime_t t_vblank, u32 last)
172{
173 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
174
175 assert_spin_locked(&dev->vblank_time_lock);
176
177 vblank->last = last;
178
179 write_seqlock(&vblank->seqlock);
180 vblank->time = t_vblank;
181 atomic64_add(vblank_count_inc, &vblank->count);
182 write_sequnlock(&vblank->seqlock);
183}
184
185static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
186{
187 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
188
189 return vblank->max_vblank_count ?: dev->max_vblank_count;
190}
191
192/*
193 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
194 * if there is no usable hardware frame counter available.
195 */
196static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
197{
198 drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
199 return 0;
200}
201
202static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
203{
204 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
205 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
206
207 if (drm_WARN_ON(dev, !crtc))
208 return 0;
209
210 if (crtc->funcs->get_vblank_counter)
211 return crtc->funcs->get_vblank_counter(crtc);
212 }
213#ifdef CONFIG_DRM_LEGACY
214 else if (dev->driver->get_vblank_counter) {
215 return dev->driver->get_vblank_counter(dev, pipe);
216 }
217#endif
218
219 return drm_vblank_no_hw_counter(dev, pipe);
220}
221
222/*
223 * Reset the stored timestamp for the current vblank count to correspond
224 * to the last vblank occurred.
225 *
226 * Only to be called from drm_crtc_vblank_on().
227 *
228 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
229 * device vblank fields.
230 */
231static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
232{
233 u32 cur_vblank;
234 bool rc;
235 ktime_t t_vblank;
236 int count = DRM_TIMESTAMP_MAXRETRIES;
237
238 spin_lock(&dev->vblank_time_lock);
239
240 /*
241 * sample the current counter to avoid random jumps
242 * when drm_vblank_enable() applies the diff
243 */
244 do {
245 cur_vblank = __get_vblank_counter(dev, pipe);
246 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
247 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
248
249 /*
250 * Only reinitialize corresponding vblank timestamp if high-precision query
251 * available and didn't fail. Otherwise reinitialize delayed at next vblank
252 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
253 */
254 if (!rc)
255 t_vblank = 0;
256
257 /*
258 * +1 to make sure user will never see the same
259 * vblank counter value before and after a modeset
260 */
261 store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
262
263 spin_unlock(&dev->vblank_time_lock);
264}
265
266/*
267 * Call back into the driver to update the appropriate vblank counter
268 * (specified by @pipe). Deal with wraparound, if it occurred, and
269 * update the last read value so we can deal with wraparound on the next
270 * call if necessary.
271 *
272 * Only necessary when going from off->on, to account for frames we
273 * didn't get an interrupt for.
274 *
275 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
276 * device vblank fields.
277 */
278static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
279 bool in_vblank_irq)
280{
281 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
282 u32 cur_vblank, diff;
283 bool rc;
284 ktime_t t_vblank;
285 int count = DRM_TIMESTAMP_MAXRETRIES;
286 int framedur_ns = vblank->framedur_ns;
287 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
288
289 /*
290 * Interrupts were disabled prior to this call, so deal with counter
291 * wrap if needed.
292 * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
293 * here if the register is small or we had vblank interrupts off for
294 * a long time.
295 *
296 * We repeat the hardware vblank counter & timestamp query until
297 * we get consistent results. This to prevent races between gpu
298 * updating its hardware counter while we are retrieving the
299 * corresponding vblank timestamp.
300 */
301 do {
302 cur_vblank = __get_vblank_counter(dev, pipe);
303 rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
304 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
305
306 if (max_vblank_count) {
307 /* trust the hw counter when it's around */
308 diff = (cur_vblank - vblank->last) & max_vblank_count;
309 } else if (rc && framedur_ns) {
310 u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
311
312 /*
313 * Figure out how many vblanks we've missed based
314 * on the difference in the timestamps and the
315 * frame/field duration.
316 */
317
318 drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
319 " diff_ns = %lld, framedur_ns = %d)\n",
320 pipe, (long long)diff_ns, framedur_ns);
321
322 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
323
324 if (diff == 0 && in_vblank_irq)
325 drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
326 pipe);
327 } else {
328 /* some kind of default for drivers w/o accurate vbl timestamping */
329 diff = in_vblank_irq ? 1 : 0;
330 }
331
332 /*
333 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
334 * interval? If so then vblank irqs keep running and it will likely
335 * happen that the hardware vblank counter is not trustworthy as it
336 * might reset at some point in that interval and vblank timestamps
337 * are not trustworthy either in that interval. Iow. this can result
338 * in a bogus diff >> 1 which must be avoided as it would cause
339 * random large forward jumps of the software vblank counter.
340 */
341 if (diff > 1 && (vblank->inmodeset & 0x2)) {
342 drm_dbg_vbl(dev,
343 "clamping vblank bump to 1 on crtc %u: diffr=%u"
344 " due to pre-modeset.\n", pipe, diff);
345 diff = 1;
346 }
347
348 drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
349 " current=%llu, diff=%u, hw=%u hw_last=%u\n",
350 pipe, (unsigned long long)atomic64_read(&vblank->count),
351 diff, cur_vblank, vblank->last);
352
353 if (diff == 0) {
354 drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
355 return;
356 }
357
358 /*
359 * Only reinitialize corresponding vblank timestamp if high-precision query
360 * available and didn't fail, or we were called from the vblank interrupt.
361 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
362 * for now, to mark the vblanktimestamp as invalid.
363 */
364 if (!rc && !in_vblank_irq)
365 t_vblank = 0;
366
367 store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
368}
369
370u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
371{
372 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
373 u64 count;
374
375 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
376 return 0;
377
378 count = atomic64_read(&vblank->count);
379
380 /*
381 * This read barrier corresponds to the implicit write barrier of the
382 * write seqlock in store_vblank(). Note that this is the only place
383 * where we need an explicit barrier, since all other access goes
384 * through drm_vblank_count_and_time(), which already has the required
385 * read barrier curtesy of the read seqlock.
386 */
387 smp_rmb();
388
389 return count;
390}
391
392/**
393 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
394 * @crtc: which counter to retrieve
395 *
396 * This function is similar to drm_crtc_vblank_count() but this function
397 * interpolates to handle a race with vblank interrupts using the high precision
398 * timestamping support.
399 *
400 * This is mostly useful for hardware that can obtain the scanout position, but
401 * doesn't have a hardware frame counter.
402 */
403u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
404{
405 struct drm_device *dev = crtc->dev;
406 unsigned int pipe = drm_crtc_index(crtc);
407 u64 vblank;
408 unsigned long flags;
409
410 drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
411 !crtc->funcs->get_vblank_timestamp,
412 "This function requires support for accurate vblank timestamps.");
413
414 spin_lock_irqsave(&dev->vblank_time_lock, flags);
415
416 drm_update_vblank_count(dev, pipe, false);
417 vblank = drm_vblank_count(dev, pipe);
418
419 spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
420
421 return vblank;
422}
423EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
424
425static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
426{
427 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
428 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
429
430 if (drm_WARN_ON(dev, !crtc))
431 return;
432
433 if (crtc->funcs->disable_vblank)
434 crtc->funcs->disable_vblank(crtc);
435 }
436#ifdef CONFIG_DRM_LEGACY
437 else {
438 dev->driver->disable_vblank(dev, pipe);
439 }
440#endif
441}
442
443/*
444 * Disable vblank irq's on crtc, make sure that last vblank count
445 * of hardware and corresponding consistent software vblank counter
446 * are preserved, even if there are any spurious vblank irq's after
447 * disable.
448 */
449void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
450{
451 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
452 unsigned long irqflags;
453
454 assert_spin_locked(&dev->vbl_lock);
455
456 /* Prevent vblank irq processing while disabling vblank irqs,
457 * so no updates of timestamps or count can happen after we've
458 * disabled. Needed to prevent races in case of delayed irq's.
459 */
460 spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
461
462 /*
463 * Update vblank count and disable vblank interrupts only if the
464 * interrupts were enabled. This avoids calling the ->disable_vblank()
465 * operation in atomic context with the hardware potentially runtime
466 * suspended.
467 */
468 if (!vblank->enabled)
469 goto out;
470
471 /*
472 * Update the count and timestamp to maintain the
473 * appearance that the counter has been ticking all along until
474 * this time. This makes the count account for the entire time
475 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
476 */
477 drm_update_vblank_count(dev, pipe, false);
478 __disable_vblank(dev, pipe);
479 vblank->enabled = false;
480
481out:
482 spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
483}
484
485static void vblank_disable_fn(struct timer_list *t)
486{
487 struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
488 struct drm_device *dev = vblank->dev;
489 unsigned int pipe = vblank->pipe;
490 unsigned long irqflags;
491
492 spin_lock_irqsave(&dev->vbl_lock, irqflags);
493 if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
494 drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
495 drm_vblank_disable_and_save(dev, pipe);
496 }
497 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
498}
499
500static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
501{
502 struct drm_vblank_crtc *vblank = ptr;
503
504 drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
505 drm_core_check_feature(dev, DRIVER_MODESET));
506
507 drm_vblank_destroy_worker(vblank);
508 del_timer_sync(&vblank->disable_timer);
509}
510
511/**
512 * drm_vblank_init - initialize vblank support
513 * @dev: DRM device
514 * @num_crtcs: number of CRTCs supported by @dev
515 *
516 * This function initializes vblank support for @num_crtcs display pipelines.
517 * Cleanup is handled automatically through a cleanup function added with
518 * drmm_add_action_or_reset().
519 *
520 * Returns:
521 * Zero on success or a negative error code on failure.
522 */
523int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
524{
525 int ret;
526 unsigned int i;
527
528 spin_lock_init(&dev->vbl_lock);
529 spin_lock_init(&dev->vblank_time_lock);
530
531 dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
532 if (!dev->vblank)
533 return -ENOMEM;
534
535 dev->num_crtcs = num_crtcs;
536
537 for (i = 0; i < num_crtcs; i++) {
538 struct drm_vblank_crtc *vblank = &dev->vblank[i];
539
540 vblank->dev = dev;
541 vblank->pipe = i;
542 init_waitqueue_head(&vblank->queue);
543 timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
544 seqlock_init(&vblank->seqlock);
545
546 ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
547 vblank);
548 if (ret)
549 return ret;
550
551 ret = drm_vblank_worker_init(vblank);
552 if (ret)
553 return ret;
554 }
555
556 return 0;
557}
558EXPORT_SYMBOL(drm_vblank_init);
559
560/**
561 * drm_dev_has_vblank - test if vblanking has been initialized for
562 * a device
563 * @dev: the device
564 *
565 * Drivers may call this function to test if vblank support is
566 * initialized for a device. For most hardware this means that vblanking
567 * can also be enabled.
568 *
569 * Atomic helpers use this function to initialize
570 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
571 *
572 * Returns:
573 * True if vblanking has been initialized for the given device, false
574 * otherwise.
575 */
576bool drm_dev_has_vblank(const struct drm_device *dev)
577{
578 return dev->num_crtcs != 0;
579}
580EXPORT_SYMBOL(drm_dev_has_vblank);
581
582/**
583 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
584 * @crtc: which CRTC's vblank waitqueue to retrieve
585 *
586 * This function returns a pointer to the vblank waitqueue for the CRTC.
587 * Drivers can use this to implement vblank waits using wait_event() and related
588 * functions.
589 */
590wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
591{
592 return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
593}
594EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
595
596
597/**
598 * drm_calc_timestamping_constants - calculate vblank timestamp constants
599 * @crtc: drm_crtc whose timestamp constants should be updated.
600 * @mode: display mode containing the scanout timings
601 *
602 * Calculate and store various constants which are later needed by vblank and
603 * swap-completion timestamping, e.g, by
604 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
605 * CRTC's true scanout timing, so they take things like panel scaling or
606 * other adjustments into account.
607 */
608void drm_calc_timestamping_constants(struct drm_crtc *crtc,
609 const struct drm_display_mode *mode)
610{
611 struct drm_device *dev = crtc->dev;
612 unsigned int pipe = drm_crtc_index(crtc);
613 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
614 int linedur_ns = 0, framedur_ns = 0;
615 int dotclock = mode->crtc_clock;
616
617 if (!drm_dev_has_vblank(dev))
618 return;
619
620 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
621 return;
622
623 /* Valid dotclock? */
624 if (dotclock > 0) {
625 int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
626
627 /*
628 * Convert scanline length in pixels and video
629 * dot clock to line duration and frame duration
630 * in nanoseconds:
631 */
632 linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
633 framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
634
635 /*
636 * Fields of interlaced scanout modes are only half a frame duration.
637 */
638 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
639 framedur_ns /= 2;
640 } else {
641 drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
642 crtc->base.id);
643 }
644
645 vblank->linedur_ns = linedur_ns;
646 vblank->framedur_ns = framedur_ns;
647 drm_mode_copy(&vblank->hwmode, mode);
648
649 drm_dbg_core(dev,
650 "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
651 crtc->base.id, mode->crtc_htotal,
652 mode->crtc_vtotal, mode->crtc_vdisplay);
653 drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
654 crtc->base.id, dotclock, framedur_ns, linedur_ns);
655}
656EXPORT_SYMBOL(drm_calc_timestamping_constants);
657
658/**
659 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
660 * timestamp helper
661 * @crtc: CRTC whose vblank timestamp to retrieve
662 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
663 * On return contains true maximum error of timestamp
664 * @vblank_time: Pointer to time which should receive the timestamp
665 * @in_vblank_irq:
666 * True when called from drm_crtc_handle_vblank(). Some drivers
667 * need to apply some workarounds for gpu-specific vblank irq quirks
668 * if flag is set.
669 * @get_scanout_position:
670 * Callback function to retrieve the scanout position. See
671 * @struct drm_crtc_helper_funcs.get_scanout_position.
672 *
673 * Implements calculation of exact vblank timestamps from given drm_display_mode
674 * timings and current video scanout position of a CRTC.
675 *
676 * The current implementation only handles standard video modes. For double scan
677 * and interlaced modes the driver is supposed to adjust the hardware mode
678 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
679 * match the scanout position reported.
680 *
681 * Note that atomic drivers must call drm_calc_timestamping_constants() before
682 * enabling a CRTC. The atomic helpers already take care of that in
683 * drm_atomic_helper_calc_timestamping_constants().
684 *
685 * Returns:
686 *
687 * Returns true on success, and false on failure, i.e. when no accurate
688 * timestamp could be acquired.
689 */
690bool
691drm_crtc_vblank_helper_get_vblank_timestamp_internal(
692 struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
693 bool in_vblank_irq,
694 drm_vblank_get_scanout_position_func get_scanout_position)
695{
696 struct drm_device *dev = crtc->dev;
697 unsigned int pipe = crtc->index;
698 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
699 struct timespec64 ts_etime, ts_vblank_time;
700 ktime_t stime, etime;
701 bool vbl_status;
702 const struct drm_display_mode *mode;
703 int vpos, hpos, i;
704 int delta_ns, duration_ns;
705
706 if (pipe >= dev->num_crtcs) {
707 drm_err(dev, "Invalid crtc %u\n", pipe);
708 return false;
709 }
710
711 /* Scanout position query not supported? Should not happen. */
712 if (!get_scanout_position) {
713 drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
714 return false;
715 }
716
717 if (drm_drv_uses_atomic_modeset(dev))
718 mode = &vblank->hwmode;
719 else
720 mode = &crtc->hwmode;
721
722 /* If mode timing undefined, just return as no-op:
723 * Happens during initial modesetting of a crtc.
724 */
725 if (mode->crtc_clock == 0) {
726 drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
727 pipe);
728 drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
729 return false;
730 }
731
732 /* Get current scanout position with system timestamp.
733 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
734 * if single query takes longer than max_error nanoseconds.
735 *
736 * This guarantees a tight bound on maximum error if
737 * code gets preempted or delayed for some reason.
738 */
739 for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
740 /*
741 * Get vertical and horizontal scanout position vpos, hpos,
742 * and bounding timestamps stime, etime, pre/post query.
743 */
744 vbl_status = get_scanout_position(crtc, in_vblank_irq,
745 &vpos, &hpos,
746 &stime, &etime,
747 mode);
748
749 /* Return as no-op if scanout query unsupported or failed. */
750 if (!vbl_status) {
751 drm_dbg_core(dev,
752 "crtc %u : scanoutpos query failed.\n",
753 pipe);
754 return false;
755 }
756
757 /* Compute uncertainty in timestamp of scanout position query. */
758 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
759
760 /* Accept result with < max_error nsecs timing uncertainty. */
761 if (duration_ns <= *max_error)
762 break;
763 }
764
765 /* Noisy system timing? */
766 if (i == DRM_TIMESTAMP_MAXRETRIES) {
767 drm_dbg_core(dev,
768 "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
769 pipe, duration_ns / 1000, *max_error / 1000, i);
770 }
771
772 /* Return upper bound of timestamp precision error. */
773 *max_error = duration_ns;
774
775 /* Convert scanout position into elapsed time at raw_time query
776 * since start of scanout at first display scanline. delta_ns
777 * can be negative if start of scanout hasn't happened yet.
778 */
779 delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
780 mode->crtc_clock);
781
782 /* Subtract time delta from raw timestamp to get final
783 * vblank_time timestamp for end of vblank.
784 */
785 *vblank_time = ktime_sub_ns(etime, delta_ns);
786
787 if (!drm_debug_enabled(DRM_UT_VBL))
788 return true;
789
790 ts_etime = ktime_to_timespec64(etime);
791 ts_vblank_time = ktime_to_timespec64(*vblank_time);
792
793 drm_dbg_vbl(dev,
794 "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
795 pipe, hpos, vpos,
796 (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
797 (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
798 duration_ns / 1000, i);
799
800 return true;
801}
802EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
803
804/**
805 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
806 * helper
807 * @crtc: CRTC whose vblank timestamp to retrieve
808 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
809 * On return contains true maximum error of timestamp
810 * @vblank_time: Pointer to time which should receive the timestamp
811 * @in_vblank_irq:
812 * True when called from drm_crtc_handle_vblank(). Some drivers
813 * need to apply some workarounds for gpu-specific vblank irq quirks
814 * if flag is set.
815 *
816 * Implements calculation of exact vblank timestamps from given drm_display_mode
817 * timings and current video scanout position of a CRTC. This can be directly
818 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
819 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
820 *
821 * The current implementation only handles standard video modes. For double scan
822 * and interlaced modes the driver is supposed to adjust the hardware mode
823 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
824 * match the scanout position reported.
825 *
826 * Note that atomic drivers must call drm_calc_timestamping_constants() before
827 * enabling a CRTC. The atomic helpers already take care of that in
828 * drm_atomic_helper_calc_timestamping_constants().
829 *
830 * Returns:
831 *
832 * Returns true on success, and false on failure, i.e. when no accurate
833 * timestamp could be acquired.
834 */
835bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
836 int *max_error,
837 ktime_t *vblank_time,
838 bool in_vblank_irq)
839{
840 return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
841 crtc, max_error, vblank_time, in_vblank_irq,
842 crtc->helper_private->get_scanout_position);
843}
844EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
845
846/**
847 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
848 * vblank interval
849 * @dev: DRM device
850 * @pipe: index of CRTC whose vblank timestamp to retrieve
851 * @tvblank: Pointer to target time which should receive the timestamp
852 * @in_vblank_irq:
853 * True when called from drm_crtc_handle_vblank(). Some drivers
854 * need to apply some workarounds for gpu-specific vblank irq quirks
855 * if flag is set.
856 *
857 * Fetches the system timestamp corresponding to the time of the most recent
858 * vblank interval on specified CRTC. May call into kms-driver to
859 * compute the timestamp with a high-precision GPU specific method.
860 *
861 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
862 * call, i.e., it isn't very precisely locked to the true vblank.
863 *
864 * Returns:
865 * True if timestamp is considered to be very precise, false otherwise.
866 */
867static bool
868drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
869 ktime_t *tvblank, bool in_vblank_irq)
870{
871 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
872 bool ret = false;
873
874 /* Define requested maximum error on timestamps (nanoseconds). */
875 int max_error = (int) drm_timestamp_precision * 1000;
876
877 /* Query driver if possible and precision timestamping enabled. */
878 if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
879 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
880
881 ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
882 tvblank, in_vblank_irq);
883 }
884
885 /* GPU high precision timestamp query unsupported or failed.
886 * Return current monotonic/gettimeofday timestamp as best estimate.
887 */
888 if (!ret)
889 *tvblank = ktime_get();
890
891 return ret;
892}
893
894/**
895 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
896 * @crtc: which counter to retrieve
897 *
898 * Fetches the "cooked" vblank count value that represents the number of
899 * vblank events since the system was booted, including lost events due to
900 * modesetting activity. Note that this timer isn't correct against a racing
901 * vblank interrupt (since it only reports the software vblank counter), see
902 * drm_crtc_accurate_vblank_count() for such use-cases.
903 *
904 * Note that for a given vblank counter value drm_crtc_handle_vblank()
905 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
906 * provide a barrier: Any writes done before calling
907 * drm_crtc_handle_vblank() will be visible to callers of the later
908 * functions, if the vblank count is the same or a later one.
909 *
910 * See also &drm_vblank_crtc.count.
911 *
912 * Returns:
913 * The software vblank counter.
914 */
915u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
916{
917 return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
918}
919EXPORT_SYMBOL(drm_crtc_vblank_count);
920
921/**
922 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
923 * system timestamp corresponding to that vblank counter value.
924 * @dev: DRM device
925 * @pipe: index of CRTC whose counter to retrieve
926 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
927 *
928 * Fetches the "cooked" vblank count value that represents the number of
929 * vblank events since the system was booted, including lost events due to
930 * modesetting activity. Returns corresponding system timestamp of the time
931 * of the vblank interval that corresponds to the current vblank counter value.
932 *
933 * This is the legacy version of drm_crtc_vblank_count_and_time().
934 */
935static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
936 ktime_t *vblanktime)
937{
938 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
939 u64 vblank_count;
940 unsigned int seq;
941
942 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
943 *vblanktime = 0;
944 return 0;
945 }
946
947 do {
948 seq = read_seqbegin(&vblank->seqlock);
949 vblank_count = atomic64_read(&vblank->count);
950 *vblanktime = vblank->time;
951 } while (read_seqretry(&vblank->seqlock, seq));
952
953 return vblank_count;
954}
955
956/**
957 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
958 * and the system timestamp corresponding to that vblank counter value
959 * @crtc: which counter to retrieve
960 * @vblanktime: Pointer to time to receive the vblank timestamp.
961 *
962 * Fetches the "cooked" vblank count value that represents the number of
963 * vblank events since the system was booted, including lost events due to
964 * modesetting activity. Returns corresponding system timestamp of the time
965 * of the vblank interval that corresponds to the current vblank counter value.
966 *
967 * Note that for a given vblank counter value drm_crtc_handle_vblank()
968 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
969 * provide a barrier: Any writes done before calling
970 * drm_crtc_handle_vblank() will be visible to callers of the later
971 * functions, if the vblank count is the same or a later one.
972 *
973 * See also &drm_vblank_crtc.count.
974 */
975u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
976 ktime_t *vblanktime)
977{
978 return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
979 vblanktime);
980}
981EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
982
983static void send_vblank_event(struct drm_device *dev,
984 struct drm_pending_vblank_event *e,
985 u64 seq, ktime_t now)
986{
987 struct timespec64 tv;
988
989 switch (e->event.base.type) {
990 case DRM_EVENT_VBLANK:
991 case DRM_EVENT_FLIP_COMPLETE:
992 tv = ktime_to_timespec64(now);
993 e->event.vbl.sequence = seq;
994 /*
995 * e->event is a user space structure, with hardcoded unsigned
996 * 32-bit seconds/microseconds. This is safe as we always use
997 * monotonic timestamps since linux-4.15
998 */
999 e->event.vbl.tv_sec = tv.tv_sec;
1000 e->event.vbl.tv_usec = tv.tv_nsec / 1000;
1001 break;
1002 case DRM_EVENT_CRTC_SEQUENCE:
1003 if (seq)
1004 e->event.seq.sequence = seq;
1005 e->event.seq.time_ns = ktime_to_ns(now);
1006 break;
1007 }
1008 trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
1009 /*
1010 * Use the same timestamp for any associated fence signal to avoid
1011 * mismatch in timestamps for vsync & fence events triggered by the
1012 * same HW event. Frameworks like SurfaceFlinger in Android expects the
1013 * retire-fence timestamp to match exactly with HW vsync as it uses it
1014 * for its software vsync modeling.
1015 */
1016 drm_send_event_timestamp_locked(dev, &e->base, now);
1017}
1018
1019/**
1020 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1021 * @crtc: the source CRTC of the vblank event
1022 * @e: the event to send
1023 *
1024 * A lot of drivers need to generate vblank events for the very next vblank
1025 * interrupt. For example when the page flip interrupt happens when the page
1026 * flip gets armed, but not when it actually executes within the next vblank
1027 * period. This helper function implements exactly the required vblank arming
1028 * behaviour.
1029 *
1030 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1031 * atomic commit must ensure that the next vblank happens at exactly the same
1032 * time as the atomic commit is committed to the hardware. This function itself
1033 * does **not** protect against the next vblank interrupt racing with either this
1034 * function call or the atomic commit operation. A possible sequence could be:
1035 *
1036 * 1. Driver commits new hardware state into vblank-synchronized registers.
1037 * 2. A vblank happens, committing the hardware state. Also the corresponding
1038 * vblank interrupt is fired off and fully processed by the interrupt
1039 * handler.
1040 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1041 * 4. The event is only send out for the next vblank, which is wrong.
1042 *
1043 * An equivalent race can happen when the driver calls
1044 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
1045 *
1046 * The only way to make this work safely is to prevent the vblank from firing
1047 * (and the hardware from committing anything else) until the entire atomic
1048 * commit sequence has run to completion. If the hardware does not have such a
1049 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1050 * Instead drivers need to manually send out the event from their interrupt
1051 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1052 * possible race with the hardware committing the atomic update.
1053 *
1054 * Caller must hold a vblank reference for the event @e acquired by a
1055 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1056 */
1057void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1058 struct drm_pending_vblank_event *e)
1059{
1060 struct drm_device *dev = crtc->dev;
1061 unsigned int pipe = drm_crtc_index(crtc);
1062
1063 assert_spin_locked(&dev->event_lock);
1064
1065 e->pipe = pipe;
1066 e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1067 list_add_tail(&e->base.link, &dev->vblank_event_list);
1068}
1069EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1070
1071/**
1072 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1073 * @crtc: the source CRTC of the vblank event
1074 * @e: the event to send
1075 *
1076 * Updates sequence # and timestamp on event for the most recently processed
1077 * vblank, and sends it to userspace. Caller must hold event lock.
1078 *
1079 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1080 * situation, especially to send out events for atomic commit operations.
1081 */
1082void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1083 struct drm_pending_vblank_event *e)
1084{
1085 struct drm_device *dev = crtc->dev;
1086 u64 seq;
1087 unsigned int pipe = drm_crtc_index(crtc);
1088 ktime_t now;
1089
1090 if (drm_dev_has_vblank(dev)) {
1091 seq = drm_vblank_count_and_time(dev, pipe, &now);
1092 } else {
1093 seq = 0;
1094
1095 now = ktime_get();
1096 }
1097 e->pipe = pipe;
1098 send_vblank_event(dev, e, seq, now);
1099}
1100EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1101
1102static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1103{
1104 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1105 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1106
1107 if (drm_WARN_ON(dev, !crtc))
1108 return 0;
1109
1110 if (crtc->funcs->enable_vblank)
1111 return crtc->funcs->enable_vblank(crtc);
1112 }
1113#ifdef CONFIG_DRM_LEGACY
1114 else if (dev->driver->enable_vblank) {
1115 return dev->driver->enable_vblank(dev, pipe);
1116 }
1117#endif
1118
1119 return -EINVAL;
1120}
1121
1122static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1123{
1124 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1125 int ret = 0;
1126
1127 assert_spin_locked(&dev->vbl_lock);
1128
1129 spin_lock(&dev->vblank_time_lock);
1130
1131 if (!vblank->enabled) {
1132 /*
1133 * Enable vblank irqs under vblank_time_lock protection.
1134 * All vblank count & timestamp updates are held off
1135 * until we are done reinitializing master counter and
1136 * timestamps. Filtercode in drm_handle_vblank() will
1137 * prevent double-accounting of same vblank interval.
1138 */
1139 ret = __enable_vblank(dev, pipe);
1140 drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
1141 pipe, ret);
1142 if (ret) {
1143 atomic_dec(&vblank->refcount);
1144 } else {
1145 drm_update_vblank_count(dev, pipe, 0);
1146 /* drm_update_vblank_count() includes a wmb so we just
1147 * need to ensure that the compiler emits the write
1148 * to mark the vblank as enabled after the call
1149 * to drm_update_vblank_count().
1150 */
1151 WRITE_ONCE(vblank->enabled, true);
1152 }
1153 }
1154
1155 spin_unlock(&dev->vblank_time_lock);
1156
1157 return ret;
1158}
1159
1160int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1161{
1162 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1163 unsigned long irqflags;
1164 int ret = 0;
1165
1166 if (!drm_dev_has_vblank(dev))
1167 return -EINVAL;
1168
1169 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1170 return -EINVAL;
1171
1172 spin_lock_irqsave(&dev->vbl_lock, irqflags);
1173 /* Going from 0->1 means we have to enable interrupts again */
1174 if (atomic_add_return(1, &vblank->refcount) == 1) {
1175 ret = drm_vblank_enable(dev, pipe);
1176 } else {
1177 if (!vblank->enabled) {
1178 atomic_dec(&vblank->refcount);
1179 ret = -EINVAL;
1180 }
1181 }
1182 spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1183
1184 return ret;
1185}
1186
1187/**
1188 * drm_crtc_vblank_get - get a reference count on vblank events
1189 * @crtc: which CRTC to own
1190 *
1191 * Acquire a reference count on vblank events to avoid having them disabled
1192 * while in use.
1193 *
1194 * Returns:
1195 * Zero on success or a negative error code on failure.
1196 */
1197int drm_crtc_vblank_get(struct drm_crtc *crtc)
1198{
1199 return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1200}
1201EXPORT_SYMBOL(drm_crtc_vblank_get);
1202
1203void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1204{
1205 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1206
1207 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1208 return;
1209
1210 if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
1211 return;
1212
1213 /* Last user schedules interrupt disable */
1214 if (atomic_dec_and_test(&vblank->refcount)) {
1215 if (drm_vblank_offdelay == 0)
1216 return;
1217 else if (drm_vblank_offdelay < 0)
1218 vblank_disable_fn(&vblank->disable_timer);
1219 else if (!dev->vblank_disable_immediate)
1220 mod_timer(&vblank->disable_timer,
1221 jiffies + ((drm_vblank_offdelay * HZ)/1000));
1222 }
1223}
1224
1225/**
1226 * drm_crtc_vblank_put - give up ownership of vblank events
1227 * @crtc: which counter to give up
1228 *
1229 * Release ownership of a given vblank counter, turning off interrupts
1230 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1231 */
1232void drm_crtc_vblank_put(struct drm_crtc *crtc)
1233{
1234 drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1235}
1236EXPORT_SYMBOL(drm_crtc_vblank_put);
1237
1238/**
1239 * drm_wait_one_vblank - wait for one vblank
1240 * @dev: DRM device
1241 * @pipe: CRTC index
1242 *
1243 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1244 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1245 * due to lack of driver support or because the crtc is off.
1246 *
1247 * This is the legacy version of drm_crtc_wait_one_vblank().
1248 */
1249void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1250{
1251 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1252 int ret;
1253 u64 last;
1254
1255 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1256 return;
1257
1258 ret = drm_vblank_get(dev, pipe);
1259 if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
1260 pipe, ret))
1261 return;
1262
1263 last = drm_vblank_count(dev, pipe);
1264
1265 ret = wait_event_timeout(vblank->queue,
1266 last != drm_vblank_count(dev, pipe),
1267 msecs_to_jiffies(100));
1268
1269 drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1270
1271 drm_vblank_put(dev, pipe);
1272}
1273EXPORT_SYMBOL(drm_wait_one_vblank);
1274
1275/**
1276 * drm_crtc_wait_one_vblank - wait for one vblank
1277 * @crtc: DRM crtc
1278 *
1279 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1280 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1281 * due to lack of driver support or because the crtc is off.
1282 */
1283void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1284{
1285 drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1286}
1287EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1288
1289/**
1290 * drm_crtc_vblank_off - disable vblank events on a CRTC
1291 * @crtc: CRTC in question
1292 *
1293 * Drivers can use this function to shut down the vblank interrupt handling when
1294 * disabling a crtc. This function ensures that the latest vblank frame count is
1295 * stored so that drm_vblank_on can restore it again.
1296 *
1297 * Drivers must use this function when the hardware vblank counter can get
1298 * reset, e.g. when suspending or disabling the @crtc in general.
1299 */
1300void drm_crtc_vblank_off(struct drm_crtc *crtc)
1301{
1302 struct drm_device *dev = crtc->dev;
1303 unsigned int pipe = drm_crtc_index(crtc);
1304 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1305 struct drm_pending_vblank_event *e, *t;
1306 ktime_t now;
1307 u64 seq;
1308
1309 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1310 return;
1311
1312 /*
1313 * Grab event_lock early to prevent vblank work from being scheduled
1314 * while we're in the middle of shutting down vblank interrupts
1315 */
1316 spin_lock_irq(&dev->event_lock);
1317
1318 spin_lock(&dev->vbl_lock);
1319 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1320 pipe, vblank->enabled, vblank->inmodeset);
1321
1322 /* Avoid redundant vblank disables without previous
1323 * drm_crtc_vblank_on(). */
1324 if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1325 drm_vblank_disable_and_save(dev, pipe);
1326
1327 wake_up(&vblank->queue);
1328
1329 /*
1330 * Prevent subsequent drm_vblank_get() from re-enabling
1331 * the vblank interrupt by bumping the refcount.
1332 */
1333 if (!vblank->inmodeset) {
1334 atomic_inc(&vblank->refcount);
1335 vblank->inmodeset = 1;
1336 }
1337 spin_unlock(&dev->vbl_lock);
1338
1339 /* Send any queued vblank events, lest the natives grow disquiet */
1340 seq = drm_vblank_count_and_time(dev, pipe, &now);
1341
1342 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1343 if (e->pipe != pipe)
1344 continue;
1345 drm_dbg_core(dev, "Sending premature vblank event on disable: "
1346 "wanted %llu, current %llu\n",
1347 e->sequence, seq);
1348 list_del(&e->base.link);
1349 drm_vblank_put(dev, pipe);
1350 send_vblank_event(dev, e, seq, now);
1351 }
1352
1353 /* Cancel any leftover pending vblank work */
1354 drm_vblank_cancel_pending_works(vblank);
1355
1356 spin_unlock_irq(&dev->event_lock);
1357
1358 /* Will be reset by the modeset helpers when re-enabling the crtc by
1359 * calling drm_calc_timestamping_constants(). */
1360 vblank->hwmode.crtc_clock = 0;
1361
1362 /* Wait for any vblank work that's still executing to finish */
1363 drm_vblank_flush_worker(vblank);
1364}
1365EXPORT_SYMBOL(drm_crtc_vblank_off);
1366
1367/**
1368 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1369 * @crtc: CRTC in question
1370 *
1371 * Drivers can use this function to reset the vblank state to off at load time.
1372 * Drivers should use this together with the drm_crtc_vblank_off() and
1373 * drm_crtc_vblank_on() functions. The difference compared to
1374 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1375 * and hence doesn't need to call any driver hooks.
1376 *
1377 * This is useful for recovering driver state e.g. on driver load, or on resume.
1378 */
1379void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1380{
1381 struct drm_device *dev = crtc->dev;
1382 unsigned int pipe = drm_crtc_index(crtc);
1383 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1384
1385 spin_lock_irq(&dev->vbl_lock);
1386 /*
1387 * Prevent subsequent drm_vblank_get() from enabling the vblank
1388 * interrupt by bumping the refcount.
1389 */
1390 if (!vblank->inmodeset) {
1391 atomic_inc(&vblank->refcount);
1392 vblank->inmodeset = 1;
1393 }
1394 spin_unlock_irq(&dev->vbl_lock);
1395
1396 drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
1397 drm_WARN_ON(dev, !list_empty(&vblank->pending_work));
1398}
1399EXPORT_SYMBOL(drm_crtc_vblank_reset);
1400
1401/**
1402 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1403 * @crtc: CRTC in question
1404 * @max_vblank_count: max hardware vblank counter value
1405 *
1406 * Update the maximum hardware vblank counter value for @crtc
1407 * at runtime. Useful for hardware where the operation of the
1408 * hardware vblank counter depends on the currently active
1409 * display configuration.
1410 *
1411 * For example, if the hardware vblank counter does not work
1412 * when a specific connector is active the maximum can be set
1413 * to zero. And when that specific connector isn't active the
1414 * maximum can again be set to the appropriate non-zero value.
1415 *
1416 * If used, must be called before drm_vblank_on().
1417 */
1418void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1419 u32 max_vblank_count)
1420{
1421 struct drm_device *dev = crtc->dev;
1422 unsigned int pipe = drm_crtc_index(crtc);
1423 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1424
1425 drm_WARN_ON(dev, dev->max_vblank_count);
1426 drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
1427
1428 vblank->max_vblank_count = max_vblank_count;
1429}
1430EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1431
1432/**
1433 * drm_crtc_vblank_on - enable vblank events on a CRTC
1434 * @crtc: CRTC in question
1435 *
1436 * This functions restores the vblank interrupt state captured with
1437 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1438 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1439 * unbalanced and so can also be unconditionally called in driver load code to
1440 * reflect the current hardware state of the crtc.
1441 */
1442void drm_crtc_vblank_on(struct drm_crtc *crtc)
1443{
1444 struct drm_device *dev = crtc->dev;
1445 unsigned int pipe = drm_crtc_index(crtc);
1446 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1447
1448 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1449 return;
1450
1451 spin_lock_irq(&dev->vbl_lock);
1452 drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1453 pipe, vblank->enabled, vblank->inmodeset);
1454
1455 /* Drop our private "prevent drm_vblank_get" refcount */
1456 if (vblank->inmodeset) {
1457 atomic_dec(&vblank->refcount);
1458 vblank->inmodeset = 0;
1459 }
1460
1461 drm_reset_vblank_timestamp(dev, pipe);
1462
1463 /*
1464 * re-enable interrupts if there are users left, or the
1465 * user wishes vblank interrupts to be enabled all the time.
1466 */
1467 if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1468 drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
1469 spin_unlock_irq(&dev->vbl_lock);
1470}
1471EXPORT_SYMBOL(drm_crtc_vblank_on);
1472
1473static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1474{
1475 ktime_t t_vblank;
1476 struct drm_vblank_crtc *vblank;
1477 int framedur_ns;
1478 u64 diff_ns;
1479 u32 cur_vblank, diff = 1;
1480 int count = DRM_TIMESTAMP_MAXRETRIES;
1481 u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
1482
1483 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1484 return;
1485
1486 assert_spin_locked(&dev->vbl_lock);
1487 assert_spin_locked(&dev->vblank_time_lock);
1488
1489 vblank = &dev->vblank[pipe];
1490 drm_WARN_ONCE(dev,
1491 drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
1492 "Cannot compute missed vblanks without frame duration\n");
1493 framedur_ns = vblank->framedur_ns;
1494
1495 do {
1496 cur_vblank = __get_vblank_counter(dev, pipe);
1497 drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1498 } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1499
1500 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1501 if (framedur_ns)
1502 diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1503
1504
1505 drm_dbg_vbl(dev,
1506 "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1507 diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1508 vblank->last = (cur_vblank - diff) & max_vblank_count;
1509}
1510
1511/**
1512 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1513 * @crtc: CRTC in question
1514 *
1515 * Power manamement features can cause frame counter resets between vblank
1516 * disable and enable. Drivers can use this function in their
1517 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1518 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1519 * vblank counter.
1520 *
1521 * Note that drivers must have race-free high-precision timestamping support,
1522 * i.e. &drm_crtc_funcs.get_vblank_timestamp must be hooked up and
1523 * &drm_driver.vblank_disable_immediate must be set to indicate the
1524 * time-stamping functions are race-free against vblank hardware counter
1525 * increments.
1526 */
1527void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1528{
1529 WARN_ON_ONCE(!crtc->funcs->get_vblank_timestamp);
1530 WARN_ON_ONCE(!crtc->dev->vblank_disable_immediate);
1531
1532 drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1533}
1534EXPORT_SYMBOL(drm_crtc_vblank_restore);
1535
1536static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1537 unsigned int pipe)
1538{
1539 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1540
1541 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1542 if (!drm_dev_has_vblank(dev))
1543 return;
1544
1545 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1546 return;
1547
1548 /*
1549 * To avoid all the problems that might happen if interrupts
1550 * were enabled/disabled around or between these calls, we just
1551 * have the kernel take a reference on the CRTC (just once though
1552 * to avoid corrupting the count if multiple, mismatch calls occur),
1553 * so that interrupts remain enabled in the interim.
1554 */
1555 if (!vblank->inmodeset) {
1556 vblank->inmodeset = 0x1;
1557 if (drm_vblank_get(dev, pipe) == 0)
1558 vblank->inmodeset |= 0x2;
1559 }
1560}
1561
1562static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1563 unsigned int pipe)
1564{
1565 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1566
1567 /* vblank is not initialized (IRQ not installed ?), or has been freed */
1568 if (!drm_dev_has_vblank(dev))
1569 return;
1570
1571 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1572 return;
1573
1574 if (vblank->inmodeset) {
1575 spin_lock_irq(&dev->vbl_lock);
1576 drm_reset_vblank_timestamp(dev, pipe);
1577 spin_unlock_irq(&dev->vbl_lock);
1578
1579 if (vblank->inmodeset & 0x2)
1580 drm_vblank_put(dev, pipe);
1581
1582 vblank->inmodeset = 0;
1583 }
1584}
1585
1586int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1587 struct drm_file *file_priv)
1588{
1589 struct drm_modeset_ctl *modeset = data;
1590 unsigned int pipe;
1591
1592 /* If drm_vblank_init() hasn't been called yet, just no-op */
1593 if (!drm_dev_has_vblank(dev))
1594 return 0;
1595
1596 /* KMS drivers handle this internally */
1597 if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1598 return 0;
1599
1600 pipe = modeset->crtc;
1601 if (pipe >= dev->num_crtcs)
1602 return -EINVAL;
1603
1604 switch (modeset->cmd) {
1605 case _DRM_PRE_MODESET:
1606 drm_legacy_vblank_pre_modeset(dev, pipe);
1607 break;
1608 case _DRM_POST_MODESET:
1609 drm_legacy_vblank_post_modeset(dev, pipe);
1610 break;
1611 default:
1612 return -EINVAL;
1613 }
1614
1615 return 0;
1616}
1617
1618static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1619 u64 req_seq,
1620 union drm_wait_vblank *vblwait,
1621 struct drm_file *file_priv)
1622{
1623 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1624 struct drm_pending_vblank_event *e;
1625 ktime_t now;
1626 u64 seq;
1627 int ret;
1628
1629 e = kzalloc(sizeof(*e), GFP_KERNEL);
1630 if (e == NULL) {
1631 ret = -ENOMEM;
1632 goto err_put;
1633 }
1634
1635 e->pipe = pipe;
1636 e->event.base.type = DRM_EVENT_VBLANK;
1637 e->event.base.length = sizeof(e->event.vbl);
1638 e->event.vbl.user_data = vblwait->request.signal;
1639 e->event.vbl.crtc_id = 0;
1640 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1641 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1642
1643 if (crtc)
1644 e->event.vbl.crtc_id = crtc->base.id;
1645 }
1646
1647 spin_lock_irq(&dev->event_lock);
1648
1649 /*
1650 * drm_crtc_vblank_off() might have been called after we called
1651 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1652 * vblank disable, so no need for further locking. The reference from
1653 * drm_vblank_get() protects against vblank disable from another source.
1654 */
1655 if (!READ_ONCE(vblank->enabled)) {
1656 ret = -EINVAL;
1657 goto err_unlock;
1658 }
1659
1660 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1661 &e->event.base);
1662
1663 if (ret)
1664 goto err_unlock;
1665
1666 seq = drm_vblank_count_and_time(dev, pipe, &now);
1667
1668 drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
1669 req_seq, seq, pipe);
1670
1671 trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1672
1673 e->sequence = req_seq;
1674 if (drm_vblank_passed(seq, req_seq)) {
1675 drm_vblank_put(dev, pipe);
1676 send_vblank_event(dev, e, seq, now);
1677 vblwait->reply.sequence = seq;
1678 } else {
1679 /* drm_handle_vblank_events will call drm_vblank_put */
1680 list_add_tail(&e->base.link, &dev->vblank_event_list);
1681 vblwait->reply.sequence = req_seq;
1682 }
1683
1684 spin_unlock_irq(&dev->event_lock);
1685
1686 return 0;
1687
1688err_unlock:
1689 spin_unlock_irq(&dev->event_lock);
1690 kfree(e);
1691err_put:
1692 drm_vblank_put(dev, pipe);
1693 return ret;
1694}
1695
1696static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1697{
1698 if (vblwait->request.sequence)
1699 return false;
1700
1701 return _DRM_VBLANK_RELATIVE ==
1702 (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1703 _DRM_VBLANK_EVENT |
1704 _DRM_VBLANK_NEXTONMISS));
1705}
1706
1707/*
1708 * Widen a 32-bit param to 64-bits.
1709 *
1710 * \param narrow 32-bit value (missing upper 32 bits)
1711 * \param near 64-bit value that should be 'close' to near
1712 *
1713 * This function returns a 64-bit value using the lower 32-bits from
1714 * 'narrow' and constructing the upper 32-bits so that the result is
1715 * as close as possible to 'near'.
1716 */
1717
1718static u64 widen_32_to_64(u32 narrow, u64 near)
1719{
1720 return near + (s32) (narrow - near);
1721}
1722
1723static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1724 struct drm_wait_vblank_reply *reply)
1725{
1726 ktime_t now;
1727 struct timespec64 ts;
1728
1729 /*
1730 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1731 * to store the seconds. This is safe as we always use monotonic
1732 * timestamps since linux-4.15.
1733 */
1734 reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1735 ts = ktime_to_timespec64(now);
1736 reply->tval_sec = (u32)ts.tv_sec;
1737 reply->tval_usec = ts.tv_nsec / 1000;
1738}
1739
1740static bool drm_wait_vblank_supported(struct drm_device *dev)
1741{
1742#if IS_ENABLED(CONFIG_DRM_LEGACY)
1743 if (unlikely(drm_core_check_feature(dev, DRIVER_LEGACY)))
1744 return dev->irq_enabled;
1745#endif
1746 return drm_dev_has_vblank(dev);
1747}
1748
1749int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1750 struct drm_file *file_priv)
1751{
1752 struct drm_crtc *crtc;
1753 struct drm_vblank_crtc *vblank;
1754 union drm_wait_vblank *vblwait = data;
1755 int ret;
1756 u64 req_seq, seq;
1757 unsigned int pipe_index;
1758 unsigned int flags, pipe, high_pipe;
1759
1760 if (!drm_wait_vblank_supported(dev))
1761 return -EOPNOTSUPP;
1762
1763 if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1764 return -EINVAL;
1765
1766 if (vblwait->request.type &
1767 ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1768 _DRM_VBLANK_HIGH_CRTC_MASK)) {
1769 drm_dbg_core(dev,
1770 "Unsupported type value 0x%x, supported mask 0x%x\n",
1771 vblwait->request.type,
1772 (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1773 _DRM_VBLANK_HIGH_CRTC_MASK));
1774 return -EINVAL;
1775 }
1776
1777 flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1778 high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1779 if (high_pipe)
1780 pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1781 else
1782 pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1783
1784 /* Convert lease-relative crtc index into global crtc index */
1785 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1786 pipe = 0;
1787 drm_for_each_crtc(crtc, dev) {
1788 if (drm_lease_held(file_priv, crtc->base.id)) {
1789 if (pipe_index == 0)
1790 break;
1791 pipe_index--;
1792 }
1793 pipe++;
1794 }
1795 } else {
1796 pipe = pipe_index;
1797 }
1798
1799 if (pipe >= dev->num_crtcs)
1800 return -EINVAL;
1801
1802 vblank = &dev->vblank[pipe];
1803
1804 /* If the counter is currently enabled and accurate, short-circuit
1805 * queries to return the cached timestamp of the last vblank.
1806 */
1807 if (dev->vblank_disable_immediate &&
1808 drm_wait_vblank_is_query(vblwait) &&
1809 READ_ONCE(vblank->enabled)) {
1810 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1811 return 0;
1812 }
1813
1814 ret = drm_vblank_get(dev, pipe);
1815 if (ret) {
1816 drm_dbg_core(dev,
1817 "crtc %d failed to acquire vblank counter, %d\n",
1818 pipe, ret);
1819 return ret;
1820 }
1821 seq = drm_vblank_count(dev, pipe);
1822
1823 switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1824 case _DRM_VBLANK_RELATIVE:
1825 req_seq = seq + vblwait->request.sequence;
1826 vblwait->request.sequence = req_seq;
1827 vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1828 break;
1829 case _DRM_VBLANK_ABSOLUTE:
1830 req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1831 break;
1832 default:
1833 ret = -EINVAL;
1834 goto done;
1835 }
1836
1837 if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1838 drm_vblank_passed(seq, req_seq)) {
1839 req_seq = seq + 1;
1840 vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1841 vblwait->request.sequence = req_seq;
1842 }
1843
1844 if (flags & _DRM_VBLANK_EVENT) {
1845 /* must hold on to the vblank ref until the event fires
1846 * drm_vblank_put will be called asynchronously
1847 */
1848 return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1849 }
1850
1851 if (req_seq != seq) {
1852 int wait;
1853
1854 drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
1855 req_seq, pipe);
1856 wait = wait_event_interruptible_timeout(vblank->queue,
1857 drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1858 !READ_ONCE(vblank->enabled),
1859 msecs_to_jiffies(3000));
1860
1861 switch (wait) {
1862 case 0:
1863 /* timeout */
1864 ret = -EBUSY;
1865 break;
1866 case -ERESTARTSYS:
1867 /* interrupted by signal */
1868 ret = -EINTR;
1869 break;
1870 default:
1871 ret = 0;
1872 break;
1873 }
1874 }
1875
1876 if (ret != -EINTR) {
1877 drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1878
1879 drm_dbg_core(dev, "crtc %d returning %u to client\n",
1880 pipe, vblwait->reply.sequence);
1881 } else {
1882 drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
1883 pipe);
1884 }
1885
1886done:
1887 drm_vblank_put(dev, pipe);
1888 return ret;
1889}
1890
1891static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1892{
1893 struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1894 bool high_prec = false;
1895 struct drm_pending_vblank_event *e, *t;
1896 ktime_t now;
1897 u64 seq;
1898
1899 assert_spin_locked(&dev->event_lock);
1900
1901 seq = drm_vblank_count_and_time(dev, pipe, &now);
1902
1903 list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1904 if (e->pipe != pipe)
1905 continue;
1906 if (!drm_vblank_passed(seq, e->sequence))
1907 continue;
1908
1909 drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
1910 e->sequence, seq);
1911
1912 list_del(&e->base.link);
1913 drm_vblank_put(dev, pipe);
1914 send_vblank_event(dev, e, seq, now);
1915 }
1916
1917 if (crtc && crtc->funcs->get_vblank_timestamp)
1918 high_prec = true;
1919
1920 trace_drm_vblank_event(pipe, seq, now, high_prec);
1921}
1922
1923/**
1924 * drm_handle_vblank - handle a vblank event
1925 * @dev: DRM device
1926 * @pipe: index of CRTC where this event occurred
1927 *
1928 * Drivers should call this routine in their vblank interrupt handlers to
1929 * update the vblank counter and send any signals that may be pending.
1930 *
1931 * This is the legacy version of drm_crtc_handle_vblank().
1932 */
1933bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1934{
1935 struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1936 unsigned long irqflags;
1937 bool disable_irq;
1938
1939 if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
1940 return false;
1941
1942 if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1943 return false;
1944
1945 spin_lock_irqsave(&dev->event_lock, irqflags);
1946
1947 /* Need timestamp lock to prevent concurrent execution with
1948 * vblank enable/disable, as this would cause inconsistent
1949 * or corrupted timestamps and vblank counts.
1950 */
1951 spin_lock(&dev->vblank_time_lock);
1952
1953 /* Vblank irq handling disabled. Nothing to do. */
1954 if (!vblank->enabled) {
1955 spin_unlock(&dev->vblank_time_lock);
1956 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1957 return false;
1958 }
1959
1960 drm_update_vblank_count(dev, pipe, true);
1961
1962 spin_unlock(&dev->vblank_time_lock);
1963
1964 wake_up(&vblank->queue);
1965
1966 /* With instant-off, we defer disabling the interrupt until after
1967 * we finish processing the following vblank after all events have
1968 * been signaled. The disable has to be last (after
1969 * drm_handle_vblank_events) so that the timestamp is always accurate.
1970 */
1971 disable_irq = (dev->vblank_disable_immediate &&
1972 drm_vblank_offdelay > 0 &&
1973 !atomic_read(&vblank->refcount));
1974
1975 drm_handle_vblank_events(dev, pipe);
1976 drm_handle_vblank_works(vblank);
1977
1978 spin_unlock_irqrestore(&dev->event_lock, irqflags);
1979
1980 if (disable_irq)
1981 vblank_disable_fn(&vblank->disable_timer);
1982
1983 return true;
1984}
1985EXPORT_SYMBOL(drm_handle_vblank);
1986
1987/**
1988 * drm_crtc_handle_vblank - handle a vblank event
1989 * @crtc: where this event occurred
1990 *
1991 * Drivers should call this routine in their vblank interrupt handlers to
1992 * update the vblank counter and send any signals that may be pending.
1993 *
1994 * This is the native KMS version of drm_handle_vblank().
1995 *
1996 * Note that for a given vblank counter value drm_crtc_handle_vblank()
1997 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
1998 * provide a barrier: Any writes done before calling
1999 * drm_crtc_handle_vblank() will be visible to callers of the later
2000 * functions, if the vblank count is the same or a later one.
2001 *
2002 * See also &drm_vblank_crtc.count.
2003 *
2004 * Returns:
2005 * True if the event was successfully handled, false on failure.
2006 */
2007bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
2008{
2009 return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
2010}
2011EXPORT_SYMBOL(drm_crtc_handle_vblank);
2012
2013/*
2014 * Get crtc VBLANK count.
2015 *
2016 * \param dev DRM device
2017 * \param data user argument, pointing to a drm_crtc_get_sequence structure.
2018 * \param file_priv drm file private for the user's open file descriptor
2019 */
2020
2021int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
2022 struct drm_file *file_priv)
2023{
2024 struct drm_crtc *crtc;
2025 struct drm_vblank_crtc *vblank;
2026 int pipe;
2027 struct drm_crtc_get_sequence *get_seq = data;
2028 ktime_t now;
2029 bool vblank_enabled;
2030 int ret;
2031
2032 if (!drm_core_check_feature(dev, DRIVER_MODESET))
2033 return -EOPNOTSUPP;
2034
2035 if (!drm_dev_has_vblank(dev))
2036 return -EOPNOTSUPP;
2037
2038 crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
2039 if (!crtc)
2040 return -ENOENT;
2041
2042 pipe = drm_crtc_index(crtc);
2043
2044 vblank = &dev->vblank[pipe];
2045 vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
2046
2047 if (!vblank_enabled) {
2048 ret = drm_crtc_vblank_get(crtc);
2049 if (ret) {
2050 drm_dbg_core(dev,
2051 "crtc %d failed to acquire vblank counter, %d\n",
2052 pipe, ret);
2053 return ret;
2054 }
2055 }
2056 drm_modeset_lock(&crtc->mutex, NULL);
2057 if (crtc->state)
2058 get_seq->active = crtc->state->enable;
2059 else
2060 get_seq->active = crtc->enabled;
2061 drm_modeset_unlock(&crtc->mutex);
2062 get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
2063 get_seq->sequence_ns = ktime_to_ns(now);
2064 if (!vblank_enabled)
2065 drm_crtc_vblank_put(crtc);
2066 return 0;
2067}
2068
2069/*
2070 * Queue a event for VBLANK sequence
2071 *
2072 * \param dev DRM device
2073 * \param data user argument, pointing to a drm_crtc_queue_sequence structure.
2074 * \param file_priv drm file private for the user's open file descriptor
2075 */
2076
2077int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2078 struct drm_file *file_priv)
2079{
2080 struct drm_crtc *crtc;
2081 struct drm_vblank_crtc *vblank;
2082 int pipe;
2083 struct drm_crtc_queue_sequence *queue_seq = data;
2084 ktime_t now;
2085 struct drm_pending_vblank_event *e;
2086 u32 flags;
2087 u64 seq;
2088 u64 req_seq;
2089 int ret;
2090
2091 if (!drm_core_check_feature(dev, DRIVER_MODESET))
2092 return -EOPNOTSUPP;
2093
2094 if (!drm_dev_has_vblank(dev))
2095 return -EOPNOTSUPP;
2096
2097 crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
2098 if (!crtc)
2099 return -ENOENT;
2100
2101 flags = queue_seq->flags;
2102 /* Check valid flag bits */
2103 if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
2104 DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
2105 return -EINVAL;
2106
2107 pipe = drm_crtc_index(crtc);
2108
2109 vblank = &dev->vblank[pipe];
2110
2111 e = kzalloc(sizeof(*e), GFP_KERNEL);
2112 if (e == NULL)
2113 return -ENOMEM;
2114
2115 ret = drm_crtc_vblank_get(crtc);
2116 if (ret) {
2117 drm_dbg_core(dev,
2118 "crtc %d failed to acquire vblank counter, %d\n",
2119 pipe, ret);
2120 goto err_free;
2121 }
2122
2123 seq = drm_vblank_count_and_time(dev, pipe, &now);
2124 req_seq = queue_seq->sequence;
2125
2126 if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
2127 req_seq += seq;
2128
2129 if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq))
2130 req_seq = seq + 1;
2131
2132 e->pipe = pipe;
2133 e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
2134 e->event.base.length = sizeof(e->event.seq);
2135 e->event.seq.user_data = queue_seq->user_data;
2136
2137 spin_lock_irq(&dev->event_lock);
2138
2139 /*
2140 * drm_crtc_vblank_off() might have been called after we called
2141 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2142 * vblank disable, so no need for further locking. The reference from
2143 * drm_crtc_vblank_get() protects against vblank disable from another source.
2144 */
2145 if (!READ_ONCE(vblank->enabled)) {
2146 ret = -EINVAL;
2147 goto err_unlock;
2148 }
2149
2150 ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
2151 &e->event.base);
2152
2153 if (ret)
2154 goto err_unlock;
2155
2156 e->sequence = req_seq;
2157
2158 if (drm_vblank_passed(seq, req_seq)) {
2159 drm_crtc_vblank_put(crtc);
2160 send_vblank_event(dev, e, seq, now);
2161 queue_seq->sequence = seq;
2162 } else {
2163 /* drm_handle_vblank_events will call drm_vblank_put */
2164 list_add_tail(&e->base.link, &dev->vblank_event_list);
2165 queue_seq->sequence = req_seq;
2166 }
2167
2168 spin_unlock_irq(&dev->event_lock);
2169 return 0;
2170
2171err_unlock:
2172 spin_unlock_irq(&dev->event_lock);
2173 drm_crtc_vblank_put(crtc);
2174err_free:
2175 kfree(e);
2176 return ret;
2177}
2178