1/*
   2 * drm_irq.c IRQ and vblank support
   3 *
   4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
   5 * \author Gareth Hughes <gareth@valinux.com>
   6 *
   7 * Permission is hereby granted, free of charge, to any person obtaining a
   8 * copy of this software and associated documentation files (the "Software"),
   9 * to deal in the Software without restriction, including without limitation
  10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11 * and/or sell copies of the Software, and to permit persons to whom the
  12 * Software is furnished to do so, subject to the following conditions:
  13 *
  14 * The above copyright notice and this permission notice (including the next
  15 * paragraph) shall be included in all copies or substantial portions of the
  16 * Software.
  17 *
  18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24 * OTHER DEALINGS IN THE SOFTWARE.
  25 */
  26
  27#include <linux/export.h>
  28#include <linux/kthread.h>
  29#include <linux/moduleparam.h>
  30
  31#include <drm/drm_crtc.h>
  32#include <drm/drm_drv.h>
  33#include <drm/drm_framebuffer.h>
  34#include <drm/drm_managed.h>
  35#include <drm/drm_modeset_helper_vtables.h>
  36#include <drm/drm_print.h>
  37#include <drm/drm_vblank.h>
  38
  39#include "drm_internal.h"
  40#include "drm_trace.h"
  41
  42/**
  43 * DOC: vblank handling
  44 *
  45 * From the computer's perspective, every time the monitor displays
  46 * a new frame the scanout engine has "scanned out" the display image
  47 * from top to bottom, one row of pixels at a time. The current row
  48 * of pixels is referred to as the current scanline.
  49 *
  50 * In addition to the display's visible area, there's usually a couple of
  51 * extra scanlines which aren't actually displayed on the screen.
  52 * These extra scanlines don't contain image data and are occasionally used
  53 * for features like audio and infoframes. The region made up of these
  54 * scanlines is referred to as the vertical blanking region, or vblank for
  55 * short.
  56 *
  57 * For historical reference, the vertical blanking period was designed to
  58 * give the electron gun (on CRTs) enough time to move back to the top of
  59 * the screen to start scanning out the next frame. Similar for horizontal
  60 * blanking periods. They were designed to give the electron gun enough
  61 * time to move back to the other side of the screen to start scanning the
  62 * next scanline.
  63 *
  64 * ::
  65 *
  66 *
  67 *    physical →   ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
  68 *    top of      |                                        |
  69 *    display     |                                        |
  70 *                |               New frame                |
  71 *                |                                        |
  72 *                |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
  73 *                |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
  74 *                |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|   updates the
  75 *                |                                        |   frame as it
  76 *                |                                        |   travels down
  77 *                |                                        |   ("scan out")
  78 *                |               Old frame                |
  79 *                |                                        |
  80 *                |                                        |
  81 *                |                                        |
  82 *                |                                        |   physical
  83 *                |                                        |   bottom of
  84 *    vertical    |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
  85 *    blanking    ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  86 *    region   →  ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  87 *                ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  88 *    start of →   ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
  89 *    new frame
  90 *
  91 * "Physical top of display" is the reference point for the high-precision/
  92 * corrected timestamp.
  93 *
  94 * On a lot of display hardware, programming needs to take effect during the
  95 * vertical blanking period so that settings like gamma, the image buffer
  96 * buffer to be scanned out, etc. can safely be changed without showing
  97 * any visual artifacts on the screen. In some unforgiving hardware, some of
  98 * this programming has to both start and end in the same vblank. To help
  99 * with the timing of the hardware programming, an interrupt is usually
 100 * available to notify the driver when it can start the updating of registers.
 101 * The interrupt is in this context named the vblank interrupt.
 102 *
 103 * The vblank interrupt may be fired at different points depending on the
 104 * hardware. Some hardware implementations will fire the interrupt when the
 105 * new frame start, other implementations will fire the interrupt at different
 106 * points in time.
 107 *
 108 * Vertical blanking plays a major role in graphics rendering. To achieve
 109 * tear-free display, users must synchronize page flips and/or rendering to
 110 * vertical blanking. The DRM API offers ioctls to perform page flips
 111 * synchronized to vertical blanking and wait for vertical blanking.
 112 *
 113 * The DRM core handles most of the vertical blanking management logic, which
 114 * involves filtering out spurious interrupts, keeping race-free blanking
 115 * counters, coping with counter wrap-around and resets and keeping use counts.
 116 * It relies on the driver to generate vertical blanking interrupts and
 117 * optionally provide a hardware vertical blanking counter.
 118 *
 119 * Drivers must initialize the vertical blanking handling core with a call to
 120 * drm_vblank_init(). Minimally, a driver needs to implement
 121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
 122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
 123 * support.
 124 *
 125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
 126 * themselves (for instance to handle page flipping operations).  The DRM core
 127 * maintains a vertical blanking use count to ensure that the interrupts are not
 128 * disabled while a user still needs them. To increment the use count, drivers
 129 * call drm_crtc_vblank_get() and release the vblank reference again with
 130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
 131 * guaranteed to be enabled.
 132 *
 133 * On many hardware disabling the vblank interrupt cannot be done in a race-free
 134 * manner, see &drm_driver.vblank_disable_immediate and
 135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
 136 * vblanks after a timer has expired, which can be configured through the
 137 * ``vblankoffdelay`` module parameter.
 138 *
 139 * Drivers for hardware without support for vertical-blanking interrupts
 140 * must not call drm_vblank_init(). For such drivers, atomic helpers will
 141 * automatically generate fake vblank events as part of the display update.
 142 * This functionality also can be controlled by the driver by enabling and
 143 * disabling struct drm_crtc_state.no_vblank.
 144 */
 145
 146/* Retry timestamp calculation up to 3 times to satisfy
 147 * drm_timestamp_precision before giving up.
 148 */
 149#define DRM_TIMESTAMP_MAXRETRIES 3
 150
 151/* Threshold in nanoseconds for detection of redundant
 152 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
 153 */
 154#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
 155
 156static bool
 157drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 158			  ktime_t *tvblank, bool in_vblank_irq);
 159
 160static unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
 161
 162static int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
 163
 164module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
 165module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
 166MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
 167MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
 168
 169static void store_vblank(struct drm_device *dev, unsigned int pipe,
 170			 u32 vblank_count_inc,
 171			 ktime_t t_vblank, u32 last)
 172{
 173	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 174
 175	assert_spin_locked(&dev->vblank_time_lock);
 176
 177	vblank->last = last;
 178
 179	write_seqlock(&vblank->seqlock);
 180	vblank->time = t_vblank;
 181	atomic64_add(vblank_count_inc, &vblank->count);
 182	write_sequnlock(&vblank->seqlock);
 183}
 184
 185static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
 186{
 187	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 188
 189	return vblank->max_vblank_count ?: dev->max_vblank_count;
 190}
 191
 192/*
 193 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
 194 * if there is no usable hardware frame counter available.
 195 */
 196static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
 197{
 198	drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
 199	return 0;
 200}
 201
 202static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 203{
 204	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 205		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 206
 207		if (drm_WARN_ON(dev, !crtc))
 208			return 0;
 209
 210		if (crtc->funcs->get_vblank_counter)
 211			return crtc->funcs->get_vblank_counter(crtc);
 212	}
 213
 214	return drm_vblank_no_hw_counter(dev, pipe);
 215}
 216
 217/*
 218 * Reset the stored timestamp for the current vblank count to correspond
 219 * to the last vblank occurred.
 220 *
 221 * Only to be called from drm_crtc_vblank_on().
 222 *
 223 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
 224 * device vblank fields.
 225 */
 226static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
 227{
 228	u32 cur_vblank;
 229	bool rc;
 230	ktime_t t_vblank;
 231	int count = DRM_TIMESTAMP_MAXRETRIES;
 232
 233	spin_lock(&dev->vblank_time_lock);
 234
 235	/*
 236	 * sample the current counter to avoid random jumps
 237	 * when drm_vblank_enable() applies the diff
 238	 */
 239	do {
 240		cur_vblank = __get_vblank_counter(dev, pipe);
 241		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
 242	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
 243
 244	/*
 245	 * Only reinitialize corresponding vblank timestamp if high-precision query
 246	 * available and didn't fail. Otherwise reinitialize delayed at next vblank
 247	 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
 248	 */
 249	if (!rc)
 250		t_vblank = 0;
 251
 252	/*
 253	 * +1 to make sure user will never see the same
 254	 * vblank counter value before and after a modeset
 255	 */
 256	store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
 257
 258	spin_unlock(&dev->vblank_time_lock);
 259}
 260
 261/*
 262 * Call back into the driver to update the appropriate vblank counter
 263 * (specified by @pipe).  Deal with wraparound, if it occurred, and
 264 * update the last read value so we can deal with wraparound on the next
 265 * call if necessary.
 266 *
 267 * Only necessary when going from off->on, to account for frames we
 268 * didn't get an interrupt for.
 269 *
 270 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
 271 * device vblank fields.
 272 */
 273static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 274				    bool in_vblank_irq)
 275{
 276	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 277	u32 cur_vblank, diff;
 278	bool rc;
 279	ktime_t t_vblank;
 280	int count = DRM_TIMESTAMP_MAXRETRIES;
 281	int framedur_ns = vblank->framedur_ns;
 282	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
 283
 284	/*
 285	 * Interrupts were disabled prior to this call, so deal with counter
 286	 * wrap if needed.
 287	 * NOTE!  It's possible we lost a full dev->max_vblank_count + 1 events
 288	 * here if the register is small or we had vblank interrupts off for
 289	 * a long time.
 290	 *
 291	 * We repeat the hardware vblank counter & timestamp query until
 292	 * we get consistent results. This to prevent races between gpu
 293	 * updating its hardware counter while we are retrieving the
 294	 * corresponding vblank timestamp.
 295	 */
 296	do {
 297		cur_vblank = __get_vblank_counter(dev, pipe);
 298		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
 299	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
 300
 301	if (max_vblank_count) {
 302		/* trust the hw counter when it's around */
 303		diff = (cur_vblank - vblank->last) & max_vblank_count;
 304	} else if (rc && framedur_ns) {
 305		u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
 306
 307		/*
 308		 * Figure out how many vblanks we've missed based
 309		 * on the difference in the timestamps and the
 310		 * frame/field duration.
 311		 */
 312
 313		drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
 314			    " diff_ns = %lld, framedur_ns = %d)\n",
 315			    pipe, (long long)diff_ns, framedur_ns);
 316
 317		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
 318
 319		if (diff == 0 && in_vblank_irq)
 320			drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
 321				    pipe);
 322	} else {
 323		/* some kind of default for drivers w/o accurate vbl timestamping */
 324		diff = in_vblank_irq ? 1 : 0;
 325	}
 326
 327	/*
 328	 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
 329	 * interval? If so then vblank irqs keep running and it will likely
 330	 * happen that the hardware vblank counter is not trustworthy as it
 331	 * might reset at some point in that interval and vblank timestamps
 332	 * are not trustworthy either in that interval. Iow. this can result
 333	 * in a bogus diff >> 1 which must be avoided as it would cause
 334	 * random large forward jumps of the software vblank counter.
 335	 */
 336	if (diff > 1 && (vblank->inmodeset & 0x2)) {
 337		drm_dbg_vbl(dev,
 338			    "clamping vblank bump to 1 on crtc %u: diffr=%u"
 339			    " due to pre-modeset.\n", pipe, diff);
 340		diff = 1;
 341	}
 342
 343	drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
 344		    " current=%llu, diff=%u, hw=%u hw_last=%u\n",
 345		    pipe, (unsigned long long)atomic64_read(&vblank->count),
 346		    diff, cur_vblank, vblank->last);
 347
 348	if (diff == 0) {
 349		drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
 350		return;
 351	}
 352
 353	/*
 354	 * Only reinitialize corresponding vblank timestamp if high-precision query
 355	 * available and didn't fail, or we were called from the vblank interrupt.
 356	 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
 357	 * for now, to mark the vblanktimestamp as invalid.
 358	 */
 359	if (!rc && !in_vblank_irq)
 360		t_vblank = 0;
 361
 362	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
 363}
 364
 365u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
 366{
 367	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 368	u64 count;
 369
 370	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
 371		return 0;
 372
 373	count = atomic64_read(&vblank->count);
 374
 375	/*
 376	 * This read barrier corresponds to the implicit write barrier of the
 377	 * write seqlock in store_vblank(). Note that this is the only place
 378	 * where we need an explicit barrier, since all other access goes
 379	 * through drm_vblank_count_and_time(), which already has the required
 380	 * read barrier curtesy of the read seqlock.
 381	 */
 382	smp_rmb();
 383
 384	return count;
 385}
 386
 387/**
 388 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
 389 * @crtc: which counter to retrieve
 390 *
 391 * This function is similar to drm_crtc_vblank_count() but this function
 392 * interpolates to handle a race with vblank interrupts using the high precision
 393 * timestamping support.
 394 *
 395 * This is mostly useful for hardware that can obtain the scanout position, but
 396 * doesn't have a hardware frame counter.
 397 */
 398u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
 399{
 400	struct drm_device *dev = crtc->dev;
 401	unsigned int pipe = drm_crtc_index(crtc);
 402	u64 vblank;
 403	unsigned long flags;
 404
 405	drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
 406		      !crtc->funcs->get_vblank_timestamp,
 407		      "This function requires support for accurate vblank timestamps.");
 408
 409	spin_lock_irqsave(&dev->vblank_time_lock, flags);
 410
 411	drm_update_vblank_count(dev, pipe, false);
 412	vblank = drm_vblank_count(dev, pipe);
 413
 414	spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
 415
 416	return vblank;
 417}
 418EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
 419
 420static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
 421{
 422	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 423		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 424
 425		if (drm_WARN_ON(dev, !crtc))
 426			return;
 427
 428		if (crtc->funcs->disable_vblank)
 429			crtc->funcs->disable_vblank(crtc);
 430	}
 431}
 432
 433/*
 434 * Disable vblank irq's on crtc, make sure that last vblank count
 435 * of hardware and corresponding consistent software vblank counter
 436 * are preserved, even if there are any spurious vblank irq's after
 437 * disable.
 438 */
 439void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
 440{
 441	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 442	unsigned long irqflags;
 443
 444	assert_spin_locked(&dev->vbl_lock);
 445
 446	/* Prevent vblank irq processing while disabling vblank irqs,
 447	 * so no updates of timestamps or count can happen after we've
 448	 * disabled. Needed to prevent races in case of delayed irq's.
 449	 */
 450	spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 451
 452	/*
 453	 * Update vblank count and disable vblank interrupts only if the
 454	 * interrupts were enabled. This avoids calling the ->disable_vblank()
 455	 * operation in atomic context with the hardware potentially runtime
 456	 * suspended.
 457	 */
 458	if (!vblank->enabled)
 459		goto out;
 460
 461	/*
 462	 * Update the count and timestamp to maintain the
 463	 * appearance that the counter has been ticking all along until
 464	 * this time. This makes the count account for the entire time
 465	 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
 466	 */
 467	drm_update_vblank_count(dev, pipe, false);
 468	__disable_vblank(dev, pipe);
 469	vblank->enabled = false;
 470
 471out:
 472	spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
 473}
 474
 475static void vblank_disable_fn(struct timer_list *t)
 476{
 477	struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
 478	struct drm_device *dev = vblank->dev;
 479	unsigned int pipe = vblank->pipe;
 480	unsigned long irqflags;
 481
 482	spin_lock_irqsave(&dev->vbl_lock, irqflags);
 483	if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
 484		drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
 485		drm_vblank_disable_and_save(dev, pipe);
 486	}
 487	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 488}
 489
 490static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
 491{
 492	struct drm_vblank_crtc *vblank = ptr;
 493
 494	drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
 495		    drm_core_check_feature(dev, DRIVER_MODESET));
 496
 497	drm_vblank_destroy_worker(vblank);
 498	del_timer_sync(&vblank->disable_timer);
 499}
 500
 501/**
 502 * drm_vblank_init - initialize vblank support
 503 * @dev: DRM device
 504 * @num_crtcs: number of CRTCs supported by @dev
 505 *
 506 * This function initializes vblank support for @num_crtcs display pipelines.
 507 * Cleanup is handled automatically through a cleanup function added with
 508 * drmm_add_action_or_reset().
 509 *
 510 * Returns:
 511 * Zero on success or a negative error code on failure.
 512 */
 513int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
 514{
 515	int ret;
 516	unsigned int i;
 517
 518	spin_lock_init(&dev->vbl_lock);
 519	spin_lock_init(&dev->vblank_time_lock);
 520
 521	dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
 522	if (!dev->vblank)
 523		return -ENOMEM;
 524
 525	dev->num_crtcs = num_crtcs;
 526
 527	for (i = 0; i < num_crtcs; i++) {
 528		struct drm_vblank_crtc *vblank = &dev->vblank[i];
 529
 530		vblank->dev = dev;
 531		vblank->pipe = i;
 532		init_waitqueue_head(&vblank->queue);
 533		timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
 534		seqlock_init(&vblank->seqlock);
 535
 536		ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
 537					       vblank);
 538		if (ret)
 539			return ret;
 540
 541		ret = drm_vblank_worker_init(vblank);
 542		if (ret)
 543			return ret;
 544	}
 545
 546	return 0;
 547}
 548EXPORT_SYMBOL(drm_vblank_init);
 549
 550/**
 551 * drm_dev_has_vblank - test if vblanking has been initialized for
 552 *                      a device
 553 * @dev: the device
 554 *
 555 * Drivers may call this function to test if vblank support is
 556 * initialized for a device. For most hardware this means that vblanking
 557 * can also be enabled.
 558 *
 559 * Atomic helpers use this function to initialize
 560 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
 561 *
 562 * Returns:
 563 * True if vblanking has been initialized for the given device, false
 564 * otherwise.
 565 */
 566bool drm_dev_has_vblank(const struct drm_device *dev)
 567{
 568	return dev->num_crtcs != 0;
 569}
 570EXPORT_SYMBOL(drm_dev_has_vblank);
 571
 572/**
 573 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
 574 * @crtc: which CRTC's vblank waitqueue to retrieve
 575 *
 576 * This function returns a pointer to the vblank waitqueue for the CRTC.
 577 * Drivers can use this to implement vblank waits using wait_event() and related
 578 * functions.
 579 */
 580wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
 581{
 582	return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
 583}
 584EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
 585
 586
 587/**
 588 * drm_calc_timestamping_constants - calculate vblank timestamp constants
 589 * @crtc: drm_crtc whose timestamp constants should be updated.
 590 * @mode: display mode containing the scanout timings
 591 *
 592 * Calculate and store various constants which are later needed by vblank and
 593 * swap-completion timestamping, e.g, by
 594 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
 595 * CRTC's true scanout timing, so they take things like panel scaling or
 596 * other adjustments into account.
 597 */
 598void drm_calc_timestamping_constants(struct drm_crtc *crtc,
 599				     const struct drm_display_mode *mode)
 600{
 601	struct drm_device *dev = crtc->dev;
 602	unsigned int pipe = drm_crtc_index(crtc);
 603	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 604	int linedur_ns = 0, framedur_ns = 0;
 605	int dotclock = mode->crtc_clock;
 606
 607	if (!drm_dev_has_vblank(dev))
 608		return;
 609
 610	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
 611		return;
 612
 613	/* Valid dotclock? */
 614	if (dotclock > 0) {
 615		int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
 616
 617		/*
 618		 * Convert scanline length in pixels and video
 619		 * dot clock to line duration and frame duration
 620		 * in nanoseconds:
 621		 */
 622		linedur_ns  = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
 623		framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
 624
 625		/*
 626		 * Fields of interlaced scanout modes are only half a frame duration.
 627		 */
 628		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 629			framedur_ns /= 2;
 630	} else {
 631		drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
 632			crtc->base.id);
 633	}
 634
 635	vblank->linedur_ns  = linedur_ns;
 636	vblank->framedur_ns = framedur_ns;
 637	drm_mode_copy(&vblank->hwmode, mode);
 638
 639	drm_dbg_core(dev,
 640		     "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
 641		     crtc->base.id, mode->crtc_htotal,
 642		     mode->crtc_vtotal, mode->crtc_vdisplay);
 643	drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
 644		     crtc->base.id, dotclock, framedur_ns, linedur_ns);
 645}
 646EXPORT_SYMBOL(drm_calc_timestamping_constants);
 647
 648/**
 649 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
 650 *                                                        timestamp helper
 651 * @crtc: CRTC whose vblank timestamp to retrieve
 652 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
 653 *             On return contains true maximum error of timestamp
 654 * @vblank_time: Pointer to time which should receive the timestamp
 655 * @in_vblank_irq:
 656 *     True when called from drm_crtc_handle_vblank().  Some drivers
 657 *     need to apply some workarounds for gpu-specific vblank irq quirks
 658 *     if flag is set.
 659 * @get_scanout_position:
 660 *     Callback function to retrieve the scanout position. See
 661 *     @struct drm_crtc_helper_funcs.get_scanout_position.
 662 *
 663 * Implements calculation of exact vblank timestamps from given drm_display_mode
 664 * timings and current video scanout position of a CRTC.
 665 *
 666 * The current implementation only handles standard video modes. For double scan
 667 * and interlaced modes the driver is supposed to adjust the hardware mode
 668 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
 669 * match the scanout position reported.
 670 *
 671 * Note that atomic drivers must call drm_calc_timestamping_constants() before
 672 * enabling a CRTC. The atomic helpers already take care of that in
 673 * drm_atomic_helper_calc_timestamping_constants().
 674 *
 675 * Returns:
 676 *
 677 * Returns true on success, and false on failure, i.e. when no accurate
 678 * timestamp could be acquired.
 679 */
 680bool
 681drm_crtc_vblank_helper_get_vblank_timestamp_internal(
 682	struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
 683	bool in_vblank_irq,
 684	drm_vblank_get_scanout_position_func get_scanout_position)
 685{
 686	struct drm_device *dev = crtc->dev;
 687	unsigned int pipe = crtc->index;
 688	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 689	struct timespec64 ts_etime, ts_vblank_time;
 690	ktime_t stime, etime;
 691	bool vbl_status;
 692	const struct drm_display_mode *mode;
 693	int vpos, hpos, i;
 694	int delta_ns, duration_ns;
 695
 696	if (pipe >= dev->num_crtcs) {
 697		drm_err(dev, "Invalid crtc %u\n", pipe);
 698		return false;
 699	}
 700
 701	/* Scanout position query not supported? Should not happen. */
 702	if (!get_scanout_position) {
 703		drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
 704		return false;
 705	}
 706
 707	if (drm_drv_uses_atomic_modeset(dev))
 708		mode = &vblank->hwmode;
 709	else
 710		mode = &crtc->hwmode;
 711
 712	/* If mode timing undefined, just return as no-op:
 713	 * Happens during initial modesetting of a crtc.
 714	 */
 715	if (mode->crtc_clock == 0) {
 716		drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
 717			     pipe);
 718		drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
 719		return false;
 720	}
 721
 722	/* Get current scanout position with system timestamp.
 723	 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
 724	 * if single query takes longer than max_error nanoseconds.
 725	 *
 726	 * This guarantees a tight bound on maximum error if
 727	 * code gets preempted or delayed for some reason.
 728	 */
 729	for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
 730		/*
 731		 * Get vertical and horizontal scanout position vpos, hpos,
 732		 * and bounding timestamps stime, etime, pre/post query.
 733		 */
 734		vbl_status = get_scanout_position(crtc, in_vblank_irq,
 735						  &vpos, &hpos,
 736						  &stime, &etime,
 737						  mode);
 738
 739		/* Return as no-op if scanout query unsupported or failed. */
 740		if (!vbl_status) {
 741			drm_dbg_core(dev,
 742				     "crtc %u : scanoutpos query failed.\n",
 743				     pipe);
 744			return false;
 745		}
 746
 747		/* Compute uncertainty in timestamp of scanout position query. */
 748		duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
 749
 750		/* Accept result with <  max_error nsecs timing uncertainty. */
 751		if (duration_ns <= *max_error)
 752			break;
 753	}
 754
 755	/* Noisy system timing? */
 756	if (i == DRM_TIMESTAMP_MAXRETRIES) {
 757		drm_dbg_core(dev,
 758			     "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
 759			     pipe, duration_ns / 1000, *max_error / 1000, i);
 760	}
 761
 762	/* Return upper bound of timestamp precision error. */
 763	*max_error = duration_ns;
 764
 765	/* Convert scanout position into elapsed time at raw_time query
 766	 * since start of scanout at first display scanline. delta_ns
 767	 * can be negative if start of scanout hasn't happened yet.
 768	 */
 769	delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
 770			   mode->crtc_clock);
 771
 772	/* Subtract time delta from raw timestamp to get final
 773	 * vblank_time timestamp for end of vblank.
 774	 */
 775	*vblank_time = ktime_sub_ns(etime, delta_ns);
 776
 777	if (!drm_debug_enabled(DRM_UT_VBL))
 778		return true;
 779
 780	ts_etime = ktime_to_timespec64(etime);
 781	ts_vblank_time = ktime_to_timespec64(*vblank_time);
 782
 783	drm_dbg_vbl(dev,
 784		    "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
 785		    pipe, hpos, vpos,
 786		    (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
 787		    (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
 788		    duration_ns / 1000, i);
 789
 790	return true;
 791}
 792EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
 793
 794/**
 795 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
 796 *                                               helper
 797 * @crtc: CRTC whose vblank timestamp to retrieve
 798 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
 799 *             On return contains true maximum error of timestamp
 800 * @vblank_time: Pointer to time which should receive the timestamp
 801 * @in_vblank_irq:
 802 *     True when called from drm_crtc_handle_vblank().  Some drivers
 803 *     need to apply some workarounds for gpu-specific vblank irq quirks
 804 *     if flag is set.
 805 *
 806 * Implements calculation of exact vblank timestamps from given drm_display_mode
 807 * timings and current video scanout position of a CRTC. This can be directly
 808 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
 809 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
 810 *
 811 * The current implementation only handles standard video modes. For double scan
 812 * and interlaced modes the driver is supposed to adjust the hardware mode
 813 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
 814 * match the scanout position reported.
 815 *
 816 * Note that atomic drivers must call drm_calc_timestamping_constants() before
 817 * enabling a CRTC. The atomic helpers already take care of that in
 818 * drm_atomic_helper_calc_timestamping_constants().
 819 *
 820 * Returns:
 821 *
 822 * Returns true on success, and false on failure, i.e. when no accurate
 823 * timestamp could be acquired.
 824 */
 825bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
 826						 int *max_error,
 827						 ktime_t *vblank_time,
 828						 bool in_vblank_irq)
 829{
 830	return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
 831		crtc, max_error, vblank_time, in_vblank_irq,
 832		crtc->helper_private->get_scanout_position);
 833}
 834EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
 835
 836/**
 837 * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most
 838 *                                  recent vblank interval
 839 * @crtc: CRTC whose vblank timestamp to retrieve
 840 * @tvblank: Pointer to target time which should receive the timestamp
 841 * @in_vblank_irq:
 842 *     True when called from drm_crtc_handle_vblank().  Some drivers
 843 *     need to apply some workarounds for gpu-specific vblank irq quirks
 844 *     if flag is set.
 845 *
 846 * Fetches the system timestamp corresponding to the time of the most recent
 847 * vblank interval on specified CRTC. May call into kms-driver to
 848 * compute the timestamp with a high-precision GPU specific method.
 849 *
 850 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
 851 * call, i.e., it isn't very precisely locked to the true vblank.
 852 *
 853 * Returns:
 854 * True if timestamp is considered to be very precise, false otherwise.
 855 */
 856static bool
 857drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank,
 858			       bool in_vblank_irq)
 859{
 860	bool ret = false;
 861
 862	/* Define requested maximum error on timestamps (nanoseconds). */
 863	int max_error = (int) drm_timestamp_precision * 1000;
 864
 865	/* Query driver if possible and precision timestamping enabled. */
 866	if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
 867		ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
 868							tvblank, in_vblank_irq);
 869	}
 870
 871	/* GPU high precision timestamp query unsupported or failed.
 872	 * Return current monotonic/gettimeofday timestamp as best estimate.
 873	 */
 874	if (!ret)
 875		*tvblank = ktime_get();
 876
 877	return ret;
 878}
 879
 880static bool
 881drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 882			  ktime_t *tvblank, bool in_vblank_irq)
 883{
 884	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 885
 886	return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq);
 887}
 888
 889/**
 890 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
 891 * @crtc: which counter to retrieve
 892 *
 893 * Fetches the "cooked" vblank count value that represents the number of
 894 * vblank events since the system was booted, including lost events due to
 895 * modesetting activity. Note that this timer isn't correct against a racing
 896 * vblank interrupt (since it only reports the software vblank counter), see
 897 * drm_crtc_accurate_vblank_count() for such use-cases.
 898 *
 899 * Note that for a given vblank counter value drm_crtc_handle_vblank()
 900 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
 901 * provide a barrier: Any writes done before calling
 902 * drm_crtc_handle_vblank() will be visible to callers of the later
 903 * functions, if the vblank count is the same or a later one.
 904 *
 905 * See also &drm_vblank_crtc.count.
 906 *
 907 * Returns:
 908 * The software vblank counter.
 909 */
 910u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
 911{
 912	return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
 913}
 914EXPORT_SYMBOL(drm_crtc_vblank_count);
 915
 916/**
 917 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
 918 *     system timestamp corresponding to that vblank counter value.
 919 * @dev: DRM device
 920 * @pipe: index of CRTC whose counter to retrieve
 921 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
 922 *
 923 * Fetches the "cooked" vblank count value that represents the number of
 924 * vblank events since the system was booted, including lost events due to
 925 * modesetting activity. Returns corresponding system timestamp of the time
 926 * of the vblank interval that corresponds to the current vblank counter value.
 927 *
 928 * This is the legacy version of drm_crtc_vblank_count_and_time().
 929 */
 930static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
 931				     ktime_t *vblanktime)
 932{
 933	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 934	u64 vblank_count;
 935	unsigned int seq;
 936
 937	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
 938		*vblanktime = 0;
 939		return 0;
 940	}
 941
 942	do {
 943		seq = read_seqbegin(&vblank->seqlock);
 944		vblank_count = atomic64_read(&vblank->count);
 945		*vblanktime = vblank->time;
 946	} while (read_seqretry(&vblank->seqlock, seq));
 947
 948	return vblank_count;
 949}
 950
 951/**
 952 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
 953 *     and the system timestamp corresponding to that vblank counter value
 954 * @crtc: which counter to retrieve
 955 * @vblanktime: Pointer to time to receive the vblank timestamp.
 956 *
 957 * Fetches the "cooked" vblank count value that represents the number of
 958 * vblank events since the system was booted, including lost events due to
 959 * modesetting activity. Returns corresponding system timestamp of the time
 960 * of the vblank interval that corresponds to the current vblank counter value.
 961 *
 962 * Note that for a given vblank counter value drm_crtc_handle_vblank()
 963 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
 964 * provide a barrier: Any writes done before calling
 965 * drm_crtc_handle_vblank() will be visible to callers of the later
 966 * functions, if the vblank count is the same or a later one.
 967 *
 968 * See also &drm_vblank_crtc.count.
 969 */
 970u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
 971				   ktime_t *vblanktime)
 972{
 973	return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
 974					 vblanktime);
 975}
 976EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
 977
 978/**
 979 * drm_crtc_next_vblank_start - calculate the time of the next vblank
 980 * @crtc: the crtc for which to calculate next vblank time
 981 * @vblanktime: pointer to time to receive the next vblank timestamp.
 982 *
 983 * Calculate the expected time of the start of the next vblank period,
 984 * based on time of previous vblank and frame duration
 985 */
 986int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
 987{
 988	unsigned int pipe = drm_crtc_index(crtc);
 989	struct drm_vblank_crtc *vblank;
 990	struct drm_display_mode *mode;
 991	u64 vblank_start;
 992
 993	if (!drm_dev_has_vblank(crtc->dev))
 994		return -EINVAL;
 995
 996	vblank = &crtc->dev->vblank[pipe];
 997	mode = &vblank->hwmode;
 998
 999	if (!vblank->framedur_ns || !vblank->linedur_ns)
1000		return -EINVAL;
1001
1002	if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false))
1003		return -EINVAL;
1004
1005	vblank_start = DIV_ROUND_DOWN_ULL(
1006			(u64)vblank->framedur_ns * mode->crtc_vblank_start,
1007			mode->crtc_vtotal);
1008	*vblanktime  = ktime_add(*vblanktime, ns_to_ktime(vblank_start));
1009
1010	return 0;
1011}
1012EXPORT_SYMBOL(drm_crtc_next_vblank_start);
1013
1014static void send_vblank_event(struct drm_device *dev,
1015		struct drm_pending_vblank_event *e,
1016		u64 seq, ktime_t now)
1017{
1018	struct timespec64 tv;
1019
1020	switch (e->event.base.type) {
1021	case DRM_EVENT_VBLANK:
1022	case DRM_EVENT_FLIP_COMPLETE:
1023		tv = ktime_to_timespec64(now);
1024		e->event.vbl.sequence = seq;
1025		/*
1026		 * e->event is a user space structure, with hardcoded unsigned
1027		 * 32-bit seconds/microseconds. This is safe as we always use
1028		 * monotonic timestamps since linux-4.15
1029		 */
1030		e->event.vbl.tv_sec = tv.tv_sec;
1031		e->event.vbl.tv_usec = tv.tv_nsec / 1000;
1032		break;
1033	case DRM_EVENT_CRTC_SEQUENCE:
1034		if (seq)
1035			e->event.seq.sequence = seq;
1036		e->event.seq.time_ns = ktime_to_ns(now);
1037		break;
1038	}
1039	trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
1040	/*
1041	 * Use the same timestamp for any associated fence signal to avoid
1042	 * mismatch in timestamps for vsync & fence events triggered by the
1043	 * same HW event. Frameworks like SurfaceFlinger in Android expects the
1044	 * retire-fence timestamp to match exactly with HW vsync as it uses it
1045	 * for its software vsync modeling.
1046	 */
1047	drm_send_event_timestamp_locked(dev, &e->base, now);
1048}
1049
1050/**
1051 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1052 * @crtc: the source CRTC of the vblank event
1053 * @e: the event to send
1054 *
1055 * A lot of drivers need to generate vblank events for the very next vblank
1056 * interrupt. For example when the page flip interrupt happens when the page
1057 * flip gets armed, but not when it actually executes within the next vblank
1058 * period. This helper function implements exactly the required vblank arming
1059 * behaviour.
1060 *
1061 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1062 * atomic commit must ensure that the next vblank happens at exactly the same
1063 * time as the atomic commit is committed to the hardware. This function itself
1064 * does **not** protect against the next vblank interrupt racing with either this
1065 * function call or the atomic commit operation. A possible sequence could be:
1066 *
1067 * 1. Driver commits new hardware state into vblank-synchronized registers.
1068 * 2. A vblank happens, committing the hardware state. Also the corresponding
1069 *    vblank interrupt is fired off and fully processed by the interrupt
1070 *    handler.
1071 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1072 * 4. The event is only send out for the next vblank, which is wrong.
1073 *
1074 * An equivalent race can happen when the driver calls
1075 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
1076 *
1077 * The only way to make this work safely is to prevent the vblank from firing
1078 * (and the hardware from committing anything else) until the entire atomic
1079 * commit sequence has run to completion. If the hardware does not have such a
1080 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1081 * Instead drivers need to manually send out the event from their interrupt
1082 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1083 * possible race with the hardware committing the atomic update.
1084 *
1085 * Caller must hold a vblank reference for the event @e acquired by a
1086 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1087 */
1088void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1089			       struct drm_pending_vblank_event *e)
1090{
1091	struct drm_device *dev = crtc->dev;
1092	unsigned int pipe = drm_crtc_index(crtc);
1093
1094	assert_spin_locked(&dev->event_lock);
1095
1096	e->pipe = pipe;
1097	e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1098	list_add_tail(&e->base.link, &dev->vblank_event_list);
1099}
1100EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1101
1102/**
1103 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1104 * @crtc: the source CRTC of the vblank event
1105 * @e: the event to send
1106 *
1107 * Updates sequence # and timestamp on event for the most recently processed
1108 * vblank, and sends it to userspace.  Caller must hold event lock.
1109 *
1110 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1111 * situation, especially to send out events for atomic commit operations.
1112 */
1113void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1114				struct drm_pending_vblank_event *e)
1115{
1116	struct drm_device *dev = crtc->dev;
1117	u64 seq;
1118	unsigned int pipe = drm_crtc_index(crtc);
1119	ktime_t now;
1120
1121	if (drm_dev_has_vblank(dev)) {
1122		seq = drm_vblank_count_and_time(dev, pipe, &now);
1123	} else {
1124		seq = 0;
1125
1126		now = ktime_get();
1127	}
1128	e->pipe = pipe;
1129	send_vblank_event(dev, e, seq, now);
1130}
1131EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1132
1133static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1134{
1135	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1136		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1137
1138		if (drm_WARN_ON(dev, !crtc))
1139			return 0;
1140
1141		if (crtc->funcs->enable_vblank)
1142			return crtc->funcs->enable_vblank(crtc);
1143	}
1144
1145	return -EINVAL;
1146}
1147
1148static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1149{
1150	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1151	int ret = 0;
1152
1153	assert_spin_locked(&dev->vbl_lock);
1154
1155	spin_lock(&dev->vblank_time_lock);
1156
1157	if (!vblank->enabled) {
1158		/*
1159		 * Enable vblank irqs under vblank_time_lock protection.
1160		 * All vblank count & timestamp updates are held off
1161		 * until we are done reinitializing master counter and
1162		 * timestamps. Filtercode in drm_handle_vblank() will
1163		 * prevent double-accounting of same vblank interval.
1164		 */
1165		ret = __enable_vblank(dev, pipe);
1166		drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
1167			     pipe, ret);
1168		if (ret) {
1169			atomic_dec(&vblank->refcount);
1170		} else {
1171			drm_update_vblank_count(dev, pipe, 0);
1172			/* drm_update_vblank_count() includes a wmb so we just
1173			 * need to ensure that the compiler emits the write
1174			 * to mark the vblank as enabled after the call
1175			 * to drm_update_vblank_count().
1176			 */
1177			WRITE_ONCE(vblank->enabled, true);
1178		}
1179	}
1180
1181	spin_unlock(&dev->vblank_time_lock);
1182
1183	return ret;
1184}
1185
1186int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1187{
1188	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1189	unsigned long irqflags;
1190	int ret = 0;
1191
1192	if (!drm_dev_has_vblank(dev))
1193		return -EINVAL;
1194
1195	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1196		return -EINVAL;
1197
1198	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1199	/* Going from 0->1 means we have to enable interrupts again */
1200	if (atomic_add_return(1, &vblank->refcount) == 1) {
1201		ret = drm_vblank_enable(dev, pipe);
1202	} else {
1203		if (!vblank->enabled) {
1204			atomic_dec(&vblank->refcount);
1205			ret = -EINVAL;
1206		}
1207	}
1208	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1209
1210	return ret;
1211}
1212
1213/**
1214 * drm_crtc_vblank_get - get a reference count on vblank events
1215 * @crtc: which CRTC to own
1216 *
1217 * Acquire a reference count on vblank events to avoid having them disabled
1218 * while in use.
1219 *
1220 * Returns:
1221 * Zero on success or a negative error code on failure.
1222 */
1223int drm_crtc_vblank_get(struct drm_crtc *crtc)
1224{
1225	return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1226}
1227EXPORT_SYMBOL(drm_crtc_vblank_get);
1228
1229void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1230{
1231	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1232
1233	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1234		return;
1235
1236	if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
1237		return;
1238
1239	/* Last user schedules interrupt disable */
1240	if (atomic_dec_and_test(&vblank->refcount)) {
1241		if (drm_vblank_offdelay == 0)
1242			return;
1243		else if (drm_vblank_offdelay < 0)
1244			vblank_disable_fn(&vblank->disable_timer);
1245		else if (!dev->vblank_disable_immediate)
1246			mod_timer(&vblank->disable_timer,
1247				  jiffies + ((drm_vblank_offdelay * HZ)/1000));
1248	}
1249}
1250
1251/**
1252 * drm_crtc_vblank_put - give up ownership of vblank events
1253 * @crtc: which counter to give up
1254 *
1255 * Release ownership of a given vblank counter, turning off interrupts
1256 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1257 */
1258void drm_crtc_vblank_put(struct drm_crtc *crtc)
1259{
1260	drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1261}
1262EXPORT_SYMBOL(drm_crtc_vblank_put);
1263
1264/**
1265 * drm_wait_one_vblank - wait for one vblank
1266 * @dev: DRM device
1267 * @pipe: CRTC index
1268 *
1269 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1270 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1271 * due to lack of driver support or because the crtc is off.
1272 *
1273 * This is the legacy version of drm_crtc_wait_one_vblank().
1274 */
1275void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1276{
1277	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1278	int ret;
1279	u64 last;
1280
1281	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1282		return;
1283
1284	ret = drm_vblank_get(dev, pipe);
1285	if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
1286		     pipe, ret))
1287		return;
1288
1289	last = drm_vblank_count(dev, pipe);
1290
1291	ret = wait_event_timeout(vblank->queue,
1292				 last != drm_vblank_count(dev, pipe),
1293				 msecs_to_jiffies(100));
1294
1295	drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1296
1297	drm_vblank_put(dev, pipe);
1298}
1299EXPORT_SYMBOL(drm_wait_one_vblank);
1300
1301/**
1302 * drm_crtc_wait_one_vblank - wait for one vblank
1303 * @crtc: DRM crtc
1304 *
1305 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1306 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1307 * due to lack of driver support or because the crtc is off.
1308 */
1309void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1310{
1311	drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1312}
1313EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1314
1315/**
1316 * drm_crtc_vblank_off - disable vblank events on a CRTC
1317 * @crtc: CRTC in question
1318 *
1319 * Drivers can use this function to shut down the vblank interrupt handling when
1320 * disabling a crtc. This function ensures that the latest vblank frame count is
1321 * stored so that drm_vblank_on can restore it again.
1322 *
1323 * Drivers must use this function when the hardware vblank counter can get
1324 * reset, e.g. when suspending or disabling the @crtc in general.
1325 */
1326void drm_crtc_vblank_off(struct drm_crtc *crtc)
1327{
1328	struct drm_device *dev = crtc->dev;
1329	unsigned int pipe = drm_crtc_index(crtc);
1330	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1331	struct drm_pending_vblank_event *e, *t;
1332	ktime_t now;
1333	u64 seq;
1334
1335	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1336		return;
1337
1338	/*
1339	 * Grab event_lock early to prevent vblank work from being scheduled
1340	 * while we're in the middle of shutting down vblank interrupts
1341	 */
1342	spin_lock_irq(&dev->event_lock);
1343
1344	spin_lock(&dev->vbl_lock);
1345	drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1346		    pipe, vblank->enabled, vblank->inmodeset);
1347
1348	/* Avoid redundant vblank disables without previous
1349	 * drm_crtc_vblank_on(). */
1350	if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1351		drm_vblank_disable_and_save(dev, pipe);
1352
1353	wake_up(&vblank->queue);
1354
1355	/*
1356	 * Prevent subsequent drm_vblank_get() from re-enabling
1357	 * the vblank interrupt by bumping the refcount.
1358	 */
1359	if (!vblank->inmodeset) {
1360		atomic_inc(&vblank->refcount);
1361		vblank->inmodeset = 1;
1362	}
1363	spin_unlock(&dev->vbl_lock);
1364
1365	/* Send any queued vblank events, lest the natives grow disquiet */
1366	seq = drm_vblank_count_and_time(dev, pipe, &now);
1367
1368	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1369		if (e->pipe != pipe)
1370			continue;
1371		drm_dbg_core(dev, "Sending premature vblank event on disable: "
1372			     "wanted %llu, current %llu\n",
1373			     e->sequence, seq);
1374		list_del(&e->base.link);
1375		drm_vblank_put(dev, pipe);
1376		send_vblank_event(dev, e, seq, now);
1377	}
1378
1379	/* Cancel any leftover pending vblank work */
1380	drm_vblank_cancel_pending_works(vblank);
1381
1382	spin_unlock_irq(&dev->event_lock);
1383
1384	/* Will be reset by the modeset helpers when re-enabling the crtc by
1385	 * calling drm_calc_timestamping_constants(). */
1386	vblank->hwmode.crtc_clock = 0;
1387
1388	/* Wait for any vblank work that's still executing to finish */
1389	drm_vblank_flush_worker(vblank);
1390}
1391EXPORT_SYMBOL(drm_crtc_vblank_off);
1392
1393/**
1394 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1395 * @crtc: CRTC in question
1396 *
1397 * Drivers can use this function to reset the vblank state to off at load time.
1398 * Drivers should use this together with the drm_crtc_vblank_off() and
1399 * drm_crtc_vblank_on() functions. The difference compared to
1400 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1401 * and hence doesn't need to call any driver hooks.
1402 *
1403 * This is useful for recovering driver state e.g. on driver load, or on resume.
1404 */
1405void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1406{
1407	struct drm_device *dev = crtc->dev;
1408	unsigned int pipe = drm_crtc_index(crtc);
1409	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1410
1411	spin_lock_irq(&dev->vbl_lock);
1412	/*
1413	 * Prevent subsequent drm_vblank_get() from enabling the vblank
1414	 * interrupt by bumping the refcount.
1415	 */
1416	if (!vblank->inmodeset) {
1417		atomic_inc(&vblank->refcount);
1418		vblank->inmodeset = 1;
1419	}
1420	spin_unlock_irq(&dev->vbl_lock);
1421
1422	drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
1423	drm_WARN_ON(dev, !list_empty(&vblank->pending_work));
1424}
1425EXPORT_SYMBOL(drm_crtc_vblank_reset);
1426
1427/**
1428 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1429 * @crtc: CRTC in question
1430 * @max_vblank_count: max hardware vblank counter value
1431 *
1432 * Update the maximum hardware vblank counter value for @crtc
1433 * at runtime. Useful for hardware where the operation of the
1434 * hardware vblank counter depends on the currently active
1435 * display configuration.
1436 *
1437 * For example, if the hardware vblank counter does not work
1438 * when a specific connector is active the maximum can be set
1439 * to zero. And when that specific connector isn't active the
1440 * maximum can again be set to the appropriate non-zero value.
1441 *
1442 * If used, must be called before drm_vblank_on().
1443 */
1444void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1445				   u32 max_vblank_count)
1446{
1447	struct drm_device *dev = crtc->dev;
1448	unsigned int pipe = drm_crtc_index(crtc);
1449	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1450
1451	drm_WARN_ON(dev, dev->max_vblank_count);
1452	drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
1453
1454	vblank->max_vblank_count = max_vblank_count;
1455}
1456EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1457
1458/**
1459 * drm_crtc_vblank_on - enable vblank events on a CRTC
1460 * @crtc: CRTC in question
1461 *
1462 * This functions restores the vblank interrupt state captured with
1463 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1464 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1465 * unbalanced and so can also be unconditionally called in driver load code to
1466 * reflect the current hardware state of the crtc.
1467 */
1468void drm_crtc_vblank_on(struct drm_crtc *crtc)
1469{
1470	struct drm_device *dev = crtc->dev;
1471	unsigned int pipe = drm_crtc_index(crtc);
1472	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1473
1474	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1475		return;
1476
1477	spin_lock_irq(&dev->vbl_lock);
1478	drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1479		    pipe, vblank->enabled, vblank->inmodeset);
1480
1481	/* Drop our private "prevent drm_vblank_get" refcount */
1482	if (vblank->inmodeset) {
1483		atomic_dec(&vblank->refcount);
1484		vblank->inmodeset = 0;
1485	}
1486
1487	drm_reset_vblank_timestamp(dev, pipe);
1488
1489	/*
1490	 * re-enable interrupts if there are users left, or the
1491	 * user wishes vblank interrupts to be enabled all the time.
1492	 */
1493	if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1494		drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
1495	spin_unlock_irq(&dev->vbl_lock);
1496}
1497EXPORT_SYMBOL(drm_crtc_vblank_on);
1498
1499static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1500{
1501	ktime_t t_vblank;
1502	struct drm_vblank_crtc *vblank;
1503	int framedur_ns;
1504	u64 diff_ns;
1505	u32 cur_vblank, diff = 1;
1506	int count = DRM_TIMESTAMP_MAXRETRIES;
1507	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
1508
1509	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1510		return;
1511
1512	assert_spin_locked(&dev->vbl_lock);
1513	assert_spin_locked(&dev->vblank_time_lock);
1514
1515	vblank = &dev->vblank[pipe];
1516	drm_WARN_ONCE(dev,
1517		      drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
1518		      "Cannot compute missed vblanks without frame duration\n");
1519	framedur_ns = vblank->framedur_ns;
1520
1521	do {
1522		cur_vblank = __get_vblank_counter(dev, pipe);
1523		drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1524	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1525
1526	diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1527	if (framedur_ns)
1528		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1529
1530
1531	drm_dbg_vbl(dev,
1532		    "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1533		    diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1534	vblank->last = (cur_vblank - diff) & max_vblank_count;
1535}
1536
1537/**
1538 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1539 * @crtc: CRTC in question
1540 *
1541 * Power manamement features can cause frame counter resets between vblank
1542 * disable and enable. Drivers can use this function in their
1543 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1544 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1545 * vblank counter.
1546 *
1547 * Note that drivers must have race-free high-precision timestamping support,
1548 * i.e.  &drm_crtc_funcs.get_vblank_timestamp must be hooked up and
1549 * &drm_driver.vblank_disable_immediate must be set to indicate the
1550 * time-stamping functions are race-free against vblank hardware counter
1551 * increments.
1552 */
1553void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1554{
1555	WARN_ON_ONCE(!crtc->funcs->get_vblank_timestamp);
1556	WARN_ON_ONCE(!crtc->dev->vblank_disable_immediate);
1557
1558	drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1559}
1560EXPORT_SYMBOL(drm_crtc_vblank_restore);
1561
1562static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1563				  u64 req_seq,
1564				  union drm_wait_vblank *vblwait,
1565				  struct drm_file *file_priv)
1566{
1567	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1568	struct drm_pending_vblank_event *e;
1569	ktime_t now;
1570	u64 seq;
1571	int ret;
1572
1573	e = kzalloc(sizeof(*e), GFP_KERNEL);
1574	if (e == NULL) {
1575		ret = -ENOMEM;
1576		goto err_put;
1577	}
1578
1579	e->pipe = pipe;
1580	e->event.base.type = DRM_EVENT_VBLANK;
1581	e->event.base.length = sizeof(e->event.vbl);
1582	e->event.vbl.user_data = vblwait->request.signal;
1583	e->event.vbl.crtc_id = 0;
1584	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1585		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1586
1587		if (crtc)
1588			e->event.vbl.crtc_id = crtc->base.id;
1589	}
1590
1591	spin_lock_irq(&dev->event_lock);
1592
1593	/*
1594	 * drm_crtc_vblank_off() might have been called after we called
1595	 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1596	 * vblank disable, so no need for further locking.  The reference from
1597	 * drm_vblank_get() protects against vblank disable from another source.
1598	 */
1599	if (!READ_ONCE(vblank->enabled)) {
1600		ret = -EINVAL;
1601		goto err_unlock;
1602	}
1603
1604	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1605					    &e->event.base);
1606
1607	if (ret)
1608		goto err_unlock;
1609
1610	seq = drm_vblank_count_and_time(dev, pipe, &now);
1611
1612	drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
1613		     req_seq, seq, pipe);
1614
1615	trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1616
1617	e->sequence = req_seq;
1618	if (drm_vblank_passed(seq, req_seq)) {
1619		drm_vblank_put(dev, pipe);
1620		send_vblank_event(dev, e, seq, now);
1621		vblwait->reply.sequence = seq;
1622	} else {
1623		/* drm_handle_vblank_events will call drm_vblank_put */
1624		list_add_tail(&e->base.link, &dev->vblank_event_list);
1625		vblwait->reply.sequence = req_seq;
1626	}
1627
1628	spin_unlock_irq(&dev->event_lock);
1629
1630	return 0;
1631
1632err_unlock:
1633	spin_unlock_irq(&dev->event_lock);
1634	kfree(e);
1635err_put:
1636	drm_vblank_put(dev, pipe);
1637	return ret;
1638}
1639
1640static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1641{
1642	if (vblwait->request.sequence)
1643		return false;
1644
1645	return _DRM_VBLANK_RELATIVE ==
1646		(vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1647					  _DRM_VBLANK_EVENT |
1648					  _DRM_VBLANK_NEXTONMISS));
1649}
1650
1651/*
1652 * Widen a 32-bit param to 64-bits.
1653 *
1654 * \param narrow 32-bit value (missing upper 32 bits)
1655 * \param near 64-bit value that should be 'close' to near
1656 *
1657 * This function returns a 64-bit value using the lower 32-bits from
1658 * 'narrow' and constructing the upper 32-bits so that the result is
1659 * as close as possible to 'near'.
1660 */
1661
1662static u64 widen_32_to_64(u32 narrow, u64 near)
1663{
1664	return near + (s32) (narrow - near);
1665}
1666
1667static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1668				  struct drm_wait_vblank_reply *reply)
1669{
1670	ktime_t now;
1671	struct timespec64 ts;
1672
1673	/*
1674	 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1675	 * to store the seconds. This is safe as we always use monotonic
1676	 * timestamps since linux-4.15.
1677	 */
1678	reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1679	ts = ktime_to_timespec64(now);
1680	reply->tval_sec = (u32)ts.tv_sec;
1681	reply->tval_usec = ts.tv_nsec / 1000;
1682}
1683
1684static bool drm_wait_vblank_supported(struct drm_device *dev)
1685{
1686	return drm_dev_has_vblank(dev);
1687}
1688
1689int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1690			  struct drm_file *file_priv)
1691{
1692	struct drm_crtc *crtc;
1693	struct drm_vblank_crtc *vblank;
1694	union drm_wait_vblank *vblwait = data;
1695	int ret;
1696	u64 req_seq, seq;
1697	unsigned int pipe_index;
1698	unsigned int flags, pipe, high_pipe;
1699
1700	if (!drm_wait_vblank_supported(dev))
1701		return -EOPNOTSUPP;
1702
1703	if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1704		return -EINVAL;
1705
1706	if (vblwait->request.type &
1707	    ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1708	      _DRM_VBLANK_HIGH_CRTC_MASK)) {
1709		drm_dbg_core(dev,
1710			     "Unsupported type value 0x%x, supported mask 0x%x\n",
1711			     vblwait->request.type,
1712			     (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1713			      _DRM_VBLANK_HIGH_CRTC_MASK));
1714		return -EINVAL;
1715	}
1716
1717	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1718	high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1719	if (high_pipe)
1720		pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1721	else
1722		pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1723
1724	/* Convert lease-relative crtc index into global crtc index */
1725	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1726		pipe = 0;
1727		drm_for_each_crtc(crtc, dev) {
1728			if (drm_lease_held(file_priv, crtc->base.id)) {
1729				if (pipe_index == 0)
1730					break;
1731				pipe_index--;
1732			}
1733			pipe++;
1734		}
1735	} else {
1736		pipe = pipe_index;
1737	}
1738
1739	if (pipe >= dev->num_crtcs)
1740		return -EINVAL;
1741
1742	vblank = &dev->vblank[pipe];
1743
1744	/* If the counter is currently enabled and accurate, short-circuit
1745	 * queries to return the cached timestamp of the last vblank.
1746	 */
1747	if (dev->vblank_disable_immediate &&
1748	    drm_wait_vblank_is_query(vblwait) &&
1749	    READ_ONCE(vblank->enabled)) {
1750		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1751		return 0;
1752	}
1753
1754	ret = drm_vblank_get(dev, pipe);
1755	if (ret) {
1756		drm_dbg_core(dev,
1757			     "crtc %d failed to acquire vblank counter, %d\n",
1758			     pipe, ret);
1759		return ret;
1760	}
1761	seq = drm_vblank_count(dev, pipe);
1762
1763	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1764	case _DRM_VBLANK_RELATIVE:
1765		req_seq = seq + vblwait->request.sequence;
1766		vblwait->request.sequence = req_seq;
1767		vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1768		break;
1769	case _DRM_VBLANK_ABSOLUTE:
1770		req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1771		break;
1772	default:
1773		ret = -EINVAL;
1774		goto done;
1775	}
1776
1777	if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1778	    drm_vblank_passed(seq, req_seq)) {
1779		req_seq = seq + 1;
1780		vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1781		vblwait->request.sequence = req_seq;
1782	}
1783
1784	if (flags & _DRM_VBLANK_EVENT) {
1785		/* must hold on to the vblank ref until the event fires
1786		 * drm_vblank_put will be called asynchronously
1787		 */
1788		return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1789	}
1790
1791	if (req_seq != seq) {
1792		int wait;
1793
1794		drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
1795			     req_seq, pipe);
1796		wait = wait_event_interruptible_timeout(vblank->queue,
1797			drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1798				      !READ_ONCE(vblank->enabled),
1799			msecs_to_jiffies(3000));
1800
1801		switch (wait) {
1802		case 0:
1803			/* timeout */
1804			ret = -EBUSY;
1805			break;
1806		case -ERESTARTSYS:
1807			/* interrupted by signal */
1808			ret = -EINTR;
1809			break;
1810		default:
1811			ret = 0;
1812			break;
1813		}
1814	}
1815
1816	if (ret != -EINTR) {
1817		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1818
1819		drm_dbg_core(dev, "crtc %d returning %u to client\n",
1820			     pipe, vblwait->reply.sequence);
1821	} else {
1822		drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
1823			     pipe);
1824	}
1825
1826done:
1827	drm_vblank_put(dev, pipe);
1828	return ret;
1829}
1830
1831static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1832{
1833	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1834	bool high_prec = false;
1835	struct drm_pending_vblank_event *e, *t;
1836	ktime_t now;
1837	u64 seq;
1838
1839	assert_spin_locked(&dev->event_lock);
1840
1841	seq = drm_vblank_count_and_time(dev, pipe, &now);
1842
1843	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1844		if (e->pipe != pipe)
1845			continue;
1846		if (!drm_vblank_passed(seq, e->sequence))
1847			continue;
1848
1849		drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
1850			     e->sequence, seq);
1851
1852		list_del(&e->base.link);
1853		drm_vblank_put(dev, pipe);
1854		send_vblank_event(dev, e, seq, now);
1855	}
1856
1857	if (crtc && crtc->funcs->get_vblank_timestamp)
1858		high_prec = true;
1859
1860	trace_drm_vblank_event(pipe, seq, now, high_prec);
1861}
1862
1863/**
1864 * drm_handle_vblank - handle a vblank event
1865 * @dev: DRM device
1866 * @pipe: index of CRTC where this event occurred
1867 *
1868 * Drivers should call this routine in their vblank interrupt handlers to
1869 * update the vblank counter and send any signals that may be pending.
1870 *
1871 * This is the legacy version of drm_crtc_handle_vblank().
1872 */
1873bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1874{
1875	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1876	unsigned long irqflags;
1877	bool disable_irq;
1878
1879	if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
1880		return false;
1881
1882	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1883		return false;
1884
1885	spin_lock_irqsave(&dev->event_lock, irqflags);
1886
1887	/* Need timestamp lock to prevent concurrent execution with
1888	 * vblank enable/disable, as this would cause inconsistent
1889	 * or corrupted timestamps and vblank counts.
1890	 */
1891	spin_lock(&dev->vblank_time_lock);
1892
1893	/* Vblank irq handling disabled. Nothing to do. */
1894	if (!vblank->enabled) {
1895		spin_unlock(&dev->vblank_time_lock);
1896		spin_unlock_irqrestore(&dev->event_lock, irqflags);
1897		return false;
1898	}
1899
1900	drm_update_vblank_count(dev, pipe, true);
1901
1902	spin_unlock(&dev->vblank_time_lock);
1903
1904	wake_up(&vblank->queue);
1905
1906	/* With instant-off, we defer disabling the interrupt until after
1907	 * we finish processing the following vblank after all events have
1908	 * been signaled. The disable has to be last (after
1909	 * drm_handle_vblank_events) so that the timestamp is always accurate.
1910	 */
1911	disable_irq = (dev->vblank_disable_immediate &&
1912		       drm_vblank_offdelay > 0 &&
1913		       !atomic_read(&vblank->refcount));
1914
1915	drm_handle_vblank_events(dev, pipe);
1916	drm_handle_vblank_works(vblank);
1917
1918	spin_unlock_irqrestore(&dev->event_lock, irqflags);
1919
1920	if (disable_irq)
1921		vblank_disable_fn(&vblank->disable_timer);
1922
1923	return true;
1924}
1925EXPORT_SYMBOL(drm_handle_vblank);
1926
1927/**
1928 * drm_crtc_handle_vblank - handle a vblank event
1929 * @crtc: where this event occurred
1930 *
1931 * Drivers should call this routine in their vblank interrupt handlers to
1932 * update the vblank counter and send any signals that may be pending.
1933 *
1934 * This is the native KMS version of drm_handle_vblank().
1935 *
1936 * Note that for a given vblank counter value drm_crtc_handle_vblank()
1937 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
1938 * provide a barrier: Any writes done before calling
1939 * drm_crtc_handle_vblank() will be visible to callers of the later
1940 * functions, if the vblank count is the same or a later one.
1941 *
1942 * See also &drm_vblank_crtc.count.
1943 *
1944 * Returns:
1945 * True if the event was successfully handled, false on failure.
1946 */
1947bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1948{
1949	return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1950}
1951EXPORT_SYMBOL(drm_crtc_handle_vblank);
1952
1953/*
1954 * Get crtc VBLANK count.
1955 *
1956 * \param dev DRM device
1957 * \param data user argument, pointing to a drm_crtc_get_sequence structure.
1958 * \param file_priv drm file private for the user's open file descriptor
1959 */
1960
1961int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1962				struct drm_file *file_priv)
1963{
1964	struct drm_crtc *crtc;
1965	struct drm_vblank_crtc *vblank;
1966	int pipe;
1967	struct drm_crtc_get_sequence *get_seq = data;
1968	ktime_t now;
1969	bool vblank_enabled;
1970	int ret;
1971
1972	if (!drm_core_check_feature(dev, DRIVER_MODESET))
1973		return -EOPNOTSUPP;
1974
1975	if (!drm_dev_has_vblank(dev))
1976		return -EOPNOTSUPP;
1977
1978	crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
1979	if (!crtc)
1980		return -ENOENT;
1981
1982	pipe = drm_crtc_index(crtc);
1983
1984	vblank = &dev->vblank[pipe];
1985	vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
1986
1987	if (!vblank_enabled) {
1988		ret = drm_crtc_vblank_get(crtc);
1989		if (ret) {
1990			drm_dbg_core(dev,
1991				     "crtc %d failed to acquire vblank counter, %d\n",
1992				     pipe, ret);
1993			return ret;
1994		}
1995	}
1996	drm_modeset_lock(&crtc->mutex, NULL);
1997	if (crtc->state)
1998		get_seq->active = crtc->state->enable;
1999	else
2000		get_seq->active = crtc->enabled;
2001	drm_modeset_unlock(&crtc->mutex);
2002	get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
2003	get_seq->sequence_ns = ktime_to_ns(now);
2004	if (!vblank_enabled)
2005		drm_crtc_vblank_put(crtc);
2006	return 0;
2007}
2008
2009/*
2010 * Queue a event for VBLANK sequence
2011 *
2012 * \param dev DRM device
2013 * \param data user argument, pointing to a drm_crtc_queue_sequence structure.
2014 * \param file_priv drm file private for the user's open file descriptor
2015 */
2016
2017int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2018				  struct drm_file *file_priv)
2019{
2020	struct drm_crtc *crtc;
2021	struct drm_vblank_crtc *vblank;
2022	int pipe;
2023	struct drm_crtc_queue_sequence *queue_seq = data;
2024	ktime_t now;
2025	struct drm_pending_vblank_event *e;
2026	u32 flags;
2027	u64 seq;
2028	u64 req_seq;
2029	int ret;
2030
2031	if (!drm_core_check_feature(dev, DRIVER_MODESET))
2032		return -EOPNOTSUPP;
2033
2034	if (!drm_dev_has_vblank(dev))
2035		return -EOPNOTSUPP;
2036
2037	crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
2038	if (!crtc)
2039		return -ENOENT;
2040
2041	flags = queue_seq->flags;
2042	/* Check valid flag bits */
2043	if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
2044		      DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
2045		return -EINVAL;
2046
2047	pipe = drm_crtc_index(crtc);
2048
2049	vblank = &dev->vblank[pipe];
2050
2051	e = kzalloc(sizeof(*e), GFP_KERNEL);
2052	if (e == NULL)
2053		return -ENOMEM;
2054
2055	ret = drm_crtc_vblank_get(crtc);
2056	if (ret) {
2057		drm_dbg_core(dev,
2058			     "crtc %d failed to acquire vblank counter, %d\n",
2059			     pipe, ret);
2060		goto err_free;
2061	}
2062
2063	seq = drm_vblank_count_and_time(dev, pipe, &now);
2064	req_seq = queue_seq->sequence;
2065
2066	if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
2067		req_seq += seq;
2068
2069	if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq))
2070		req_seq = seq + 1;
2071
2072	e->pipe = pipe;
2073	e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
2074	e->event.base.length = sizeof(e->event.seq);
2075	e->event.seq.user_data = queue_seq->user_data;
2076
2077	spin_lock_irq(&dev->event_lock);
2078
2079	/*
2080	 * drm_crtc_vblank_off() might have been called after we called
2081	 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2082	 * vblank disable, so no need for further locking.  The reference from
2083	 * drm_crtc_vblank_get() protects against vblank disable from another source.
2084	 */
2085	if (!READ_ONCE(vblank->enabled)) {
2086		ret = -EINVAL;
2087		goto err_unlock;
2088	}
2089
2090	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
2091					    &e->event.base);
2092
2093	if (ret)
2094		goto err_unlock;
2095
2096	e->sequence = req_seq;
2097
2098	if (drm_vblank_passed(seq, req_seq)) {
2099		drm_crtc_vblank_put(crtc);
2100		send_vblank_event(dev, e, seq, now);
2101		queue_seq->sequence = seq;
2102	} else {
2103		/* drm_handle_vblank_events will call drm_vblank_put */
2104		list_add_tail(&e->base.link, &dev->vblank_event_list);
2105		queue_seq->sequence = req_seq;
2106	}
2107
2108	spin_unlock_irq(&dev->event_lock);
2109	return 0;
2110
2111err_unlock:
2112	spin_unlock_irq(&dev->event_lock);
2113	drm_crtc_vblank_put(crtc);
2114err_free:
2115	kfree(e);
2116	return ret;
2117}
2118