1/*
   2 * drm_irq.c IRQ and vblank support
   3 *
   4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
   5 * \author Gareth Hughes <gareth@valinux.com>
   6 *
   7 * Permission is hereby granted, free of charge, to any person obtaining a
   8 * copy of this software and associated documentation files (the "Software"),
   9 * to deal in the Software without restriction, including without limitation
  10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11 * and/or sell copies of the Software, and to permit persons to whom the
  12 * Software is furnished to do so, subject to the following conditions:
  13 *
  14 * The above copyright notice and this permission notice (including the next
  15 * paragraph) shall be included in all copies or substantial portions of the
  16 * Software.
  17 *
  18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24 * OTHER DEALINGS IN THE SOFTWARE.
  25 */
  26
  27#include <linux/export.h>
  28#include <linux/moduleparam.h>
  29
  30#include <drm/drm_crtc.h>
  31#include <drm/drm_drv.h>
  32#include <drm/drm_framebuffer.h>
  33#include <drm/drm_print.h>
  34#include <drm/drm_vblank.h>
 
 
  35
  36#include "drm_internal.h"
  37#include "drm_trace.h"
 
  38
  39/**
  40 * DOC: vblank handling
  41 *
  42 * Vertical blanking plays a major role in graphics rendering. To achieve
  43 * tear-free display, users must synchronize page flips and/or rendering to
  44 * vertical blanking. The DRM API offers ioctls to perform page flips
  45 * synchronized to vertical blanking and wait for vertical blanking.
  46 *
  47 * The DRM core handles most of the vertical blanking management logic, which
  48 * involves filtering out spurious interrupts, keeping race-free blanking
  49 * counters, coping with counter wrap-around and resets and keeping use counts.
  50 * It relies on the driver to generate vertical blanking interrupts and
  51 * optionally provide a hardware vertical blanking counter.
  52 *
  53 * Drivers must initialize the vertical blanking handling core with a call to
  54 * drm_vblank_init(). Minimally, a driver needs to implement
  55 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
  56 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
  57 * support.
  58 *
  59 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
  60 * themselves (for instance to handle page flipping operations).  The DRM core
  61 * maintains a vertical blanking use count to ensure that the interrupts are not
  62 * disabled while a user still needs them. To increment the use count, drivers
  63 * call drm_crtc_vblank_get() and release the vblank reference again with
  64 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
  65 * guaranteed to be enabled.
  66 *
  67 * On many hardware disabling the vblank interrupt cannot be done in a race-free
  68 * manner, see &drm_driver.vblank_disable_immediate and
  69 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
  70 * vblanks after a timer has expired, which can be configured through the
  71 * ``vblankoffdelay`` module parameter.
  72 */
  73
  74/* Retry timestamp calculation up to 3 times to satisfy
  75 * drm_timestamp_precision before giving up.
  76 */
  77#define DRM_TIMESTAMP_MAXRETRIES 3
  78
  79/* Threshold in nanoseconds for detection of redundant
  80 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
  81 */
  82#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
  83
  84static bool
  85drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
  86			  ktime_t *tvblank, bool in_vblank_irq);
  87
  88static unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
  89
  90static int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
  91
  92module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
  93module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
  94MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
  95MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
  96
  97static void store_vblank(struct drm_device *dev, unsigned int pipe,
  98			 u32 vblank_count_inc,
  99			 ktime_t t_vblank, u32 last)
 100{
 101	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 102
 103	assert_spin_locked(&dev->vblank_time_lock);
 104
 105	vblank->last = last;
 106
 107	write_seqlock(&vblank->seqlock);
 108	vblank->time = t_vblank;
 109	vblank->count += vblank_count_inc;
 110	write_sequnlock(&vblank->seqlock);
 111}
 112
 113static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
 114{
 115	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 116
 117	return vblank->max_vblank_count ?: dev->max_vblank_count;
 118}
 119
 120/*
 121 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
 122 * if there is no useable hardware frame counter available.
 123 */
 124static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
 125{
 126	WARN_ON_ONCE(drm_max_vblank_count(dev, pipe) != 0);
 127	return 0;
 128}
 129
 130static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 131{
 132	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 133		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 134
 135		if (WARN_ON(!crtc))
 136			return 0;
 137
 138		if (crtc->funcs->get_vblank_counter)
 139			return crtc->funcs->get_vblank_counter(crtc);
 140	}
 141
 142	if (dev->driver->get_vblank_counter)
 143		return dev->driver->get_vblank_counter(dev, pipe);
 144
 145	return drm_vblank_no_hw_counter(dev, pipe);
 146}
 147
 148/*
 149 * Reset the stored timestamp for the current vblank count to correspond
 150 * to the last vblank occurred.
 151 *
 152 * Only to be called from drm_crtc_vblank_on().
 153 *
 154 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
 155 * device vblank fields.
 156 */
 157static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
 158{
 159	u32 cur_vblank;
 160	bool rc;
 161	ktime_t t_vblank;
 162	int count = DRM_TIMESTAMP_MAXRETRIES;
 163
 164	spin_lock(&dev->vblank_time_lock);
 165
 166	/*
 167	 * sample the current counter to avoid random jumps
 168	 * when drm_vblank_enable() applies the diff
 169	 */
 170	do {
 171		cur_vblank = __get_vblank_counter(dev, pipe);
 172		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
 173	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
 174
 175	/*
 176	 * Only reinitialize corresponding vblank timestamp if high-precision query
 177	 * available and didn't fail. Otherwise reinitialize delayed at next vblank
 178	 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
 179	 */
 180	if (!rc)
 181		t_vblank = 0;
 182
 183	/*
 184	 * +1 to make sure user will never see the same
 185	 * vblank counter value before and after a modeset
 186	 */
 187	store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
 188
 189	spin_unlock(&dev->vblank_time_lock);
 190}
 191
 192/*
 193 * Call back into the driver to update the appropriate vblank counter
 194 * (specified by @pipe).  Deal with wraparound, if it occurred, and
 195 * update the last read value so we can deal with wraparound on the next
 196 * call if necessary.
 197 *
 198 * Only necessary when going from off->on, to account for frames we
 199 * didn't get an interrupt for.
 200 *
 201 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
 202 * device vblank fields.
 203 */
 204static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 205				    bool in_vblank_irq)
 206{
 207	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 208	u32 cur_vblank, diff;
 209	bool rc;
 210	ktime_t t_vblank;
 211	int count = DRM_TIMESTAMP_MAXRETRIES;
 212	int framedur_ns = vblank->framedur_ns;
 213	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
 214
 215	/*
 216	 * Interrupts were disabled prior to this call, so deal with counter
 217	 * wrap if needed.
 218	 * NOTE!  It's possible we lost a full dev->max_vblank_count + 1 events
 219	 * here if the register is small or we had vblank interrupts off for
 220	 * a long time.
 221	 *
 222	 * We repeat the hardware vblank counter & timestamp query until
 223	 * we get consistent results. This to prevent races between gpu
 224	 * updating its hardware counter while we are retrieving the
 225	 * corresponding vblank timestamp.
 226	 */
 227	do {
 228		cur_vblank = __get_vblank_counter(dev, pipe);
 229		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
 230	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
 231
 232	if (max_vblank_count) {
 233		/* trust the hw counter when it's around */
 234		diff = (cur_vblank - vblank->last) & max_vblank_count;
 235	} else if (rc && framedur_ns) {
 236		u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
 237
 238		/*
 239		 * Figure out how many vblanks we've missed based
 240		 * on the difference in the timestamps and the
 241		 * frame/field duration.
 242		 */
 243
 244		DRM_DEBUG_VBL("crtc %u: Calculating number of vblanks."
 245			      " diff_ns = %lld, framedur_ns = %d)\n",
 246			      pipe, (long long) diff_ns, framedur_ns);
 247
 248		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
 249
 250		if (diff == 0 && in_vblank_irq)
 251			DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored\n",
 252				      pipe);
 
 253	} else {
 254		/* some kind of default for drivers w/o accurate vbl timestamping */
 255		diff = in_vblank_irq ? 1 : 0;
 256	}
 257
 258	/*
 259	 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
 260	 * interval? If so then vblank irqs keep running and it will likely
 261	 * happen that the hardware vblank counter is not trustworthy as it
 262	 * might reset at some point in that interval and vblank timestamps
 263	 * are not trustworthy either in that interval. Iow. this can result
 264	 * in a bogus diff >> 1 which must be avoided as it would cause
 265	 * random large forward jumps of the software vblank counter.
 266	 */
 267	if (diff > 1 && (vblank->inmodeset & 0x2)) {
 268		DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
 269			      " due to pre-modeset.\n", pipe, diff);
 270		diff = 1;
 271	}
 272
 273	DRM_DEBUG_VBL("updating vblank count on crtc %u:"
 274		      " current=%llu, diff=%u, hw=%u hw_last=%u\n",
 275		      pipe, vblank->count, diff, cur_vblank, vblank->last);
 276
 277	if (diff == 0) {
 278		WARN_ON_ONCE(cur_vblank != vblank->last);
 279		return;
 280	}
 281
 282	/*
 283	 * Only reinitialize corresponding vblank timestamp if high-precision query
 284	 * available and didn't fail, or we were called from the vblank interrupt.
 285	 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
 286	 * for now, to mark the vblanktimestamp as invalid.
 287	 */
 288	if (!rc && !in_vblank_irq)
 289		t_vblank = 0;
 290
 291	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
 292}
 293
 294static u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
 295{
 296	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 297
 298	if (WARN_ON(pipe >= dev->num_crtcs))
 299		return 0;
 300
 301	return vblank->count;
 302}
 303
 304/**
 305 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
 306 * @crtc: which counter to retrieve
 307 *
 308 * This function is similar to drm_crtc_vblank_count() but this function
 309 * interpolates to handle a race with vblank interrupts using the high precision
 310 * timestamping support.
 311 *
 312 * This is mostly useful for hardware that can obtain the scanout position, but
 313 * doesn't have a hardware frame counter.
 314 */
 315u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
 316{
 317	struct drm_device *dev = crtc->dev;
 318	unsigned int pipe = drm_crtc_index(crtc);
 319	u64 vblank;
 320	unsigned long flags;
 321
 322	WARN_ONCE(drm_debug & DRM_UT_VBL && !dev->driver->get_vblank_timestamp,
 323		  "This function requires support for accurate vblank timestamps.");
 324
 325	spin_lock_irqsave(&dev->vblank_time_lock, flags);
 326
 327	drm_update_vblank_count(dev, pipe, false);
 328	vblank = drm_vblank_count(dev, pipe);
 329
 330	spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
 331
 332	return vblank;
 333}
 334EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
 335
 336static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
 337{
 338	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 339		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 340
 341		if (WARN_ON(!crtc))
 342			return;
 343
 344		if (crtc->funcs->disable_vblank) {
 345			crtc->funcs->disable_vblank(crtc);
 346			return;
 347		}
 348	}
 349
 350	dev->driver->disable_vblank(dev, pipe);
 351}
 352
 353/*
 354 * Disable vblank irq's on crtc, make sure that last vblank count
 355 * of hardware and corresponding consistent software vblank counter
 356 * are preserved, even if there are any spurious vblank irq's after
 357 * disable.
 358 */
 359void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
 360{
 361	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 362	unsigned long irqflags;
 363
 364	assert_spin_locked(&dev->vbl_lock);
 365
 366	/* Prevent vblank irq processing while disabling vblank irqs,
 367	 * so no updates of timestamps or count can happen after we've
 368	 * disabled. Needed to prevent races in case of delayed irq's.
 369	 */
 370	spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 371
 372	/*
 373	 * Update vblank count and disable vblank interrupts only if the
 374	 * interrupts were enabled. This avoids calling the ->disable_vblank()
 375	 * operation in atomic context with the hardware potentially runtime
 376	 * suspended.
 377	 */
 378	if (!vblank->enabled)
 379		goto out;
 380
 381	/*
 382	 * Update the count and timestamp to maintain the
 383	 * appearance that the counter has been ticking all along until
 384	 * this time. This makes the count account for the entire time
 385	 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
 386	 */
 387	drm_update_vblank_count(dev, pipe, false);
 388	__disable_vblank(dev, pipe);
 389	vblank->enabled = false;
 390
 391out:
 392	spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
 393}
 394
 395static void vblank_disable_fn(struct timer_list *t)
 396{
 397	struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
 398	struct drm_device *dev = vblank->dev;
 399	unsigned int pipe = vblank->pipe;
 400	unsigned long irqflags;
 401
 402	spin_lock_irqsave(&dev->vbl_lock, irqflags);
 403	if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
 404		DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
 405		drm_vblank_disable_and_save(dev, pipe);
 406	}
 407	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 408}
 409
 410void drm_vblank_cleanup(struct drm_device *dev)
 411{
 412	unsigned int pipe;
 413
 414	/* Bail if the driver didn't call drm_vblank_init() */
 415	if (dev->num_crtcs == 0)
 416		return;
 417
 418	for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
 419		struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 420
 421		WARN_ON(READ_ONCE(vblank->enabled) &&
 422			drm_core_check_feature(dev, DRIVER_MODESET));
 423
 424		del_timer_sync(&vblank->disable_timer);
 425	}
 426
 427	kfree(dev->vblank);
 428
 429	dev->num_crtcs = 0;
 430}
 431
 432/**
 433 * drm_vblank_init - initialize vblank support
 434 * @dev: DRM device
 435 * @num_crtcs: number of CRTCs supported by @dev
 436 *
 437 * This function initializes vblank support for @num_crtcs display pipelines.
 438 * Cleanup is handled by the DRM core, or through calling drm_dev_fini() for
 439 * drivers with a &drm_driver.release callback.
 440 *
 441 * Returns:
 442 * Zero on success or a negative error code on failure.
 443 */
 444int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
 445{
 446	int ret = -ENOMEM;
 447	unsigned int i;
 448
 449	spin_lock_init(&dev->vbl_lock);
 450	spin_lock_init(&dev->vblank_time_lock);
 451
 452	dev->num_crtcs = num_crtcs;
 453
 454	dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
 455	if (!dev->vblank)
 456		goto err;
 457
 458	for (i = 0; i < num_crtcs; i++) {
 459		struct drm_vblank_crtc *vblank = &dev->vblank[i];
 460
 461		vblank->dev = dev;
 462		vblank->pipe = i;
 463		init_waitqueue_head(&vblank->queue);
 464		timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
 465		seqlock_init(&vblank->seqlock);
 466	}
 467
 468	DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
 469
 470	/* Driver specific high-precision vblank timestamping supported? */
 471	if (dev->driver->get_vblank_timestamp)
 472		DRM_INFO("Driver supports precise vblank timestamp query.\n");
 473	else
 474		DRM_INFO("No driver support for vblank timestamp query.\n");
 475
 476	/* Must have precise timestamping for reliable vblank instant disable */
 477	if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
 478		dev->vblank_disable_immediate = false;
 479		DRM_INFO("Setting vblank_disable_immediate to false because "
 480			 "get_vblank_timestamp == NULL\n");
 481	}
 482
 483	return 0;
 484
 485err:
 486	dev->num_crtcs = 0;
 487	return ret;
 488}
 489EXPORT_SYMBOL(drm_vblank_init);
 490
 491/**
 492 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
 493 * @crtc: which CRTC's vblank waitqueue to retrieve
 494 *
 495 * This function returns a pointer to the vblank waitqueue for the CRTC.
 496 * Drivers can use this to implement vblank waits using wait_event() and related
 497 * functions.
 498 */
 499wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
 500{
 501	return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
 502}
 503EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
 504
 505
 506/**
 507 * drm_calc_timestamping_constants - calculate vblank timestamp constants
 508 * @crtc: drm_crtc whose timestamp constants should be updated.
 509 * @mode: display mode containing the scanout timings
 510 *
 511 * Calculate and store various constants which are later needed by vblank and
 512 * swap-completion timestamping, e.g, by
 513 * drm_calc_vbltimestamp_from_scanoutpos(). They are derived from CRTC's true
 514 * scanout timing, so they take things like panel scaling or other adjustments
 515 * into account.
 516 */
 517void drm_calc_timestamping_constants(struct drm_crtc *crtc,
 518				     const struct drm_display_mode *mode)
 519{
 520	struct drm_device *dev = crtc->dev;
 521	unsigned int pipe = drm_crtc_index(crtc);
 522	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 523	int linedur_ns = 0, framedur_ns = 0;
 524	int dotclock = mode->crtc_clock;
 525
 526	if (!dev->num_crtcs)
 527		return;
 528
 529	if (WARN_ON(pipe >= dev->num_crtcs))
 530		return;
 531
 532	/* Valid dotclock? */
 533	if (dotclock > 0) {
 534		int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
 535
 536		/*
 537		 * Convert scanline length in pixels and video
 538		 * dot clock to line duration and frame duration
 539		 * in nanoseconds:
 540		 */
 541		linedur_ns  = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
 542		framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
 543
 544		/*
 545		 * Fields of interlaced scanout modes are only half a frame duration.
 546		 */
 547		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 548			framedur_ns /= 2;
 549	} else
 550		DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
 551			  crtc->base.id);
 552
 553	vblank->linedur_ns  = linedur_ns;
 554	vblank->framedur_ns = framedur_ns;
 555	vblank->hwmode = *mode;
 556
 557	DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
 558		  crtc->base.id, mode->crtc_htotal,
 559		  mode->crtc_vtotal, mode->crtc_vdisplay);
 560	DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
 561		  crtc->base.id, dotclock, framedur_ns, linedur_ns);
 562}
 563EXPORT_SYMBOL(drm_calc_timestamping_constants);
 564
 565/**
 566 * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper
 567 * @dev: DRM device
 568 * @pipe: index of CRTC whose vblank timestamp to retrieve
 569 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
 570 *             On return contains true maximum error of timestamp
 571 * @vblank_time: Pointer to time which should receive the timestamp
 572 * @in_vblank_irq:
 573 *     True when called from drm_crtc_handle_vblank().  Some drivers
 574 *     need to apply some workarounds for gpu-specific vblank irq quirks
 575 *     if flag is set.
 576 *
 577 * Implements calculation of exact vblank timestamps from given drm_display_mode
 578 * timings and current video scanout position of a CRTC. This can be directly
 579 * used as the &drm_driver.get_vblank_timestamp implementation of a kms driver
 580 * if &drm_driver.get_scanout_position is implemented.
 581 *
 582 * The current implementation only handles standard video modes. For double scan
 583 * and interlaced modes the driver is supposed to adjust the hardware mode
 584 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
 585 * match the scanout position reported.
 586 *
 587 * Note that atomic drivers must call drm_calc_timestamping_constants() before
 588 * enabling a CRTC. The atomic helpers already take care of that in
 589 * drm_atomic_helper_update_legacy_modeset_state().
 590 *
 591 * Returns:
 592 *
 593 * Returns true on success, and false on failure, i.e. when no accurate
 594 * timestamp could be acquired.
 595 */
 596bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
 597					   unsigned int pipe,
 598					   int *max_error,
 599					   ktime_t *vblank_time,
 600					   bool in_vblank_irq)
 601{
 602	struct timespec64 ts_etime, ts_vblank_time;
 603	ktime_t stime, etime;
 604	bool vbl_status;
 605	struct drm_crtc *crtc;
 606	const struct drm_display_mode *mode;
 607	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 608	int vpos, hpos, i;
 609	int delta_ns, duration_ns;
 610
 611	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 612		return false;
 613
 614	crtc = drm_crtc_from_index(dev, pipe);
 615
 616	if (pipe >= dev->num_crtcs || !crtc) {
 617		DRM_ERROR("Invalid crtc %u\n", pipe);
 618		return false;
 619	}
 620
 621	/* Scanout position query not supported? Should not happen. */
 622	if (!dev->driver->get_scanout_position) {
 623		DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
 624		return false;
 625	}
 626
 627	if (drm_drv_uses_atomic_modeset(dev))
 628		mode = &vblank->hwmode;
 629	else
 630		mode = &crtc->hwmode;
 631
 632	/* If mode timing undefined, just return as no-op:
 633	 * Happens during initial modesetting of a crtc.
 634	 */
 635	if (mode->crtc_clock == 0) {
 636		DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
 637		WARN_ON_ONCE(drm_drv_uses_atomic_modeset(dev));
 638
 639		return false;
 640	}
 641
 642	/* Get current scanout position with system timestamp.
 643	 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
 644	 * if single query takes longer than max_error nanoseconds.
 645	 *
 646	 * This guarantees a tight bound on maximum error if
 647	 * code gets preempted or delayed for some reason.
 648	 */
 649	for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
 650		/*
 651		 * Get vertical and horizontal scanout position vpos, hpos,
 652		 * and bounding timestamps stime, etime, pre/post query.
 653		 */
 654		vbl_status = dev->driver->get_scanout_position(dev, pipe,
 655							       in_vblank_irq,
 656							       &vpos, &hpos,
 657							       &stime, &etime,
 658							       mode);
 659
 660		/* Return as no-op if scanout query unsupported or failed. */
 661		if (!vbl_status) {
 662			DRM_DEBUG("crtc %u : scanoutpos query failed.\n",
 663				  pipe);
 664			return false;
 665		}
 666
 667		/* Compute uncertainty in timestamp of scanout position query. */
 668		duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
 669
 670		/* Accept result with <  max_error nsecs timing uncertainty. */
 671		if (duration_ns <= *max_error)
 672			break;
 673	}
 674
 675	/* Noisy system timing? */
 676	if (i == DRM_TIMESTAMP_MAXRETRIES) {
 677		DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
 678			  pipe, duration_ns/1000, *max_error/1000, i);
 679	}
 680
 681	/* Return upper bound of timestamp precision error. */
 682	*max_error = duration_ns;
 683
 684	/* Convert scanout position into elapsed time at raw_time query
 685	 * since start of scanout at first display scanline. delta_ns
 686	 * can be negative if start of scanout hasn't happened yet.
 687	 */
 688	delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
 689			   mode->crtc_clock);
 690
 691	/* Subtract time delta from raw timestamp to get final
 692	 * vblank_time timestamp for end of vblank.
 693	 */
 694	*vblank_time = ktime_sub_ns(etime, delta_ns);
 695
 696	if ((drm_debug & DRM_UT_VBL) == 0)
 697		return true;
 698
 699	ts_etime = ktime_to_timespec64(etime);
 700	ts_vblank_time = ktime_to_timespec64(*vblank_time);
 701
 702	DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
 703		      pipe, hpos, vpos,
 704		      (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
 705		      (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
 706		      duration_ns / 1000, i);
 707
 708	return true;
 709}
 710EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
 711
 712/**
 713 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
 714 *                             vblank interval
 715 * @dev: DRM device
 716 * @pipe: index of CRTC whose vblank timestamp to retrieve
 717 * @tvblank: Pointer to target time which should receive the timestamp
 718 * @in_vblank_irq:
 719 *     True when called from drm_crtc_handle_vblank().  Some drivers
 720 *     need to apply some workarounds for gpu-specific vblank irq quirks
 721 *     if flag is set.
 722 *
 723 * Fetches the system timestamp corresponding to the time of the most recent
 724 * vblank interval on specified CRTC. May call into kms-driver to
 725 * compute the timestamp with a high-precision GPU specific method.
 726 *
 727 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
 728 * call, i.e., it isn't very precisely locked to the true vblank.
 729 *
 730 * Returns:
 731 * True if timestamp is considered to be very precise, false otherwise.
 732 */
 733static bool
 734drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 735			  ktime_t *tvblank, bool in_vblank_irq)
 736{
 737	bool ret = false;
 738
 739	/* Define requested maximum error on timestamps (nanoseconds). */
 740	int max_error = (int) drm_timestamp_precision * 1000;
 741
 742	/* Query driver if possible and precision timestamping enabled. */
 743	if (dev->driver->get_vblank_timestamp && (max_error > 0))
 744		ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
 745							tvblank, in_vblank_irq);
 746
 747	/* GPU high precision timestamp query unsupported or failed.
 748	 * Return current monotonic/gettimeofday timestamp as best estimate.
 749	 */
 750	if (!ret)
 751		*tvblank = ktime_get();
 752
 753	return ret;
 754}
 755
 756/**
 757 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
 758 * @crtc: which counter to retrieve
 759 *
 760 * Fetches the "cooked" vblank count value that represents the number of
 761 * vblank events since the system was booted, including lost events due to
 762 * modesetting activity. Note that this timer isn't correct against a racing
 763 * vblank interrupt (since it only reports the software vblank counter), see
 764 * drm_crtc_accurate_vblank_count() for such use-cases.
 765 *
 766 * Returns:
 767 * The software vblank counter.
 768 */
 769u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
 770{
 771	return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
 772}
 773EXPORT_SYMBOL(drm_crtc_vblank_count);
 774
 775/**
 776 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
 777 *     system timestamp corresponding to that vblank counter value.
 778 * @dev: DRM device
 779 * @pipe: index of CRTC whose counter to retrieve
 780 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
 781 *
 782 * Fetches the "cooked" vblank count value that represents the number of
 783 * vblank events since the system was booted, including lost events due to
 784 * modesetting activity. Returns corresponding system timestamp of the time
 785 * of the vblank interval that corresponds to the current vblank counter value.
 786 *
 787 * This is the legacy version of drm_crtc_vblank_count_and_time().
 788 */
 789static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
 790				     ktime_t *vblanktime)
 791{
 792	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 793	u64 vblank_count;
 794	unsigned int seq;
 795
 796	if (WARN_ON(pipe >= dev->num_crtcs)) {
 797		*vblanktime = 0;
 798		return 0;
 799	}
 800
 801	do {
 802		seq = read_seqbegin(&vblank->seqlock);
 803		vblank_count = vblank->count;
 804		*vblanktime = vblank->time;
 805	} while (read_seqretry(&vblank->seqlock, seq));
 806
 807	return vblank_count;
 808}
 809
 810/**
 811 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
 812 *     and the system timestamp corresponding to that vblank counter value
 813 * @crtc: which counter to retrieve
 814 * @vblanktime: Pointer to time to receive the vblank timestamp.
 815 *
 816 * Fetches the "cooked" vblank count value that represents the number of
 817 * vblank events since the system was booted, including lost events due to
 818 * modesetting activity. Returns corresponding system timestamp of the time
 819 * of the vblank interval that corresponds to the current vblank counter value.
 820 */
 821u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
 822				   ktime_t *vblanktime)
 823{
 824	return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
 825					 vblanktime);
 826}
 827EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
 828
 829static void send_vblank_event(struct drm_device *dev,
 830		struct drm_pending_vblank_event *e,
 831		u64 seq, ktime_t now)
 832{
 833	struct timespec64 tv;
 834
 835	switch (e->event.base.type) {
 836	case DRM_EVENT_VBLANK:
 837	case DRM_EVENT_FLIP_COMPLETE:
 838		tv = ktime_to_timespec64(now);
 839		e->event.vbl.sequence = seq;
 840		/*
 841		 * e->event is a user space structure, with hardcoded unsigned
 842		 * 32-bit seconds/microseconds. This is safe as we always use
 843		 * monotonic timestamps since linux-4.15
 844		 */
 845		e->event.vbl.tv_sec = tv.tv_sec;
 846		e->event.vbl.tv_usec = tv.tv_nsec / 1000;
 847		break;
 848	case DRM_EVENT_CRTC_SEQUENCE:
 849		if (seq)
 850			e->event.seq.sequence = seq;
 851		e->event.seq.time_ns = ktime_to_ns(now);
 852		break;
 853	}
 854	trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
 855	drm_send_event_locked(dev, &e->base);
 856}
 857
 858/**
 859 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
 860 * @crtc: the source CRTC of the vblank event
 861 * @e: the event to send
 862 *
 863 * A lot of drivers need to generate vblank events for the very next vblank
 864 * interrupt. For example when the page flip interrupt happens when the page
 865 * flip gets armed, but not when it actually executes within the next vblank
 866 * period. This helper function implements exactly the required vblank arming
 867 * behaviour.
 868 *
 869 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
 870 * atomic commit must ensure that the next vblank happens at exactly the same
 871 * time as the atomic commit is committed to the hardware. This function itself
 872 * does **not** protect against the next vblank interrupt racing with either this
 873 * function call or the atomic commit operation. A possible sequence could be:
 874 *
 875 * 1. Driver commits new hardware state into vblank-synchronized registers.
 876 * 2. A vblank happens, committing the hardware state. Also the corresponding
 877 *    vblank interrupt is fired off and fully processed by the interrupt
 878 *    handler.
 879 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
 880 * 4. The event is only send out for the next vblank, which is wrong.
 881 *
 882 * An equivalent race can happen when the driver calls
 883 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
 884 *
 885 * The only way to make this work safely is to prevent the vblank from firing
 886 * (and the hardware from committing anything else) until the entire atomic
 887 * commit sequence has run to completion. If the hardware does not have such a
 888 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
 889 * Instead drivers need to manually send out the event from their interrupt
 890 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
 891 * possible race with the hardware committing the atomic update.
 892 *
 893 * Caller must hold a vblank reference for the event @e acquired by a
 894 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
 895 */
 896void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
 897			       struct drm_pending_vblank_event *e)
 898{
 899	struct drm_device *dev = crtc->dev;
 900	unsigned int pipe = drm_crtc_index(crtc);
 901
 902	assert_spin_locked(&dev->event_lock);
 903
 904	e->pipe = pipe;
 905	e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
 906	list_add_tail(&e->base.link, &dev->vblank_event_list);
 907}
 908EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
 909
 910/**
 911 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
 912 * @crtc: the source CRTC of the vblank event
 913 * @e: the event to send
 914 *
 915 * Updates sequence # and timestamp on event for the most recently processed
 916 * vblank, and sends it to userspace.  Caller must hold event lock.
 917 *
 918 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
 919 * situation, especially to send out events for atomic commit operations.
 920 */
 921void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
 922				struct drm_pending_vblank_event *e)
 923{
 924	struct drm_device *dev = crtc->dev;
 925	u64 seq;
 926	unsigned int pipe = drm_crtc_index(crtc);
 927	ktime_t now;
 928
 929	if (dev->num_crtcs > 0) {
 930		seq = drm_vblank_count_and_time(dev, pipe, &now);
 931	} else {
 932		seq = 0;
 933
 934		now = ktime_get();
 935	}
 936	e->pipe = pipe;
 937	send_vblank_event(dev, e, seq, now);
 938}
 939EXPORT_SYMBOL(drm_crtc_send_vblank_event);
 940
 941static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
 942{
 943	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 944		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 945
 946		if (WARN_ON(!crtc))
 947			return 0;
 948
 949		if (crtc->funcs->enable_vblank)
 950			return crtc->funcs->enable_vblank(crtc);
 951	}
 952
 953	return dev->driver->enable_vblank(dev, pipe);
 954}
 955
 956static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
 957{
 958	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 959	int ret = 0;
 960
 961	assert_spin_locked(&dev->vbl_lock);
 962
 963	spin_lock(&dev->vblank_time_lock);
 964
 965	if (!vblank->enabled) {
 966		/*
 967		 * Enable vblank irqs under vblank_time_lock protection.
 968		 * All vblank count & timestamp updates are held off
 969		 * until we are done reinitializing master counter and
 970		 * timestamps. Filtercode in drm_handle_vblank() will
 971		 * prevent double-accounting of same vblank interval.
 972		 */
 973		ret = __enable_vblank(dev, pipe);
 974		DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
 975		if (ret) {
 976			atomic_dec(&vblank->refcount);
 977		} else {
 978			drm_update_vblank_count(dev, pipe, 0);
 979			/* drm_update_vblank_count() includes a wmb so we just
 980			 * need to ensure that the compiler emits the write
 981			 * to mark the vblank as enabled after the call
 982			 * to drm_update_vblank_count().
 983			 */
 984			WRITE_ONCE(vblank->enabled, true);
 985		}
 986	}
 987
 988	spin_unlock(&dev->vblank_time_lock);
 989
 990	return ret;
 991}
 992
 993static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
 994{
 995	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 996	unsigned long irqflags;
 997	int ret = 0;
 998
 999	if (!dev->num_crtcs)
1000		return -EINVAL;
1001
1002	if (WARN_ON(pipe >= dev->num_crtcs))
1003		return -EINVAL;
1004
1005	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1006	/* Going from 0->1 means we have to enable interrupts again */
1007	if (atomic_add_return(1, &vblank->refcount) == 1) {
1008		ret = drm_vblank_enable(dev, pipe);
1009	} else {
1010		if (!vblank->enabled) {
1011			atomic_dec(&vblank->refcount);
1012			ret = -EINVAL;
1013		}
1014	}
1015	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1016
1017	return ret;
1018}
1019
1020/**
1021 * drm_crtc_vblank_get - get a reference count on vblank events
1022 * @crtc: which CRTC to own
1023 *
1024 * Acquire a reference count on vblank events to avoid having them disabled
1025 * while in use.
1026 *
1027 * Returns:
1028 * Zero on success or a negative error code on failure.
1029 */
1030int drm_crtc_vblank_get(struct drm_crtc *crtc)
1031{
1032	return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1033}
1034EXPORT_SYMBOL(drm_crtc_vblank_get);
1035
1036static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1037{
1038	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1039
1040	if (WARN_ON(pipe >= dev->num_crtcs))
1041		return;
1042
1043	if (WARN_ON(atomic_read(&vblank->refcount) == 0))
1044		return;
1045
1046	/* Last user schedules interrupt disable */
1047	if (atomic_dec_and_test(&vblank->refcount)) {
1048		if (drm_vblank_offdelay == 0)
1049			return;
1050		else if (drm_vblank_offdelay < 0)
1051			vblank_disable_fn(&vblank->disable_timer);
1052		else if (!dev->vblank_disable_immediate)
1053			mod_timer(&vblank->disable_timer,
1054				  jiffies + ((drm_vblank_offdelay * HZ)/1000));
1055	}
1056}
1057
1058/**
1059 * drm_crtc_vblank_put - give up ownership of vblank events
1060 * @crtc: which counter to give up
1061 *
1062 * Release ownership of a given vblank counter, turning off interrupts
1063 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1064 */
1065void drm_crtc_vblank_put(struct drm_crtc *crtc)
1066{
1067	drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1068}
1069EXPORT_SYMBOL(drm_crtc_vblank_put);
1070
1071/**
1072 * drm_wait_one_vblank - wait for one vblank
1073 * @dev: DRM device
1074 * @pipe: CRTC index
1075 *
1076 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1077 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1078 * due to lack of driver support or because the crtc is off.
1079 *
1080 * This is the legacy version of drm_crtc_wait_one_vblank().
1081 */
1082void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1083{
1084	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1085	int ret;
1086	u64 last;
1087
1088	if (WARN_ON(pipe >= dev->num_crtcs))
1089		return;
1090
1091	ret = drm_vblank_get(dev, pipe);
1092	if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
1093		return;
1094
1095	last = drm_vblank_count(dev, pipe);
1096
1097	ret = wait_event_timeout(vblank->queue,
1098				 last != drm_vblank_count(dev, pipe),
1099				 msecs_to_jiffies(100));
1100
1101	WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1102
1103	drm_vblank_put(dev, pipe);
1104}
1105EXPORT_SYMBOL(drm_wait_one_vblank);
1106
1107/**
1108 * drm_crtc_wait_one_vblank - wait for one vblank
1109 * @crtc: DRM crtc
1110 *
1111 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1112 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1113 * due to lack of driver support or because the crtc is off.
1114 */
1115void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1116{
1117	drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1118}
1119EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1120
1121/**
1122 * drm_crtc_vblank_off - disable vblank events on a CRTC
1123 * @crtc: CRTC in question
1124 *
1125 * Drivers can use this function to shut down the vblank interrupt handling when
1126 * disabling a crtc. This function ensures that the latest vblank frame count is
1127 * stored so that drm_vblank_on can restore it again.
1128 *
1129 * Drivers must use this function when the hardware vblank counter can get
1130 * reset, e.g. when suspending or disabling the @crtc in general.
1131 */
1132void drm_crtc_vblank_off(struct drm_crtc *crtc)
1133{
1134	struct drm_device *dev = crtc->dev;
1135	unsigned int pipe = drm_crtc_index(crtc);
1136	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1137	struct drm_pending_vblank_event *e, *t;
1138
1139	ktime_t now;
1140	unsigned long irqflags;
1141	u64 seq;
1142
1143	if (WARN_ON(pipe >= dev->num_crtcs))
1144		return;
1145
1146	spin_lock_irqsave(&dev->event_lock, irqflags);
1147
1148	spin_lock(&dev->vbl_lock);
1149	DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1150		      pipe, vblank->enabled, vblank->inmodeset);
1151
1152	/* Avoid redundant vblank disables without previous
1153	 * drm_crtc_vblank_on(). */
1154	if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1155		drm_vblank_disable_and_save(dev, pipe);
1156
1157	wake_up(&vblank->queue);
1158
1159	/*
1160	 * Prevent subsequent drm_vblank_get() from re-enabling
1161	 * the vblank interrupt by bumping the refcount.
1162	 */
1163	if (!vblank->inmodeset) {
1164		atomic_inc(&vblank->refcount);
1165		vblank->inmodeset = 1;
1166	}
1167	spin_unlock(&dev->vbl_lock);
1168
1169	/* Send any queued vblank events, lest the natives grow disquiet */
1170	seq = drm_vblank_count_and_time(dev, pipe, &now);
1171
1172	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1173		if (e->pipe != pipe)
1174			continue;
1175		DRM_DEBUG("Sending premature vblank event on disable: "
1176			  "wanted %llu, current %llu\n",
1177			  e->sequence, seq);
1178		list_del(&e->base.link);
1179		drm_vblank_put(dev, pipe);
1180		send_vblank_event(dev, e, seq, now);
1181	}
1182	spin_unlock_irqrestore(&dev->event_lock, irqflags);
1183
1184	/* Will be reset by the modeset helpers when re-enabling the crtc by
1185	 * calling drm_calc_timestamping_constants(). */
1186	vblank->hwmode.crtc_clock = 0;
1187}
1188EXPORT_SYMBOL(drm_crtc_vblank_off);
1189
1190/**
1191 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1192 * @crtc: CRTC in question
1193 *
1194 * Drivers can use this function to reset the vblank state to off at load time.
1195 * Drivers should use this together with the drm_crtc_vblank_off() and
1196 * drm_crtc_vblank_on() functions. The difference compared to
1197 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1198 * and hence doesn't need to call any driver hooks.
1199 *
1200 * This is useful for recovering driver state e.g. on driver load, or on resume.
1201 */
1202void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1203{
1204	struct drm_device *dev = crtc->dev;
1205	unsigned long irqflags;
1206	unsigned int pipe = drm_crtc_index(crtc);
1207	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1208
1209	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1210	/*
1211	 * Prevent subsequent drm_vblank_get() from enabling the vblank
1212	 * interrupt by bumping the refcount.
1213	 */
1214	if (!vblank->inmodeset) {
1215		atomic_inc(&vblank->refcount);
1216		vblank->inmodeset = 1;
1217	}
1218	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1219
1220	WARN_ON(!list_empty(&dev->vblank_event_list));
1221}
1222EXPORT_SYMBOL(drm_crtc_vblank_reset);
1223
1224/**
1225 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1226 * @crtc: CRTC in question
1227 * @max_vblank_count: max hardware vblank counter value
1228 *
1229 * Update the maximum hardware vblank counter value for @crtc
1230 * at runtime. Useful for hardware where the operation of the
1231 * hardware vblank counter depends on the currently active
1232 * display configuration.
1233 *
1234 * For example, if the hardware vblank counter does not work
1235 * when a specific connector is active the maximum can be set
1236 * to zero. And when that specific connector isn't active the
1237 * maximum can again be set to the appropriate non-zero value.
1238 *
1239 * If used, must be called before drm_vblank_on().
1240 */
1241void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1242				   u32 max_vblank_count)
1243{
1244	struct drm_device *dev = crtc->dev;
1245	unsigned int pipe = drm_crtc_index(crtc);
1246	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1247
1248	WARN_ON(dev->max_vblank_count);
1249	WARN_ON(!READ_ONCE(vblank->inmodeset));
1250
1251	vblank->max_vblank_count = max_vblank_count;
1252}
1253EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1254
1255/**
1256 * drm_crtc_vblank_on - enable vblank events on a CRTC
1257 * @crtc: CRTC in question
1258 *
1259 * This functions restores the vblank interrupt state captured with
1260 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1261 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1262 * unbalanced and so can also be unconditionally called in driver load code to
1263 * reflect the current hardware state of the crtc.
1264 */
1265void drm_crtc_vblank_on(struct drm_crtc *crtc)
1266{
1267	struct drm_device *dev = crtc->dev;
1268	unsigned int pipe = drm_crtc_index(crtc);
1269	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1270	unsigned long irqflags;
1271
1272	if (WARN_ON(pipe >= dev->num_crtcs))
1273		return;
1274
1275	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1276	DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1277		      pipe, vblank->enabled, vblank->inmodeset);
1278
1279	/* Drop our private "prevent drm_vblank_get" refcount */
1280	if (vblank->inmodeset) {
1281		atomic_dec(&vblank->refcount);
1282		vblank->inmodeset = 0;
1283	}
1284
1285	drm_reset_vblank_timestamp(dev, pipe);
1286
1287	/*
1288	 * re-enable interrupts if there are users left, or the
1289	 * user wishes vblank interrupts to be enabled all the time.
1290	 */
1291	if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1292		WARN_ON(drm_vblank_enable(dev, pipe));
1293	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1294}
1295EXPORT_SYMBOL(drm_crtc_vblank_on);
1296
1297/**
1298 * drm_vblank_restore - estimate missed vblanks and update vblank count.
1299 * @dev: DRM device
1300 * @pipe: CRTC index
1301 *
1302 * Power manamement features can cause frame counter resets between vblank
1303 * disable and enable. Drivers can use this function in their
1304 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1305 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1306 * vblank counter.
1307 *
1308 * This function is the legacy version of drm_crtc_vblank_restore().
1309 */
1310void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1311{
1312	ktime_t t_vblank;
1313	struct drm_vblank_crtc *vblank;
1314	int framedur_ns;
1315	u64 diff_ns;
1316	u32 cur_vblank, diff = 1;
1317	int count = DRM_TIMESTAMP_MAXRETRIES;
1318
1319	if (WARN_ON(pipe >= dev->num_crtcs))
1320		return;
1321
1322	assert_spin_locked(&dev->vbl_lock);
1323	assert_spin_locked(&dev->vblank_time_lock);
1324
1325	vblank = &dev->vblank[pipe];
1326	WARN_ONCE((drm_debug & DRM_UT_VBL) && !vblank->framedur_ns,
1327		  "Cannot compute missed vblanks without frame duration\n");
1328	framedur_ns = vblank->framedur_ns;
1329
1330	do {
1331		cur_vblank = __get_vblank_counter(dev, pipe);
1332		drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1333	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1334
1335	diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1336	if (framedur_ns)
1337		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1338
1339
1340	DRM_DEBUG_VBL("missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1341		      diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1342	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
1343}
1344EXPORT_SYMBOL(drm_vblank_restore);
1345
1346/**
1347 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1348 * @crtc: CRTC in question
1349 *
1350 * Power manamement features can cause frame counter resets between vblank
1351 * disable and enable. Drivers can use this function in their
1352 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1353 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1354 * vblank counter.
1355 */
1356void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1357{
1358	drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1359}
1360EXPORT_SYMBOL(drm_crtc_vblank_restore);
1361
1362static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1363					  unsigned int pipe)
1364{
1365	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1366
1367	/* vblank is not initialized (IRQ not installed ?), or has been freed */
1368	if (!dev->num_crtcs)
1369		return;
1370
1371	if (WARN_ON(pipe >= dev->num_crtcs))
1372		return;
1373
1374	/*
1375	 * To avoid all the problems that might happen if interrupts
1376	 * were enabled/disabled around or between these calls, we just
1377	 * have the kernel take a reference on the CRTC (just once though
1378	 * to avoid corrupting the count if multiple, mismatch calls occur),
1379	 * so that interrupts remain enabled in the interim.
1380	 */
1381	if (!vblank->inmodeset) {
1382		vblank->inmodeset = 0x1;
1383		if (drm_vblank_get(dev, pipe) == 0)
1384			vblank->inmodeset |= 0x2;
1385	}
1386}
1387
1388static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1389					   unsigned int pipe)
1390{
1391	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1392	unsigned long irqflags;
1393
1394	/* vblank is not initialized (IRQ not installed ?), or has been freed */
1395	if (!dev->num_crtcs)
1396		return;
1397
1398	if (WARN_ON(pipe >= dev->num_crtcs))
1399		return;
1400
1401	if (vblank->inmodeset) {
1402		spin_lock_irqsave(&dev->vbl_lock, irqflags);
1403		drm_reset_vblank_timestamp(dev, pipe);
1404		spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1405
1406		if (vblank->inmodeset & 0x2)
1407			drm_vblank_put(dev, pipe);
1408
1409		vblank->inmodeset = 0;
1410	}
1411}
1412
1413int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1414				 struct drm_file *file_priv)
1415{
1416	struct drm_modeset_ctl *modeset = data;
1417	unsigned int pipe;
1418
1419	/* If drm_vblank_init() hasn't been called yet, just no-op */
1420	if (!dev->num_crtcs)
1421		return 0;
1422
1423	/* KMS drivers handle this internally */
1424	if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1425		return 0;
1426
1427	pipe = modeset->crtc;
1428	if (pipe >= dev->num_crtcs)
1429		return -EINVAL;
1430
1431	switch (modeset->cmd) {
1432	case _DRM_PRE_MODESET:
1433		drm_legacy_vblank_pre_modeset(dev, pipe);
1434		break;
1435	case _DRM_POST_MODESET:
1436		drm_legacy_vblank_post_modeset(dev, pipe);
1437		break;
1438	default:
1439		return -EINVAL;
1440	}
1441
1442	return 0;
1443}
1444
1445static inline bool vblank_passed(u64 seq, u64 ref)
1446{
1447	return (seq - ref) <= (1 << 23);
1448}
1449
1450static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1451				  u64 req_seq,
1452				  union drm_wait_vblank *vblwait,
1453				  struct drm_file *file_priv)
1454{
1455	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1456	struct drm_pending_vblank_event *e;
1457	ktime_t now;
1458	unsigned long flags;
1459	u64 seq;
1460	int ret;
1461
1462	e = kzalloc(sizeof(*e), GFP_KERNEL);
1463	if (e == NULL) {
1464		ret = -ENOMEM;
1465		goto err_put;
1466	}
1467
1468	e->pipe = pipe;
1469	e->event.base.type = DRM_EVENT_VBLANK;
1470	e->event.base.length = sizeof(e->event.vbl);
1471	e->event.vbl.user_data = vblwait->request.signal;
1472	e->event.vbl.crtc_id = 0;
1473	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1474		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1475		if (crtc)
1476			e->event.vbl.crtc_id = crtc->base.id;
1477	}
1478
1479	spin_lock_irqsave(&dev->event_lock, flags);
1480
1481	/*
1482	 * drm_crtc_vblank_off() might have been called after we called
1483	 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1484	 * vblank disable, so no need for further locking.  The reference from
1485	 * drm_vblank_get() protects against vblank disable from another source.
1486	 */
1487	if (!READ_ONCE(vblank->enabled)) {
1488		ret = -EINVAL;
1489		goto err_unlock;
1490	}
1491
1492	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1493					    &e->event.base);
1494
1495	if (ret)
1496		goto err_unlock;
1497
1498	seq = drm_vblank_count_and_time(dev, pipe, &now);
1499
1500	DRM_DEBUG("event on vblank count %llu, current %llu, crtc %u\n",
1501		  req_seq, seq, pipe);
1502
1503	trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1504
1505	e->sequence = req_seq;
1506	if (vblank_passed(seq, req_seq)) {
1507		drm_vblank_put(dev, pipe);
1508		send_vblank_event(dev, e, seq, now);
1509		vblwait->reply.sequence = seq;
1510	} else {
1511		/* drm_handle_vblank_events will call drm_vblank_put */
1512		list_add_tail(&e->base.link, &dev->vblank_event_list);
1513		vblwait->reply.sequence = req_seq;
1514	}
1515
1516	spin_unlock_irqrestore(&dev->event_lock, flags);
1517
1518	return 0;
1519
1520err_unlock:
1521	spin_unlock_irqrestore(&dev->event_lock, flags);
1522	kfree(e);
1523err_put:
1524	drm_vblank_put(dev, pipe);
1525	return ret;
1526}
1527
1528static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1529{
1530	if (vblwait->request.sequence)
1531		return false;
1532
1533	return _DRM_VBLANK_RELATIVE ==
1534		(vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1535					  _DRM_VBLANK_EVENT |
1536					  _DRM_VBLANK_NEXTONMISS));
1537}
1538
1539/*
1540 * Widen a 32-bit param to 64-bits.
1541 *
1542 * \param narrow 32-bit value (missing upper 32 bits)
1543 * \param near 64-bit value that should be 'close' to near
1544 *
1545 * This function returns a 64-bit value using the lower 32-bits from
1546 * 'narrow' and constructing the upper 32-bits so that the result is
1547 * as close as possible to 'near'.
1548 */
1549
1550static u64 widen_32_to_64(u32 narrow, u64 near)
1551{
1552	return near + (s32) (narrow - near);
1553}
1554
1555static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1556				  struct drm_wait_vblank_reply *reply)
1557{
1558	ktime_t now;
1559	struct timespec64 ts;
1560
1561	/*
1562	 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1563	 * to store the seconds. This is safe as we always use monotonic
1564	 * timestamps since linux-4.15.
1565	 */
1566	reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1567	ts = ktime_to_timespec64(now);
1568	reply->tval_sec = (u32)ts.tv_sec;
1569	reply->tval_usec = ts.tv_nsec / 1000;
1570}
1571
1572int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1573			  struct drm_file *file_priv)
1574{
1575	struct drm_crtc *crtc;
1576	struct drm_vblank_crtc *vblank;
1577	union drm_wait_vblank *vblwait = data;
1578	int ret;
1579	u64 req_seq, seq;
1580	unsigned int pipe_index;
1581	unsigned int flags, pipe, high_pipe;
1582
1583	if (!dev->irq_enabled)
1584		return -EINVAL;
1585
1586	if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1587		return -EINVAL;
1588
1589	if (vblwait->request.type &
1590	    ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1591	      _DRM_VBLANK_HIGH_CRTC_MASK)) {
1592		DRM_DEBUG("Unsupported type value 0x%x, supported mask 0x%x\n",
1593			  vblwait->request.type,
1594			  (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1595			   _DRM_VBLANK_HIGH_CRTC_MASK));
1596		return -EINVAL;
1597	}
1598
1599	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1600	high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1601	if (high_pipe)
1602		pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1603	else
1604		pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1605
1606	/* Convert lease-relative crtc index into global crtc index */
1607	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1608		pipe = 0;
1609		drm_for_each_crtc(crtc, dev) {
1610			if (drm_lease_held(file_priv, crtc->base.id)) {
1611				if (pipe_index == 0)
1612					break;
1613				pipe_index--;
1614			}
1615			pipe++;
1616		}
1617	} else {
1618		pipe = pipe_index;
1619	}
1620
1621	if (pipe >= dev->num_crtcs)
1622		return -EINVAL;
1623
1624	vblank = &dev->vblank[pipe];
1625
1626	/* If the counter is currently enabled and accurate, short-circuit
1627	 * queries to return the cached timestamp of the last vblank.
1628	 */
1629	if (dev->vblank_disable_immediate &&
1630	    drm_wait_vblank_is_query(vblwait) &&
1631	    READ_ONCE(vblank->enabled)) {
1632		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1633		return 0;
1634	}
1635
1636	ret = drm_vblank_get(dev, pipe);
1637	if (ret) {
1638		DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1639		return ret;
1640	}
1641	seq = drm_vblank_count(dev, pipe);
1642
1643	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1644	case _DRM_VBLANK_RELATIVE:
1645		req_seq = seq + vblwait->request.sequence;
1646		vblwait->request.sequence = req_seq;
1647		vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1648		break;
1649	case _DRM_VBLANK_ABSOLUTE:
1650		req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1651		break;
1652	default:
1653		ret = -EINVAL;
1654		goto done;
1655	}
1656
1657	if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1658	    vblank_passed(seq, req_seq)) {
1659		req_seq = seq + 1;
1660		vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1661		vblwait->request.sequence = req_seq;
1662	}
1663
1664	if (flags & _DRM_VBLANK_EVENT) {
1665		/* must hold on to the vblank ref until the event fires
1666		 * drm_vblank_put will be called asynchronously
1667		 */
1668		return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1669	}
1670
1671	if (req_seq != seq) {
1672		int wait;
1673
1674		DRM_DEBUG("waiting on vblank count %llu, crtc %u\n",
1675			  req_seq, pipe);
1676		wait = wait_event_interruptible_timeout(vblank->queue,
1677			vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1678				      !READ_ONCE(vblank->enabled),
1679			msecs_to_jiffies(3000));
1680
1681		switch (wait) {
1682		case 0:
1683			/* timeout */
1684			ret = -EBUSY;
1685			break;
1686		case -ERESTARTSYS:
1687			/* interrupted by signal */
1688			ret = -EINTR;
1689			break;
1690		default:
1691			ret = 0;
1692			break;
1693		}
1694	}
1695
1696	if (ret != -EINTR) {
1697		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1698
1699		DRM_DEBUG("crtc %d returning %u to client\n",
1700			  pipe, vblwait->reply.sequence);
1701	} else {
1702		DRM_DEBUG("crtc %d vblank wait interrupted by signal\n", pipe);
1703	}
1704
1705done:
1706	drm_vblank_put(dev, pipe);
1707	return ret;
1708}
1709
1710static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1711{
1712	struct drm_pending_vblank_event *e, *t;
1713	ktime_t now;
1714	u64 seq;
1715
1716	assert_spin_locked(&dev->event_lock);
1717
1718	seq = drm_vblank_count_and_time(dev, pipe, &now);
1719
1720	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1721		if (e->pipe != pipe)
1722			continue;
1723		if (!vblank_passed(seq, e->sequence))
1724			continue;
1725
1726		DRM_DEBUG("vblank event on %llu, current %llu\n",
1727			  e->sequence, seq);
1728
1729		list_del(&e->base.link);
1730		drm_vblank_put(dev, pipe);
1731		send_vblank_event(dev, e, seq, now);
1732	}
1733
1734	trace_drm_vblank_event(pipe, seq);
1735}
1736
1737/**
1738 * drm_handle_vblank - handle a vblank event
1739 * @dev: DRM device
1740 * @pipe: index of CRTC where this event occurred
1741 *
1742 * Drivers should call this routine in their vblank interrupt handlers to
1743 * update the vblank counter and send any signals that may be pending.
1744 *
1745 * This is the legacy version of drm_crtc_handle_vblank().
1746 */
1747bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1748{
1749	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1750	unsigned long irqflags;
1751	bool disable_irq;
1752
1753	if (WARN_ON_ONCE(!dev->num_crtcs))
1754		return false;
1755
1756	if (WARN_ON(pipe >= dev->num_crtcs))
1757		return false;
1758
1759	spin_lock_irqsave(&dev->event_lock, irqflags);
1760
1761	/* Need timestamp lock to prevent concurrent execution with
1762	 * vblank enable/disable, as this would cause inconsistent
1763	 * or corrupted timestamps and vblank counts.
1764	 */
1765	spin_lock(&dev->vblank_time_lock);
1766
1767	/* Vblank irq handling disabled. Nothing to do. */
1768	if (!vblank->enabled) {
1769		spin_unlock(&dev->vblank_time_lock);
1770		spin_unlock_irqrestore(&dev->event_lock, irqflags);
1771		return false;
1772	}
1773
1774	drm_update_vblank_count(dev, pipe, true);
1775
1776	spin_unlock(&dev->vblank_time_lock);
1777
1778	wake_up(&vblank->queue);
1779
1780	/* With instant-off, we defer disabling the interrupt until after
1781	 * we finish processing the following vblank after all events have
1782	 * been signaled. The disable has to be last (after
1783	 * drm_handle_vblank_events) so that the timestamp is always accurate.
1784	 */
1785	disable_irq = (dev->vblank_disable_immediate &&
1786		       drm_vblank_offdelay > 0 &&
1787		       !atomic_read(&vblank->refcount));
1788
1789	drm_handle_vblank_events(dev, pipe);
1790
1791	spin_unlock_irqrestore(&dev->event_lock, irqflags);
1792
1793	if (disable_irq)
1794		vblank_disable_fn(&vblank->disable_timer);
1795
1796	return true;
1797}
1798EXPORT_SYMBOL(drm_handle_vblank);
1799
1800/**
1801 * drm_crtc_handle_vblank - handle a vblank event
1802 * @crtc: where this event occurred
1803 *
1804 * Drivers should call this routine in their vblank interrupt handlers to
1805 * update the vblank counter and send any signals that may be pending.
1806 *
1807 * This is the native KMS version of drm_handle_vblank().
1808 *
1809 * Returns:
1810 * True if the event was successfully handled, false on failure.
1811 */
1812bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1813{
1814	return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1815}
1816EXPORT_SYMBOL(drm_crtc_handle_vblank);
1817
1818/*
1819 * Get crtc VBLANK count.
1820 *
1821 * \param dev DRM device
1822 * \param data user arguement, pointing to a drm_crtc_get_sequence structure.
1823 * \param file_priv drm file private for the user's open file descriptor
1824 */
1825
1826int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1827				struct drm_file *file_priv)
1828{
1829	struct drm_crtc *crtc;
1830	struct drm_vblank_crtc *vblank;
1831	int pipe;
1832	struct drm_crtc_get_sequence *get_seq = data;
1833	ktime_t now;
1834	bool vblank_enabled;
1835	int ret;
1836
1837	if (!drm_core_check_feature(dev, DRIVER_MODESET))
1838		return -EOPNOTSUPP;
1839
1840	if (!dev->irq_enabled)
1841		return -EINVAL;
1842
1843	crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
1844	if (!crtc)
1845		return -ENOENT;
1846
1847	pipe = drm_crtc_index(crtc);
1848
1849	vblank = &dev->vblank[pipe];
1850	vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
1851
1852	if (!vblank_enabled) {
1853		ret = drm_crtc_vblank_get(crtc);
1854		if (ret) {
1855			DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1856			return ret;
1857		}
1858	}
1859	drm_modeset_lock(&crtc->mutex, NULL);
1860	if (crtc->state)
1861		get_seq->active = crtc->state->enable;
1862	else
1863		get_seq->active = crtc->enabled;
1864	drm_modeset_unlock(&crtc->mutex);
1865	get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1866	get_seq->sequence_ns = ktime_to_ns(now);
1867	if (!vblank_enabled)
1868		drm_crtc_vblank_put(crtc);
1869	return 0;
1870}
1871
1872/*
1873 * Queue a event for VBLANK sequence
1874 *
1875 * \param dev DRM device
1876 * \param data user arguement, pointing to a drm_crtc_queue_sequence structure.
1877 * \param file_priv drm file private for the user's open file descriptor
1878 */
1879
1880int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
1881				  struct drm_file *file_priv)
1882{
1883	struct drm_crtc *crtc;
1884	struct drm_vblank_crtc *vblank;
1885	int pipe;
1886	struct drm_crtc_queue_sequence *queue_seq = data;
1887	ktime_t now;
1888	struct drm_pending_vblank_event *e;
1889	u32 flags;
1890	u64 seq;
1891	u64 req_seq;
1892	int ret;
1893	unsigned long spin_flags;
1894
1895	if (!drm_core_check_feature(dev, DRIVER_MODESET))
1896		return -EOPNOTSUPP;
1897
1898	if (!dev->irq_enabled)
1899		return -EINVAL;
1900
1901	crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
1902	if (!crtc)
1903		return -ENOENT;
1904
1905	flags = queue_seq->flags;
1906	/* Check valid flag bits */
1907	if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
1908		      DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
1909		return -EINVAL;
1910
1911	pipe = drm_crtc_index(crtc);
1912
1913	vblank = &dev->vblank[pipe];
1914
1915	e = kzalloc(sizeof(*e), GFP_KERNEL);
1916	if (e == NULL)
1917		return -ENOMEM;
1918
1919	ret = drm_crtc_vblank_get(crtc);
1920	if (ret) {
1921		DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1922		goto err_free;
1923	}
1924
1925	seq = drm_vblank_count_and_time(dev, pipe, &now);
1926	req_seq = queue_seq->sequence;
1927
1928	if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
1929		req_seq += seq;
1930
1931	if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq))
1932		req_seq = seq + 1;
1933
1934	e->pipe = pipe;
1935	e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
1936	e->event.base.length = sizeof(e->event.seq);
1937	e->event.seq.user_data = queue_seq->user_data;
1938
1939	spin_lock_irqsave(&dev->event_lock, spin_flags);
1940
1941	/*
1942	 * drm_crtc_vblank_off() might have been called after we called
1943	 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1944	 * vblank disable, so no need for further locking.  The reference from
1945	 * drm_crtc_vblank_get() protects against vblank disable from another source.
1946	 */
1947	if (!READ_ONCE(vblank->enabled)) {
1948		ret = -EINVAL;
1949		goto err_unlock;
1950	}
1951
1952	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1953					    &e->event.base);
1954
1955	if (ret)
1956		goto err_unlock;
1957
1958	e->sequence = req_seq;
1959
1960	if (vblank_passed(seq, req_seq)) {
1961		drm_crtc_vblank_put(crtc);
1962		send_vblank_event(dev, e, seq, now);
1963		queue_seq->sequence = seq;
1964	} else {
1965		/* drm_handle_vblank_events will call drm_vblank_put */
1966		list_add_tail(&e->base.link, &dev->vblank_event_list);
1967		queue_seq->sequence = req_seq;
1968	}
1969
1970	spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1971	return 0;
1972
1973err_unlock:
1974	spin_unlock_irqrestore(&dev->event_lock, spin_flags);
1975	drm_crtc_vblank_put(crtc);
1976err_free:
1977	kfree(e);
1978	return ret;
1979}