1/*
   2 * drm_irq.c IRQ and vblank support
   3 *
   4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
   5 * \author Gareth Hughes <gareth@valinux.com>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
   6 */
   7
   8/*
   9 * Created: Fri Mar 19 14:30:16 1999 by faith@valinux.com
  10 *
  11 * Copyright 1999, 2000 Precision Insight, Inc., Cedar Park, Texas.
  12 * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California.
  13 * All Rights Reserved.
  14 *
  15 * Permission is hereby granted, free of charge, to any person obtaining a
  16 * copy of this software and associated documentation files (the "Software"),
  17 * to deal in the Software without restriction, including without limitation
  18 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  19 * and/or sell copies of the Software, and to permit persons to whom the
  20 * Software is furnished to do so, subject to the following conditions:
  21 *
  22 * The above copyright notice and this permission notice (including the next
  23 * paragraph) shall be included in all copies or substantial portions of the
  24 * Software.
  25 *
  26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  27 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  28 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  29 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  30 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  31 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  32 * OTHER DEALINGS IN THE SOFTWARE.
  33 */
  34
  35#include <drm/drmP.h>
  36#include "drm_trace.h"
  37#include "drm_internal.h"
  38
 
  39#include <linux/interrupt.h>	/* For task queue support */
  40#include <linux/slab.h>
  41
  42#include <linux/vgaarb.h>
  43#include <linux/export.h>
  44
  45/* Access macro for slots in vblank timestamp ringbuffer. */
  46#define vblanktimestamp(dev, pipe, count) \
  47	((dev)->vblank[pipe].time[(count) % DRM_VBLANKTIME_RBSIZE])
  48
  49/* Retry timestamp calculation up to 3 times to satisfy
  50 * drm_timestamp_precision before giving up.
  51 */
  52#define DRM_TIMESTAMP_MAXRETRIES 3
  53
  54/* Threshold in nanoseconds for detection of redundant
  55 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
  56 */
  57#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
  58
  59static bool
  60drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
  61			  struct timeval *tvblank, unsigned flags);
  62
  63static unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
  64
  65/*
  66 * Default to use monotonic timestamps for wait-for-vblank and page-flip
  67 * complete events.
  68 */
  69unsigned int drm_timestamp_monotonic = 1;
  70
  71static int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
  72
  73module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
  74module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
  75module_param_named(timestamp_monotonic, drm_timestamp_monotonic, int, 0600);
  76MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
  77MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
  78MODULE_PARM_DESC(timestamp_monotonic, "Use monotonic timestamps");
  79
  80static void store_vblank(struct drm_device *dev, unsigned int pipe,
  81			 u32 vblank_count_inc,
  82			 struct timeval *t_vblank, u32 last)
  83{
  84	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
  85	u32 tslot;
  86
  87	assert_spin_locked(&dev->vblank_time_lock);
  88
  89	vblank->last = last;
  90
  91	/* All writers hold the spinlock, but readers are serialized by
  92	 * the latching of vblank->count below.
  93	 */
  94	tslot = vblank->count + vblank_count_inc;
  95	vblanktimestamp(dev, pipe, tslot) = *t_vblank;
  96
  97	/*
  98	 * vblank timestamp updates are protected on the write side with
  99	 * vblank_time_lock, but on the read side done locklessly using a
 100	 * sequence-lock on the vblank counter. Ensure correct ordering using
 101	 * memory barrriers. We need the barrier both before and also after the
 102	 * counter update to synchronize with the next timestamp write.
 103	 * The read-side barriers for this are in drm_vblank_count_and_time.
 104	 */
 105	smp_wmb();
 106	vblank->count += vblank_count_inc;
 107	smp_wmb();
 108}
 109
 110/**
 111 * drm_reset_vblank_timestamp - reset the last timestamp to the last vblank
 112 * @dev: DRM device
 113 * @pipe: index of CRTC for which to reset the timestamp
 114 *
 115 * Reset the stored timestamp for the current vblank count to correspond
 116 * to the last vblank occurred.
 117 *
 118 * Only to be called from drm_vblank_on().
 119 *
 120 * Note: caller must hold dev->vbl_lock since this reads & writes
 121 * device vblank fields.
 122 */
 123static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
 124{
 125	u32 cur_vblank;
 126	bool rc;
 127	struct timeval t_vblank;
 128	int count = DRM_TIMESTAMP_MAXRETRIES;
 129
 130	spin_lock(&dev->vblank_time_lock);
 131
 132	/*
 133	 * sample the current counter to avoid random jumps
 134	 * when drm_vblank_enable() applies the diff
 135	 */
 136	do {
 137		cur_vblank = dev->driver->get_vblank_counter(dev, pipe);
 138		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, 0);
 139	} while (cur_vblank != dev->driver->get_vblank_counter(dev, pipe) && --count > 0);
 140
 141	/*
 142	 * Only reinitialize corresponding vblank timestamp if high-precision query
 143	 * available and didn't fail. Otherwise reinitialize delayed at next vblank
 144	 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
 145	 */
 146	if (!rc)
 147		t_vblank = (struct timeval) {0, 0};
 148
 149	/*
 150	 * +1 to make sure user will never see the same
 151	 * vblank counter value before and after a modeset
 152	 */
 153	store_vblank(dev, pipe, 1, &t_vblank, cur_vblank);
 154
 155	spin_unlock(&dev->vblank_time_lock);
 156}
 157
 158/**
 159 * drm_update_vblank_count - update the master vblank counter
 160 * @dev: DRM device
 161 * @pipe: counter to update
 162 *
 163 * Call back into the driver to update the appropriate vblank counter
 164 * (specified by @pipe).  Deal with wraparound, if it occurred, and
 165 * update the last read value so we can deal with wraparound on the next
 166 * call if necessary.
 167 *
 168 * Only necessary when going from off->on, to account for frames we
 169 * didn't get an interrupt for.
 170 *
 171 * Note: caller must hold dev->vbl_lock since this reads & writes
 172 * device vblank fields.
 173 */
 174static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 175				    unsigned long flags)
 176{
 177	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 178	u32 cur_vblank, diff;
 179	bool rc;
 180	struct timeval t_vblank;
 181	int count = DRM_TIMESTAMP_MAXRETRIES;
 182	int framedur_ns = vblank->framedur_ns;
 183
 184	/*
 185	 * Interrupts were disabled prior to this call, so deal with counter
 186	 * wrap if needed.
 187	 * NOTE!  It's possible we lost a full dev->max_vblank_count + 1 events
 188	 * here if the register is small or we had vblank interrupts off for
 189	 * a long time.
 190	 *
 191	 * We repeat the hardware vblank counter & timestamp query until
 192	 * we get consistent results. This to prevent races between gpu
 193	 * updating its hardware counter while we are retrieving the
 194	 * corresponding vblank timestamp.
 195	 */
 196	do {
 197		cur_vblank = dev->driver->get_vblank_counter(dev, pipe);
 198		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, flags);
 199	} while (cur_vblank != dev->driver->get_vblank_counter(dev, pipe) && --count > 0);
 200
 201	if (dev->max_vblank_count != 0) {
 202		/* trust the hw counter when it's around */
 203		diff = (cur_vblank - vblank->last) & dev->max_vblank_count;
 204	} else if (rc && framedur_ns) {
 205		const struct timeval *t_old;
 206		u64 diff_ns;
 207
 208		t_old = &vblanktimestamp(dev, pipe, vblank->count);
 209		diff_ns = timeval_to_ns(&t_vblank) - timeval_to_ns(t_old);
 210
 211		/*
 212		 * Figure out how many vblanks we've missed based
 213		 * on the difference in the timestamps and the
 214		 * frame/field duration.
 215		 */
 216		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
 217
 218		if (diff == 0 && flags & DRM_CALLED_FROM_VBLIRQ)
 219			DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored."
 220				      " diff_ns = %lld, framedur_ns = %d)\n",
 221				      pipe, (long long) diff_ns, framedur_ns);
 222	} else {
 223		/* some kind of default for drivers w/o accurate vbl timestamping */
 224		diff = (flags & DRM_CALLED_FROM_VBLIRQ) != 0;
 225	}
 226
 227	/*
 228	 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
 229	 * interval? If so then vblank irqs keep running and it will likely
 230	 * happen that the hardware vblank counter is not trustworthy as it
 231	 * might reset at some point in that interval and vblank timestamps
 232	 * are not trustworthy either in that interval. Iow. this can result
 233	 * in a bogus diff >> 1 which must be avoided as it would cause
 234	 * random large forward jumps of the software vblank counter.
 235	 */
 236	if (diff > 1 && (vblank->inmodeset & 0x2)) {
 237		DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
 238			      " due to pre-modeset.\n", pipe, diff);
 239		diff = 1;
 240	}
 241
 242	/*
 243	 * FIMXE: Need to replace this hack with proper seqlocks.
 244	 *
 245	 * Restrict the bump of the software vblank counter to a safe maximum
 246	 * value of +1 whenever there is the possibility that concurrent readers
 247	 * of vblank timestamps could be active at the moment, as the current
 248	 * implementation of the timestamp caching and updating is not safe
 249	 * against concurrent readers for calls to store_vblank() with a bump
 250	 * of anything but +1. A bump != 1 would very likely return corrupted
 251	 * timestamps to userspace, because the same slot in the cache could
 252	 * be concurrently written by store_vblank() and read by one of those
 253	 * readers without the read-retry logic detecting the collision.
 254	 *
 255	 * Concurrent readers can exist when we are called from the
 256	 * drm_vblank_off() or drm_vblank_on() functions and other non-vblank-
 257	 * irq callers. However, all those calls to us are happening with the
 258	 * vbl_lock locked to prevent drm_vblank_get(), so the vblank refcount
 259	 * can't increase while we are executing. Therefore a zero refcount at
 260	 * this point is safe for arbitrary counter bumps if we are called
 261	 * outside vblank irq, a non-zero count is not 100% safe. Unfortunately
 262	 * we must also accept a refcount of 1, as whenever we are called from
 263	 * drm_vblank_get() -> drm_vblank_enable() the refcount will be 1 and
 264	 * we must let that one pass through in order to not lose vblank counts
 265	 * during vblank irq off - which would completely defeat the whole
 266	 * point of this routine.
 267	 *
 268	 * Whenever we are called from vblank irq, we have to assume concurrent
 269	 * readers exist or can show up any time during our execution, even if
 270	 * the refcount is currently zero, as vblank irqs are usually only
 271	 * enabled due to the presence of readers, and because when we are called
 272	 * from vblank irq we can't hold the vbl_lock to protect us from sudden
 273	 * bumps in vblank refcount. Therefore also restrict bumps to +1 when
 274	 * called from vblank irq.
 275	 */
 276	if ((diff > 1) && (atomic_read(&vblank->refcount) > 1 ||
 277	    (flags & DRM_CALLED_FROM_VBLIRQ))) {
 278		DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u "
 279			      "refcount %u, vblirq %u\n", pipe, diff,
 280			      atomic_read(&vblank->refcount),
 281			      (flags & DRM_CALLED_FROM_VBLIRQ) != 0);
 282		diff = 1;
 283	}
 284
 285	DRM_DEBUG_VBL("updating vblank count on crtc %u:"
 286		      " current=%u, diff=%u, hw=%u hw_last=%u\n",
 287		      pipe, vblank->count, diff, cur_vblank, vblank->last);
 288
 289	if (diff == 0) {
 290		WARN_ON_ONCE(cur_vblank != vblank->last);
 291		return;
 292	}
 293
 294	/*
 295	 * Only reinitialize corresponding vblank timestamp if high-precision query
 296	 * available and didn't fail, or we were called from the vblank interrupt.
 297	 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
 298	 * for now, to mark the vblanktimestamp as invalid.
 299	 */
 300	if (!rc && (flags & DRM_CALLED_FROM_VBLIRQ) == 0)
 301		t_vblank = (struct timeval) {0, 0};
 302
 303	store_vblank(dev, pipe, diff, &t_vblank, cur_vblank);
 304}
 305
 306/*
 307 * Disable vblank irq's on crtc, make sure that last vblank count
 308 * of hardware and corresponding consistent software vblank counter
 309 * are preserved, even if there are any spurious vblank irq's after
 310 * disable.
 311 */
 312static void vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
 313{
 314	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 315	unsigned long irqflags;
 316
 317	/* Prevent vblank irq processing while disabling vblank irqs,
 318	 * so no updates of timestamps or count can happen after we've
 319	 * disabled. Needed to prevent races in case of delayed irq's.
 320	 */
 321	spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 322
 323	/*
 324	 * Only disable vblank interrupts if they're enabled. This avoids
 325	 * calling the ->disable_vblank() operation in atomic context with the
 326	 * hardware potentially runtime suspended.
 327	 */
 328	if (vblank->enabled) {
 329		dev->driver->disable_vblank(dev, pipe);
 330		vblank->enabled = false;
 331	}
 332
 333	/*
 334	 * Always update the count and timestamp to maintain the
 335	 * appearance that the counter has been ticking all along until
 336	 * this time. This makes the count account for the entire time
 337	 * between drm_vblank_on() and drm_vblank_off().
 338	 */
 339	drm_update_vblank_count(dev, pipe, 0);
 340
 341	spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
 342}
 343
 344static void vblank_disable_fn(unsigned long arg)
 345{
 346	struct drm_vblank_crtc *vblank = (void *)arg;
 347	struct drm_device *dev = vblank->dev;
 348	unsigned int pipe = vblank->pipe;
 349	unsigned long irqflags;
 350
 351	if (!dev->vblank_disable_allowed)
 352		return;
 353
 354	spin_lock_irqsave(&dev->vbl_lock, irqflags);
 355	if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
 356		DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
 357		vblank_disable_and_save(dev, pipe);
 358	}
 359	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 360}
 361
 362/**
 363 * drm_vblank_cleanup - cleanup vblank support
 364 * @dev: DRM device
 365 *
 366 * This function cleans up any resources allocated in drm_vblank_init.
 367 */
 368void drm_vblank_cleanup(struct drm_device *dev)
 369{
 370	unsigned int pipe;
 371
 372	/* Bail if the driver didn't call drm_vblank_init() */
 373	if (dev->num_crtcs == 0)
 374		return;
 375
 376	for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
 377		struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 378
 379		WARN_ON(vblank->enabled &&
 380			drm_core_check_feature(dev, DRIVER_MODESET));
 381
 382		del_timer_sync(&vblank->disable_timer);
 383	}
 384
 385	kfree(dev->vblank);
 386
 387	dev->num_crtcs = 0;
 388}
 389EXPORT_SYMBOL(drm_vblank_cleanup);
 390
 391/**
 392 * drm_vblank_init - initialize vblank support
 393 * @dev: DRM device
 394 * @num_crtcs: number of CRTCs supported by @dev
 395 *
 396 * This function initializes vblank support for @num_crtcs display pipelines.
 397 *
 398 * Returns:
 399 * Zero on success or a negative error code on failure.
 
 
 
 
 
 
 400 */
 401int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
 402{
 403	int ret = -ENOMEM;
 404	unsigned int i;
 405
 406	spin_lock_init(&dev->vbl_lock);
 407	spin_lock_init(&dev->vblank_time_lock);
 408
 409	dev->num_crtcs = num_crtcs;
 410
 411	dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
 412	if (!dev->vblank)
 413		goto err;
 414
 415	for (i = 0; i < num_crtcs; i++) {
 416		struct drm_vblank_crtc *vblank = &dev->vblank[i];
 417
 418		vblank->dev = dev;
 419		vblank->pipe = i;
 420		init_waitqueue_head(&vblank->queue);
 421		setup_timer(&vblank->disable_timer, vblank_disable_fn,
 422			    (unsigned long)vblank);
 423	}
 424
 425	DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
 426
 427	/* Driver specific high-precision vblank timestamping supported? */
 428	if (dev->driver->get_vblank_timestamp)
 429		DRM_INFO("Driver supports precise vblank timestamp query.\n");
 430	else
 431		DRM_INFO("No driver support for vblank timestamp query.\n");
 432
 433	/* Must have precise timestamping for reliable vblank instant disable */
 434	if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
 435		dev->vblank_disable_immediate = false;
 436		DRM_INFO("Setting vblank_disable_immediate to false because "
 437			 "get_vblank_timestamp == NULL\n");
 438	}
 439
 440	dev->vblank_disable_allowed = false;
 441
 442	return 0;
 443
 444err:
 445	dev->num_crtcs = 0;
 446	return ret;
 447}
 448EXPORT_SYMBOL(drm_vblank_init);
 449
 450static void drm_irq_vgaarb_nokms(void *cookie, bool state)
 451{
 452	struct drm_device *dev = cookie;
 453
 454	if (dev->driver->vgaarb_irq) {
 455		dev->driver->vgaarb_irq(dev, state);
 456		return;
 457	}
 458
 459	if (!dev->irq_enabled)
 460		return;
 461
 462	if (state) {
 463		if (dev->driver->irq_uninstall)
 464			dev->driver->irq_uninstall(dev);
 465	} else {
 466		if (dev->driver->irq_preinstall)
 467			dev->driver->irq_preinstall(dev);
 468		if (dev->driver->irq_postinstall)
 469			dev->driver->irq_postinstall(dev);
 470	}
 471}
 472
 473/**
 474 * drm_irq_install - install IRQ handler
 475 * @dev: DRM device
 476 * @irq: IRQ number to install the handler for
 477 *
 478 * Initializes the IRQ related data. Installs the handler, calling the driver
 479 * irq_preinstall() and irq_postinstall() functions before and after the
 480 * installation.
 481 *
 482 * This is the simplified helper interface provided for drivers with no special
 483 * needs. Drivers which need to install interrupt handlers for multiple
 484 * interrupts must instead set drm_device->irq_enabled to signal the DRM core
 485 * that vblank interrupts are available.
 486 *
 
 
 
 
 
 487 * Returns:
 488 * Zero on success or a negative error code on failure.
 489 */
 490int drm_irq_install(struct drm_device *dev, int irq)
 491{
 492	int ret;
 493	unsigned long sh_flags = 0;
 494
 495	if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
 496		return -EINVAL;
 497
 498	if (irq == 0)
 499		return -EINVAL;
 500
 501	/* Driver must have been initialized */
 502	if (!dev->dev_private)
 503		return -EINVAL;
 504
 505	if (dev->irq_enabled)
 506		return -EBUSY;
 507	dev->irq_enabled = true;
 508
 509	DRM_DEBUG("irq=%d\n", irq);
 510
 511	/* Before installing handler */
 512	if (dev->driver->irq_preinstall)
 513		dev->driver->irq_preinstall(dev);
 514
 515	/* Install handler */
 516	if (drm_core_check_feature(dev, DRIVER_IRQ_SHARED))
 517		sh_flags = IRQF_SHARED;
 518
 519	ret = request_irq(irq, dev->driver->irq_handler,
 520			  sh_flags, dev->driver->name, dev);
 521
 522	if (ret < 0) {
 523		dev->irq_enabled = false;
 524		return ret;
 525	}
 526
 527	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 528		vga_client_register(dev->pdev, (void *)dev, drm_irq_vgaarb_nokms, NULL);
 529
 530	/* After installing handler */
 531	if (dev->driver->irq_postinstall)
 532		ret = dev->driver->irq_postinstall(dev);
 533
 534	if (ret < 0) {
 535		dev->irq_enabled = false;
 536		if (!drm_core_check_feature(dev, DRIVER_MODESET))
 537			vga_client_register(dev->pdev, NULL, NULL, NULL);
 538		free_irq(irq, dev);
 539	} else {
 540		dev->irq = irq;
 541	}
 542
 543	return ret;
 544}
 545EXPORT_SYMBOL(drm_irq_install);
 546
 547/**
 548 * drm_irq_uninstall - uninstall the IRQ handler
 549 * @dev: DRM device
 550 *
 551 * Calls the driver's irq_uninstall() function and unregisters the IRQ handler.
 552 * This should only be called by drivers which used drm_irq_install() to set up
 553 * their interrupt handler. Other drivers must only reset
 554 * drm_device->irq_enabled to false.
 555 *
 556 * Note that for kernel modesetting drivers it is a bug if this function fails.
 557 * The sanity checks are only to catch buggy user modesetting drivers which call
 558 * the same function through an ioctl.
 559 *
 560 * Returns:
 561 * Zero on success or a negative error code on failure.
 562 */
 563int drm_irq_uninstall(struct drm_device *dev)
 564{
 565	unsigned long irqflags;
 566	bool irq_enabled;
 567	int i;
 568
 569	if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
 570		return -EINVAL;
 571
 572	irq_enabled = dev->irq_enabled;
 573	dev->irq_enabled = false;
 574
 575	/*
 576	 * Wake up any waiters so they don't hang. This is just to paper over
 577	 * isssues for UMS drivers which aren't in full control of their
 578	 * vblank/irq handling. KMS drivers must ensure that vblanks are all
 579	 * disabled when uninstalling the irq handler.
 580	 */
 581	if (dev->num_crtcs) {
 582		spin_lock_irqsave(&dev->vbl_lock, irqflags);
 583		for (i = 0; i < dev->num_crtcs; i++) {
 584			struct drm_vblank_crtc *vblank = &dev->vblank[i];
 585
 586			if (!vblank->enabled)
 587				continue;
 588
 589			WARN_ON(drm_core_check_feature(dev, DRIVER_MODESET));
 590
 591			vblank_disable_and_save(dev, i);
 592			wake_up(&vblank->queue);
 593		}
 594		spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 595	}
 596
 597	if (!irq_enabled)
 598		return -EINVAL;
 599
 600	DRM_DEBUG("irq=%d\n", dev->irq);
 601
 602	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 603		vga_client_register(dev->pdev, NULL, NULL, NULL);
 604
 605	if (dev->driver->irq_uninstall)
 606		dev->driver->irq_uninstall(dev);
 607
 608	free_irq(dev->irq, dev);
 609
 610	return 0;
 611}
 612EXPORT_SYMBOL(drm_irq_uninstall);
 613
 614/*
 615 * IRQ control ioctl.
 616 *
 617 * \param inode device inode.
 618 * \param file_priv DRM file private.
 619 * \param cmd command.
 620 * \param arg user argument, pointing to a drm_control structure.
 621 * \return zero on success or a negative number on failure.
 622 *
 623 * Calls irq_install() or irq_uninstall() according to \p arg.
 624 */
 625int drm_control(struct drm_device *dev, void *data,
 626		struct drm_file *file_priv)
 627{
 628	struct drm_control *ctl = data;
 629	int ret = 0, irq;
 630
 631	/* if we haven't irq we fallback for compatibility reasons -
 632	 * this used to be a separate function in drm_dma.h
 633	 */
 634
 635	if (!drm_core_check_feature(dev, DRIVER_HAVE_IRQ))
 636		return 0;
 637	if (drm_core_check_feature(dev, DRIVER_MODESET))
 638		return 0;
 639	/* UMS was only ever support on pci devices. */
 640	if (WARN_ON(!dev->pdev))
 641		return -EINVAL;
 642
 643	switch (ctl->func) {
 644	case DRM_INST_HANDLER:
 645		irq = dev->pdev->irq;
 646
 647		if (dev->if_version < DRM_IF_VERSION(1, 2) &&
 648		    ctl->irq != irq)
 649			return -EINVAL;
 650		mutex_lock(&dev->struct_mutex);
 651		ret = drm_irq_install(dev, irq);
 652		mutex_unlock(&dev->struct_mutex);
 653
 654		return ret;
 655	case DRM_UNINST_HANDLER:
 656		mutex_lock(&dev->struct_mutex);
 657		ret = drm_irq_uninstall(dev);
 658		mutex_unlock(&dev->struct_mutex);
 659
 660		return ret;
 661	default:
 662		return -EINVAL;
 663	}
 664}
 665
 666/**
 667 * drm_calc_timestamping_constants - calculate vblank timestamp constants
 668 * @crtc: drm_crtc whose timestamp constants should be updated.
 669 * @mode: display mode containing the scanout timings
 670 *
 671 * Calculate and store various constants which are later
 672 * needed by vblank and swap-completion timestamping, e.g,
 673 * by drm_calc_vbltimestamp_from_scanoutpos(). They are
 674 * derived from CRTC's true scanout timing, so they take
 675 * things like panel scaling or other adjustments into account.
 676 */
 677void drm_calc_timestamping_constants(struct drm_crtc *crtc,
 678				     const struct drm_display_mode *mode)
 679{
 680	struct drm_device *dev = crtc->dev;
 681	unsigned int pipe = drm_crtc_index(crtc);
 682	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 683	int linedur_ns = 0, framedur_ns = 0;
 684	int dotclock = mode->crtc_clock;
 685
 686	if (!dev->num_crtcs)
 687		return;
 688
 689	if (WARN_ON(pipe >= dev->num_crtcs))
 690		return;
 691
 692	/* Valid dotclock? */
 693	if (dotclock > 0) {
 694		int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
 695
 696		/*
 697		 * Convert scanline length in pixels and video
 698		 * dot clock to line duration and frame duration
 699		 * in nanoseconds:
 700		 */
 701		linedur_ns  = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
 702		framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
 703
 704		/*
 705		 * Fields of interlaced scanout modes are only half a frame duration.
 706		 */
 707		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 708			framedur_ns /= 2;
 709	} else
 710		DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
 711			  crtc->base.id);
 712
 713	vblank->linedur_ns  = linedur_ns;
 714	vblank->framedur_ns = framedur_ns;
 715
 716	DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
 717		  crtc->base.id, mode->crtc_htotal,
 718		  mode->crtc_vtotal, mode->crtc_vdisplay);
 719	DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
 720		  crtc->base.id, dotclock, framedur_ns, linedur_ns);
 721}
 722EXPORT_SYMBOL(drm_calc_timestamping_constants);
 723
 724/**
 725 * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper
 726 * @dev: DRM device
 727 * @pipe: index of CRTC whose vblank timestamp to retrieve
 728 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
 729 *             On return contains true maximum error of timestamp
 730 * @vblank_time: Pointer to struct timeval which should receive the timestamp
 731 * @flags: Flags to pass to driver:
 732 *         0 = Default,
 733 *         DRM_CALLED_FROM_VBLIRQ = If function is called from vbl IRQ handler
 734 * @mode: mode which defines the scanout timings
 735 *
 736 * Implements calculation of exact vblank timestamps from given drm_display_mode
 737 * timings and current video scanout position of a CRTC. This can be called from
 738 * within get_vblank_timestamp() implementation of a kms driver to implement the
 739 * actual timestamping.
 740 *
 741 * Should return timestamps conforming to the OML_sync_control OpenML
 742 * extension specification. The timestamp corresponds to the end of
 743 * the vblank interval, aka start of scanout of topmost-leftmost display
 744 * pixel in the following video frame.
 745 *
 746 * Requires support for optional dev->driver->get_scanout_position()
 747 * in kms driver, plus a bit of setup code to provide a drm_display_mode
 748 * that corresponds to the true scanout timing.
 749 *
 750 * The current implementation only handles standard video modes. It
 751 * returns as no operation if a doublescan or interlaced video mode is
 752 * active. Higher level code is expected to handle this.
 753 *
 754 * Returns:
 755 * Negative value on error, failure or if not supported in current
 756 * video mode:
 757 *
 758 * -EINVAL   - Invalid CRTC.
 759 * -EAGAIN   - Temporary unavailable, e.g., called before initial modeset.
 760 * -ENOTSUPP - Function not supported in current display mode.
 761 * -EIO      - Failed, e.g., due to failed scanout position query.
 762 *
 763 * Returns or'ed positive status flags on success:
 764 *
 765 * DRM_VBLANKTIME_SCANOUTPOS_METHOD - Signal this method used for timestamping.
 766 * DRM_VBLANKTIME_INVBL - Timestamp taken while scanout was in vblank interval.
 767 *
 768 */
 769int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
 770					  unsigned int pipe,
 771					  int *max_error,
 772					  struct timeval *vblank_time,
 773					  unsigned flags,
 774					  const struct drm_display_mode *mode)
 775{
 776	struct timeval tv_etime;
 777	ktime_t stime, etime;
 778	unsigned int vbl_status;
 779	int ret = DRM_VBLANKTIME_SCANOUTPOS_METHOD;
 780	int vpos, hpos, i;
 781	int delta_ns, duration_ns;
 782
 783	if (pipe >= dev->num_crtcs) {
 784		DRM_ERROR("Invalid crtc %u\n", pipe);
 785		return -EINVAL;
 786	}
 787
 788	/* Scanout position query not supported? Should not happen. */
 789	if (!dev->driver->get_scanout_position) {
 790		DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
 791		return -EIO;
 792	}
 793
 794	/* If mode timing undefined, just return as no-op:
 795	 * Happens during initial modesetting of a crtc.
 796	 */
 797	if (mode->crtc_clock == 0) {
 798		DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
 799		return -EAGAIN;
 800	}
 801
 802	/* Get current scanout position with system timestamp.
 803	 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
 804	 * if single query takes longer than max_error nanoseconds.
 805	 *
 806	 * This guarantees a tight bound on maximum error if
 807	 * code gets preempted or delayed for some reason.
 808	 */
 809	for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
 810		/*
 811		 * Get vertical and horizontal scanout position vpos, hpos,
 812		 * and bounding timestamps stime, etime, pre/post query.
 813		 */
 814		vbl_status = dev->driver->get_scanout_position(dev, pipe, flags,
 815							       &vpos, &hpos,
 816							       &stime, &etime,
 817							       mode);
 818
 819		/* Return as no-op if scanout query unsupported or failed. */
 820		if (!(vbl_status & DRM_SCANOUTPOS_VALID)) {
 821			DRM_DEBUG("crtc %u : scanoutpos query failed [0x%x].\n",
 822				  pipe, vbl_status);
 823			return -EIO;
 824		}
 825
 826		/* Compute uncertainty in timestamp of scanout position query. */
 827		duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
 828
 829		/* Accept result with <  max_error nsecs timing uncertainty. */
 830		if (duration_ns <= *max_error)
 831			break;
 832	}
 833
 834	/* Noisy system timing? */
 835	if (i == DRM_TIMESTAMP_MAXRETRIES) {
 836		DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
 837			  pipe, duration_ns/1000, *max_error/1000, i);
 838	}
 839
 840	/* Return upper bound of timestamp precision error. */
 841	*max_error = duration_ns;
 842
 843	/* Check if in vblank area:
 844	 * vpos is >=0 in video scanout area, but negative
 845	 * within vblank area, counting down the number of lines until
 846	 * start of scanout.
 847	 */
 848	if (vbl_status & DRM_SCANOUTPOS_IN_VBLANK)
 849		ret |= DRM_VBLANKTIME_IN_VBLANK;
 850
 851	/* Convert scanout position into elapsed time at raw_time query
 852	 * since start of scanout at first display scanline. delta_ns
 853	 * can be negative if start of scanout hasn't happened yet.
 854	 */
 855	delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
 856			   mode->crtc_clock);
 857
 858	if (!drm_timestamp_monotonic)
 859		etime = ktime_mono_to_real(etime);
 860
 861	/* save this only for debugging purposes */
 862	tv_etime = ktime_to_timeval(etime);
 863	/* Subtract time delta from raw timestamp to get final
 864	 * vblank_time timestamp for end of vblank.
 865	 */
 866	if (delta_ns < 0)
 867		etime = ktime_add_ns(etime, -delta_ns);
 868	else
 869		etime = ktime_sub_ns(etime, delta_ns);
 870	*vblank_time = ktime_to_timeval(etime);
 871
 872	DRM_DEBUG_VBL("crtc %u : v 0x%x p(%d,%d)@ %ld.%ld -> %ld.%ld [e %d us, %d rep]\n",
 873		      pipe, vbl_status, hpos, vpos,
 874		      (long)tv_etime.tv_sec, (long)tv_etime.tv_usec,
 875		      (long)vblank_time->tv_sec, (long)vblank_time->tv_usec,
 876		      duration_ns/1000, i);
 877
 878	return ret;
 879}
 880EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
 881
 882static struct timeval get_drm_timestamp(void)
 883{
 884	ktime_t now;
 885
 886	now = drm_timestamp_monotonic ? ktime_get() : ktime_get_real();
 887	return ktime_to_timeval(now);
 888}
 889
 890/**
 891 * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
 892 *                             vblank interval
 893 * @dev: DRM device
 894 * @pipe: index of CRTC whose vblank timestamp to retrieve
 895 * @tvblank: Pointer to target struct timeval which should receive the timestamp
 896 * @flags: Flags to pass to driver:
 897 *         0 = Default,
 898 *         DRM_CALLED_FROM_VBLIRQ = If function is called from vbl IRQ handler
 899 *
 900 * Fetches the system timestamp corresponding to the time of the most recent
 901 * vblank interval on specified CRTC. May call into kms-driver to
 902 * compute the timestamp with a high-precision GPU specific method.
 903 *
 904 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
 905 * call, i.e., it isn't very precisely locked to the true vblank.
 906 *
 907 * Returns:
 908 * True if timestamp is considered to be very precise, false otherwise.
 909 */
 910static bool
 911drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 912			  struct timeval *tvblank, unsigned flags)
 913{
 914	int ret;
 915
 916	/* Define requested maximum error on timestamps (nanoseconds). */
 917	int max_error = (int) drm_timestamp_precision * 1000;
 918
 919	/* Query driver if possible and precision timestamping enabled. */
 920	if (dev->driver->get_vblank_timestamp && (max_error > 0)) {
 921		ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
 922							tvblank, flags);
 923		if (ret > 0)
 924			return true;
 925	}
 926
 927	/* GPU high precision timestamp query unsupported or failed.
 928	 * Return current monotonic/gettimeofday timestamp as best estimate.
 929	 */
 930	*tvblank = get_drm_timestamp();
 931
 932	return false;
 933}
 934
 935/**
 936 * drm_vblank_count - retrieve "cooked" vblank counter value
 937 * @dev: DRM device
 938 * @pipe: index of CRTC for which to retrieve the counter
 939 *
 940 * Fetches the "cooked" vblank count value that represents the number of
 941 * vblank events since the system was booted, including lost events due to
 942 * modesetting activity.
 943 *
 944 * This is the legacy version of drm_crtc_vblank_count().
 945 *
 946 * Returns:
 947 * The software vblank counter.
 948 */
 949u32 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
 950{
 951	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 952
 953	if (WARN_ON(pipe >= dev->num_crtcs))
 954		return 0;
 955
 956	return vblank->count;
 957}
 958EXPORT_SYMBOL(drm_vblank_count);
 959
 960/**
 961 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
 962 * @crtc: which counter to retrieve
 963 *
 964 * Fetches the "cooked" vblank count value that represents the number of
 965 * vblank events since the system was booted, including lost events due to
 966 * modesetting activity.
 967 *
 968 * This is the native KMS version of drm_vblank_count().
 969 *
 970 * Returns:
 971 * The software vblank counter.
 972 */
 973u32 drm_crtc_vblank_count(struct drm_crtc *crtc)
 974{
 975	return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
 976}
 977EXPORT_SYMBOL(drm_crtc_vblank_count);
 978
 979/**
 980 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
 981 *     system timestamp corresponding to that vblank counter value.
 982 * @dev: DRM device
 983 * @pipe: index of CRTC whose counter to retrieve
 984 * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
 985 *
 986 * Fetches the "cooked" vblank count value that represents the number of
 987 * vblank events since the system was booted, including lost events due to
 988 * modesetting activity. Returns corresponding system timestamp of the time
 989 * of the vblank interval that corresponds to the current vblank counter value.
 990 *
 991 * This is the legacy version of drm_crtc_vblank_count_and_time().
 992 */
 993u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
 994			      struct timeval *vblanktime)
 995{
 996	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 997	int count = DRM_TIMESTAMP_MAXRETRIES;
 998	u32 cur_vblank;
 999
1000	if (WARN_ON(pipe >= dev->num_crtcs))
1001		return 0;
1002
1003	/*
1004	 * Vblank timestamps are read lockless. To ensure consistency the vblank
1005	 * counter is rechecked and ordering is ensured using memory barriers.
1006	 * This works like a seqlock. The write-side barriers are in store_vblank.
1007	 */
1008	do {
1009		cur_vblank = vblank->count;
1010		smp_rmb();
1011		*vblanktime = vblanktimestamp(dev, pipe, cur_vblank);
1012		smp_rmb();
1013	} while (cur_vblank != vblank->count && --count > 0);
1014
1015	return cur_vblank;
1016}
1017EXPORT_SYMBOL(drm_vblank_count_and_time);
1018
1019/**
1020 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
1021 *     and the system timestamp corresponding to that vblank counter value
1022 * @crtc: which counter to retrieve
1023 * @vblanktime: Pointer to struct timeval to receive the vblank timestamp.
1024 *
1025 * Fetches the "cooked" vblank count value that represents the number of
1026 * vblank events since the system was booted, including lost events due to
1027 * modesetting activity. Returns corresponding system timestamp of the time
1028 * of the vblank interval that corresponds to the current vblank counter value.
1029 *
1030 * This is the native KMS version of drm_vblank_count_and_time().
1031 */
1032u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
1033				   struct timeval *vblanktime)
1034{
1035	return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
1036					 vblanktime);
1037}
1038EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
1039
1040static void send_vblank_event(struct drm_device *dev,
1041		struct drm_pending_vblank_event *e,
1042		unsigned long seq, struct timeval *now)
1043{
1044	e->event.sequence = seq;
1045	e->event.tv_sec = now->tv_sec;
1046	e->event.tv_usec = now->tv_usec;
1047
1048	drm_send_event_locked(dev, &e->base);
1049
1050	trace_drm_vblank_event_delivered(e->base.pid, e->pipe,
1051					 e->event.sequence);
1052}
1053
1054/**
1055 * drm_arm_vblank_event - arm vblank event after pageflip
1056 * @dev: DRM device
1057 * @pipe: CRTC index
1058 * @e: the event to prepare to send
1059 *
1060 * A lot of drivers need to generate vblank events for the very next vblank
1061 * interrupt. For example when the page flip interrupt happens when the page
1062 * flip gets armed, but not when it actually executes within the next vblank
1063 * period. This helper function implements exactly the required vblank arming
1064 * behaviour.
1065 *
1066 * Caller must hold event lock. Caller must also hold a vblank reference for
1067 * the event @e, which will be dropped when the next vblank arrives.
1068 *
1069 * This is the legacy version of drm_crtc_arm_vblank_event().
1070 */
1071void drm_arm_vblank_event(struct drm_device *dev, unsigned int pipe,
1072			  struct drm_pending_vblank_event *e)
1073{
1074	assert_spin_locked(&dev->event_lock);
1075
1076	e->pipe = pipe;
1077	e->event.sequence = drm_vblank_count(dev, pipe);
1078	list_add_tail(&e->base.link, &dev->vblank_event_list);
1079}
1080EXPORT_SYMBOL(drm_arm_vblank_event);
1081
1082/**
1083 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1084 * @crtc: the source CRTC of the vblank event
1085 * @e: the event to send
1086 *
1087 * A lot of drivers need to generate vblank events for the very next vblank
1088 * interrupt. For example when the page flip interrupt happens when the page
1089 * flip gets armed, but not when it actually executes within the next vblank
1090 * period. This helper function implements exactly the required vblank arming
1091 * behaviour.
1092 *
1093 * Caller must hold event lock. Caller must also hold a vblank reference for
1094 * the event @e, which will be dropped when the next vblank arrives.
1095 *
1096 * This is the native KMS version of drm_arm_vblank_event().
1097 */
1098void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1099			       struct drm_pending_vblank_event *e)
1100{
1101	drm_arm_vblank_event(crtc->dev, drm_crtc_index(crtc), e);
1102}
1103EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1104
1105/**
1106 * drm_send_vblank_event - helper to send vblank event after pageflip
1107 * @dev: DRM device
1108 * @pipe: CRTC index
1109 * @e: the event to send
1110 *
1111 * Updates sequence # and timestamp on event, and sends it to userspace.
1112 * Caller must hold event lock.
1113 *
1114 * This is the legacy version of drm_crtc_send_vblank_event().
1115 */
1116void drm_send_vblank_event(struct drm_device *dev, unsigned int pipe,
1117			   struct drm_pending_vblank_event *e)
1118{
1119	struct timeval now;
1120	unsigned int seq;
1121
1122	if (dev->num_crtcs > 0) {
1123		seq = drm_vblank_count_and_time(dev, pipe, &now);
1124	} else {
1125		seq = 0;
1126
1127		now = get_drm_timestamp();
1128	}
1129	e->pipe = pipe;
1130	send_vblank_event(dev, e, seq, &now);
1131}
1132EXPORT_SYMBOL(drm_send_vblank_event);
1133
1134/**
1135 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1136 * @crtc: the source CRTC of the vblank event
1137 * @e: the event to send
1138 *
1139 * Updates sequence # and timestamp on event, and sends it to userspace.
1140 * Caller must hold event lock.
1141 *
1142 * This is the native KMS version of drm_send_vblank_event().
1143 */
1144void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1145				struct drm_pending_vblank_event *e)
1146{
1147	drm_send_vblank_event(crtc->dev, drm_crtc_index(crtc), e);
1148}
1149EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1150
1151/**
1152 * drm_vblank_enable - enable the vblank interrupt on a CRTC
1153 * @dev: DRM device
1154 * @pipe: CRTC index
1155 *
1156 * Returns:
1157 * Zero on success or a negative error code on failure.
1158 */
1159static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1160{
1161	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1162	int ret = 0;
1163
1164	assert_spin_locked(&dev->vbl_lock);
1165
1166	spin_lock(&dev->vblank_time_lock);
1167
1168	if (!vblank->enabled) {
1169		/*
1170		 * Enable vblank irqs under vblank_time_lock protection.
1171		 * All vblank count & timestamp updates are held off
1172		 * until we are done reinitializing master counter and
1173		 * timestamps. Filtercode in drm_handle_vblank() will
1174		 * prevent double-accounting of same vblank interval.
1175		 */
1176		ret = dev->driver->enable_vblank(dev, pipe);
1177		DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
1178		if (ret)
1179			atomic_dec(&vblank->refcount);
1180		else {
1181			vblank->enabled = true;
1182			drm_update_vblank_count(dev, pipe, 0);
1183		}
1184	}
1185
1186	spin_unlock(&dev->vblank_time_lock);
1187
1188	return ret;
1189}
1190
1191/**
1192 * drm_vblank_get - get a reference count on vblank events
1193 * @dev: DRM device
1194 * @pipe: index of CRTC to own
1195 *
1196 * Acquire a reference count on vblank events to avoid having them disabled
1197 * while in use.
1198 *
1199 * This is the legacy version of drm_crtc_vblank_get().
1200 *
1201 * Returns:
1202 * Zero on success or a negative error code on failure.
1203 */
1204int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1205{
1206	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1207	unsigned long irqflags;
1208	int ret = 0;
1209
1210	if (!dev->num_crtcs)
1211		return -EINVAL;
1212
1213	if (WARN_ON(pipe >= dev->num_crtcs))
1214		return -EINVAL;
1215
1216	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1217	/* Going from 0->1 means we have to enable interrupts again */
1218	if (atomic_add_return(1, &vblank->refcount) == 1) {
1219		ret = drm_vblank_enable(dev, pipe);
1220	} else {
1221		if (!vblank->enabled) {
1222			atomic_dec(&vblank->refcount);
1223			ret = -EINVAL;
1224		}
1225	}
1226	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1227
1228	return ret;
1229}
1230EXPORT_SYMBOL(drm_vblank_get);
1231
1232/**
1233 * drm_crtc_vblank_get - get a reference count on vblank events
1234 * @crtc: which CRTC to own
1235 *
1236 * Acquire a reference count on vblank events to avoid having them disabled
1237 * while in use.
1238 *
1239 * This is the native kms version of drm_vblank_get().
1240 *
1241 * Returns:
1242 * Zero on success or a negative error code on failure.
1243 */
1244int drm_crtc_vblank_get(struct drm_crtc *crtc)
1245{
1246	return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1247}
1248EXPORT_SYMBOL(drm_crtc_vblank_get);
1249
1250/**
1251 * drm_vblank_put - release ownership of vblank events
1252 * @dev: DRM device
1253 * @pipe: index of CRTC to release
1254 *
1255 * Release ownership of a given vblank counter, turning off interrupts
1256 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1257 *
1258 * This is the legacy version of drm_crtc_vblank_put().
1259 */
1260void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1261{
1262	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1263
1264	if (WARN_ON(pipe >= dev->num_crtcs))
1265		return;
1266
1267	if (WARN_ON(atomic_read(&vblank->refcount) == 0))
1268		return;
1269
1270	/* Last user schedules interrupt disable */
1271	if (atomic_dec_and_test(&vblank->refcount)) {
1272		if (drm_vblank_offdelay == 0)
1273			return;
1274		else if (dev->vblank_disable_immediate || drm_vblank_offdelay < 0)
1275			vblank_disable_fn((unsigned long)vblank);
1276		else
1277			mod_timer(&vblank->disable_timer,
1278				  jiffies + ((drm_vblank_offdelay * HZ)/1000));
1279	}
1280}
1281EXPORT_SYMBOL(drm_vblank_put);
1282
1283/**
1284 * drm_crtc_vblank_put - give up ownership of vblank events
1285 * @crtc: which counter to give up
1286 *
1287 * Release ownership of a given vblank counter, turning off interrupts
1288 * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1289 *
1290 * This is the native kms version of drm_vblank_put().
1291 */
1292void drm_crtc_vblank_put(struct drm_crtc *crtc)
1293{
1294	drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1295}
1296EXPORT_SYMBOL(drm_crtc_vblank_put);
1297
1298/**
1299 * drm_wait_one_vblank - wait for one vblank
1300 * @dev: DRM device
1301 * @pipe: CRTC index
1302 *
1303 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1304 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1305 * due to lack of driver support or because the crtc is off.
1306 */
1307void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1308{
1309	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1310	int ret;
1311	u32 last;
1312
1313	if (WARN_ON(pipe >= dev->num_crtcs))
1314		return;
1315
1316	ret = drm_vblank_get(dev, pipe);
1317	if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
1318		return;
1319
1320	last = drm_vblank_count(dev, pipe);
1321
1322	ret = wait_event_timeout(vblank->queue,
1323				 last != drm_vblank_count(dev, pipe),
1324				 msecs_to_jiffies(100));
1325
1326	WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1327
1328	drm_vblank_put(dev, pipe);
1329}
1330EXPORT_SYMBOL(drm_wait_one_vblank);
1331
1332/**
1333 * drm_crtc_wait_one_vblank - wait for one vblank
1334 * @crtc: DRM crtc
1335 *
1336 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1337 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1338 * due to lack of driver support or because the crtc is off.
1339 */
1340void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1341{
1342	drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1343}
1344EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1345
1346/**
1347 * drm_vblank_off - disable vblank events on a CRTC
1348 * @dev: DRM device
1349 * @pipe: CRTC index
1350 *
1351 * Drivers can use this function to shut down the vblank interrupt handling when
1352 * disabling a crtc. This function ensures that the latest vblank frame count is
1353 * stored so that drm_vblank_on() can restore it again.
1354 *
1355 * Drivers must use this function when the hardware vblank counter can get
1356 * reset, e.g. when suspending.
1357 *
1358 * This is the legacy version of drm_crtc_vblank_off().
1359 */
1360void drm_vblank_off(struct drm_device *dev, unsigned int pipe)
1361{
1362	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1363	struct drm_pending_vblank_event *e, *t;
1364	struct timeval now;
1365	unsigned long irqflags;
1366	unsigned int seq;
1367
1368	if (WARN_ON(pipe >= dev->num_crtcs))
1369		return;
1370
1371	spin_lock_irqsave(&dev->event_lock, irqflags);
1372
1373	spin_lock(&dev->vbl_lock);
1374	DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1375		      pipe, vblank->enabled, vblank->inmodeset);
1376
1377	/* Avoid redundant vblank disables without previous drm_vblank_on(). */
1378	if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1379		vblank_disable_and_save(dev, pipe);
1380
1381	wake_up(&vblank->queue);
1382
1383	/*
1384	 * Prevent subsequent drm_vblank_get() from re-enabling
1385	 * the vblank interrupt by bumping the refcount.
1386	 */
1387	if (!vblank->inmodeset) {
1388		atomic_inc(&vblank->refcount);
1389		vblank->inmodeset = 1;
1390	}
1391	spin_unlock(&dev->vbl_lock);
1392
1393	/* Send any queued vblank events, lest the natives grow disquiet */
1394	seq = drm_vblank_count_and_time(dev, pipe, &now);
1395
1396	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1397		if (e->pipe != pipe)
1398			continue;
1399		DRM_DEBUG("Sending premature vblank event on disable: "
1400			  "wanted %d, current %d\n",
1401			  e->event.sequence, seq);
1402		list_del(&e->base.link);
1403		drm_vblank_put(dev, pipe);
1404		send_vblank_event(dev, e, seq, &now);
1405	}
1406	spin_unlock_irqrestore(&dev->event_lock, irqflags);
1407}
1408EXPORT_SYMBOL(drm_vblank_off);
1409
1410/**
1411 * drm_crtc_vblank_off - disable vblank events on a CRTC
1412 * @crtc: CRTC in question
1413 *
1414 * Drivers can use this function to shut down the vblank interrupt handling when
1415 * disabling a crtc. This function ensures that the latest vblank frame count is
1416 * stored so that drm_vblank_on can restore it again.
1417 *
1418 * Drivers must use this function when the hardware vblank counter can get
1419 * reset, e.g. when suspending.
1420 *
1421 * This is the native kms version of drm_vblank_off().
1422 */
1423void drm_crtc_vblank_off(struct drm_crtc *crtc)
1424{
1425	drm_vblank_off(crtc->dev, drm_crtc_index(crtc));
1426}
1427EXPORT_SYMBOL(drm_crtc_vblank_off);
1428
1429/**
1430 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1431 * @crtc: CRTC in question
1432 *
1433 * Drivers can use this function to reset the vblank state to off at load time.
1434 * Drivers should use this together with the drm_crtc_vblank_off() and
1435 * drm_crtc_vblank_on() functions. The difference compared to
1436 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1437 * and hence doesn't need to call any driver hooks.
1438 */
1439void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1440{
1441	struct drm_device *dev = crtc->dev;
1442	unsigned long irqflags;
1443	unsigned int pipe = drm_crtc_index(crtc);
1444	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1445
1446	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1447	/*
1448	 * Prevent subsequent drm_vblank_get() from enabling the vblank
1449	 * interrupt by bumping the refcount.
1450	 */
1451	if (!vblank->inmodeset) {
1452		atomic_inc(&vblank->refcount);
1453		vblank->inmodeset = 1;
1454	}
1455	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1456
1457	WARN_ON(!list_empty(&dev->vblank_event_list));
1458}
1459EXPORT_SYMBOL(drm_crtc_vblank_reset);
1460
1461/**
1462 * drm_vblank_on - enable vblank events on a CRTC
1463 * @dev: DRM device
1464 * @pipe: CRTC index
1465 *
1466 * This functions restores the vblank interrupt state captured with
1467 * drm_vblank_off() again. Note that calls to drm_vblank_on() and
1468 * drm_vblank_off() can be unbalanced and so can also be unconditionally called
1469 * in driver load code to reflect the current hardware state of the crtc.
1470 *
1471 * This is the legacy version of drm_crtc_vblank_on().
1472 */
1473void drm_vblank_on(struct drm_device *dev, unsigned int pipe)
1474{
1475	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1476	unsigned long irqflags;
1477
1478	if (WARN_ON(pipe >= dev->num_crtcs))
1479		return;
1480
1481	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1482	DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1483		      pipe, vblank->enabled, vblank->inmodeset);
1484
1485	/* Drop our private "prevent drm_vblank_get" refcount */
1486	if (vblank->inmodeset) {
1487		atomic_dec(&vblank->refcount);
1488		vblank->inmodeset = 0;
1489	}
1490
1491	drm_reset_vblank_timestamp(dev, pipe);
1492
1493	/*
1494	 * re-enable interrupts if there are users left, or the
1495	 * user wishes vblank interrupts to be enabled all the time.
1496	 */
1497	if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1498		WARN_ON(drm_vblank_enable(dev, pipe));
1499	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1500}
1501EXPORT_SYMBOL(drm_vblank_on);
1502
1503/**
1504 * drm_crtc_vblank_on - enable vblank events on a CRTC
1505 * @crtc: CRTC in question
1506 *
1507 * This functions restores the vblank interrupt state captured with
1508 * drm_vblank_off() again. Note that calls to drm_vblank_on() and
1509 * drm_vblank_off() can be unbalanced and so can also be unconditionally called
1510 * in driver load code to reflect the current hardware state of the crtc.
1511 *
1512 * This is the native kms version of drm_vblank_on().
1513 */
1514void drm_crtc_vblank_on(struct drm_crtc *crtc)
1515{
1516	drm_vblank_on(crtc->dev, drm_crtc_index(crtc));
1517}
1518EXPORT_SYMBOL(drm_crtc_vblank_on);
1519
1520/**
1521 * drm_vblank_pre_modeset - account for vblanks across mode sets
1522 * @dev: DRM device
1523 * @pipe: CRTC index
1524 *
1525 * Account for vblank events across mode setting events, which will likely
1526 * reset the hardware frame counter.
1527 *
1528 * This is done by grabbing a temporary vblank reference to ensure that the
1529 * vblank interrupt keeps running across the modeset sequence. With this the
1530 * software-side vblank frame counting will ensure that there are no jumps or
1531 * discontinuities.
1532 *
1533 * Unfortunately this approach is racy and also doesn't work when the vblank
1534 * interrupt stops running, e.g. across system suspend resume. It is therefore
1535 * highly recommended that drivers use the newer drm_vblank_off() and
1536 * drm_vblank_on() instead. drm_vblank_pre_modeset() only works correctly when
1537 * using "cooked" software vblank frame counters and not relying on any hardware
1538 * counters.
1539 *
1540 * Drivers must call drm_vblank_post_modeset() when re-enabling the same crtc
1541 * again.
1542 */
1543void drm_vblank_pre_modeset(struct drm_device *dev, unsigned int pipe)
1544{
1545	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1546
1547	/* vblank is not initialized (IRQ not installed ?), or has been freed */
1548	if (!dev->num_crtcs)
1549		return;
1550
1551	if (WARN_ON(pipe >= dev->num_crtcs))
1552		return;
1553
1554	/*
1555	 * To avoid all the problems that might happen if interrupts
1556	 * were enabled/disabled around or between these calls, we just
1557	 * have the kernel take a reference on the CRTC (just once though
1558	 * to avoid corrupting the count if multiple, mismatch calls occur),
1559	 * so that interrupts remain enabled in the interim.
1560	 */
1561	if (!vblank->inmodeset) {
1562		vblank->inmodeset = 0x1;
1563		if (drm_vblank_get(dev, pipe) == 0)
1564			vblank->inmodeset |= 0x2;
1565	}
1566}
1567EXPORT_SYMBOL(drm_vblank_pre_modeset);
1568
1569/**
1570 * drm_vblank_post_modeset - undo drm_vblank_pre_modeset changes
1571 * @dev: DRM device
1572 * @pipe: CRTC index
1573 *
1574 * This function again drops the temporary vblank reference acquired in
1575 * drm_vblank_pre_modeset.
1576 */
1577void drm_vblank_post_modeset(struct drm_device *dev, unsigned int pipe)
1578{
1579	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1580	unsigned long irqflags;
1581
1582	/* vblank is not initialized (IRQ not installed ?), or has been freed */
1583	if (!dev->num_crtcs)
1584		return;
1585
1586	if (WARN_ON(pipe >= dev->num_crtcs))
1587		return;
1588
1589	if (vblank->inmodeset) {
1590		spin_lock_irqsave(&dev->vbl_lock, irqflags);
1591		dev->vblank_disable_allowed = true;
1592		drm_reset_vblank_timestamp(dev, pipe);
1593		spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1594
1595		if (vblank->inmodeset & 0x2)
1596			drm_vblank_put(dev, pipe);
1597
1598		vblank->inmodeset = 0;
1599	}
1600}
1601EXPORT_SYMBOL(drm_vblank_post_modeset);
1602
1603/*
1604 * drm_modeset_ctl - handle vblank event counter changes across mode switch
1605 * @DRM_IOCTL_ARGS: standard ioctl arguments
1606 *
1607 * Applications should call the %_DRM_PRE_MODESET and %_DRM_POST_MODESET
1608 * ioctls around modesetting so that any lost vblank events are accounted for.
1609 *
1610 * Generally the counter will reset across mode sets.  If interrupts are
1611 * enabled around this call, we don't have to do anything since the counter
1612 * will have already been incremented.
1613 */
1614int drm_modeset_ctl(struct drm_device *dev, void *data,
1615		    struct drm_file *file_priv)
1616{
1617	struct drm_modeset_ctl *modeset = data;
1618	unsigned int pipe;
1619
1620	/* If drm_vblank_init() hasn't been called yet, just no-op */
1621	if (!dev->num_crtcs)
1622		return 0;
1623
1624	/* KMS drivers handle this internally */
1625	if (drm_core_check_feature(dev, DRIVER_MODESET))
1626		return 0;
1627
1628	pipe = modeset->crtc;
1629	if (pipe >= dev->num_crtcs)
1630		return -EINVAL;
1631
1632	switch (modeset->cmd) {
1633	case _DRM_PRE_MODESET:
1634		drm_vblank_pre_modeset(dev, pipe);
1635		break;
1636	case _DRM_POST_MODESET:
1637		drm_vblank_post_modeset(dev, pipe);
1638		break;
1639	default:
1640		return -EINVAL;
1641	}
1642
1643	return 0;
1644}
1645
1646static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1647				  union drm_wait_vblank *vblwait,
1648				  struct drm_file *file_priv)
1649{
1650	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1651	struct drm_pending_vblank_event *e;
1652	struct timeval now;
1653	unsigned long flags;
1654	unsigned int seq;
1655	int ret;
1656
1657	e = kzalloc(sizeof(*e), GFP_KERNEL);
1658	if (e == NULL) {
1659		ret = -ENOMEM;
1660		goto err_put;
1661	}
1662
1663	e->pipe = pipe;
1664	e->base.pid = current->pid;
1665	e->event.base.type = DRM_EVENT_VBLANK;
1666	e->event.base.length = sizeof(e->event);
1667	e->event.user_data = vblwait->request.signal;
1668
1669	spin_lock_irqsave(&dev->event_lock, flags);
1670
1671	/*
1672	 * drm_vblank_off() might have been called after we called
1673	 * drm_vblank_get(). drm_vblank_off() holds event_lock
1674	 * around the vblank disable, so no need for further locking.
1675	 * The reference from drm_vblank_get() protects against
1676	 * vblank disable from another source.
1677	 */
1678	if (!vblank->enabled) {
1679		ret = -EINVAL;
1680		goto err_unlock;
1681	}
1682
1683	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1684					    &e->event.base);
1685
1686	if (ret)
1687		goto err_unlock;
1688
1689	seq = drm_vblank_count_and_time(dev, pipe, &now);
1690
1691	if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) &&
1692	    (seq - vblwait->request.sequence) <= (1 << 23)) {
1693		vblwait->request.sequence = seq + 1;
1694		vblwait->reply.sequence = vblwait->request.sequence;
1695	}
1696
1697	DRM_DEBUG("event on vblank count %d, current %d, crtc %u\n",
1698		  vblwait->request.sequence, seq, pipe);
1699
1700	trace_drm_vblank_event_queued(current->pid, pipe,
1701				      vblwait->request.sequence);
1702
1703	e->event.sequence = vblwait->request.sequence;
1704	if ((seq - vblwait->request.sequence) <= (1 << 23)) {
1705		drm_vblank_put(dev, pipe);
1706		send_vblank_event(dev, e, seq, &now);
1707		vblwait->reply.sequence = seq;
1708	} else {
1709		/* drm_handle_vblank_events will call drm_vblank_put */
1710		list_add_tail(&e->base.link, &dev->vblank_event_list);
1711		vblwait->reply.sequence = vblwait->request.sequence;
1712	}
1713
1714	spin_unlock_irqrestore(&dev->event_lock, flags);
1715
1716	return 0;
1717
1718err_unlock:
1719	spin_unlock_irqrestore(&dev->event_lock, flags);
1720	kfree(e);
1721err_put:
1722	drm_vblank_put(dev, pipe);
1723	return ret;
1724}
1725
1726/*
1727 * Wait for VBLANK.
1728 *
1729 * \param inode device inode.
1730 * \param file_priv DRM file private.
1731 * \param cmd command.
1732 * \param data user argument, pointing to a drm_wait_vblank structure.
1733 * \return zero on success or a negative number on failure.
1734 *
1735 * This function enables the vblank interrupt on the pipe requested, then
1736 * sleeps waiting for the requested sequence number to occur, and drops
1737 * the vblank interrupt refcount afterwards. (vblank IRQ disable follows that
1738 * after a timeout with no further vblank waits scheduled).
1739 */
1740int drm_wait_vblank(struct drm_device *dev, void *data,
1741		    struct drm_file *file_priv)
1742{
1743	struct drm_vblank_crtc *vblank;
1744	union drm_wait_vblank *vblwait = data;
1745	int ret;
1746	unsigned int flags, seq, pipe, high_pipe;
1747
1748	if (!dev->irq_enabled)
1749		return -EINVAL;
1750
1751	if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1752		return -EINVAL;
1753
1754	if (vblwait->request.type &
1755	    ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1756	      _DRM_VBLANK_HIGH_CRTC_MASK)) {
1757		DRM_ERROR("Unsupported type value 0x%x, supported mask 0x%x\n",
1758			  vblwait->request.type,
1759			  (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1760			   _DRM_VBLANK_HIGH_CRTC_MASK));
1761		return -EINVAL;
1762	}
1763
1764	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1765	high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1766	if (high_pipe)
1767		pipe = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1768	else
1769		pipe = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1770	if (pipe >= dev->num_crtcs)
1771		return -EINVAL;
1772
1773	vblank = &dev->vblank[pipe];
1774
1775	ret = drm_vblank_get(dev, pipe);
1776	if (ret) {
1777		DRM_DEBUG("failed to acquire vblank counter, %d\n", ret);
1778		return ret;
1779	}
1780	seq = drm_vblank_count(dev, pipe);
1781
1782	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1783	case _DRM_VBLANK_RELATIVE:
1784		vblwait->request.sequence += seq;
1785		vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1786	case _DRM_VBLANK_ABSOLUTE:
1787		break;
1788	default:
1789		ret = -EINVAL;
1790		goto done;
1791	}
1792
1793	if (flags & _DRM_VBLANK_EVENT) {
1794		/* must hold on to the vblank ref until the event fires
1795		 * drm_vblank_put will be called asynchronously
1796		 */
1797		return drm_queue_vblank_event(dev, pipe, vblwait, file_priv);
1798	}
1799
1800	if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1801	    (seq - vblwait->request.sequence) <= (1<<23)) {
1802		vblwait->request.sequence = seq + 1;
1803	}
1804
1805	DRM_DEBUG("waiting on vblank count %d, crtc %u\n",
1806		  vblwait->request.sequence, pipe);
1807	vblank->last_wait = vblwait->request.sequence;
1808	DRM_WAIT_ON(ret, vblank->queue, 3 * HZ,
1809		    (((drm_vblank_count(dev, pipe) -
1810		       vblwait->request.sequence) <= (1 << 23)) ||
1811		     !vblank->enabled ||
1812		     !dev->irq_enabled));
1813
1814	if (ret != -EINTR) {
1815		struct timeval now;
1816
1817		vblwait->reply.sequence = drm_vblank_count_and_time(dev, pipe, &now);
1818		vblwait->reply.tval_sec = now.tv_sec;
1819		vblwait->reply.tval_usec = now.tv_usec;
1820
1821		DRM_DEBUG("returning %d to client\n",
1822			  vblwait->reply.sequence);
1823	} else {
1824		DRM_DEBUG("vblank wait interrupted by signal\n");
1825	}
1826
1827done:
1828	drm_vblank_put(dev, pipe);
1829	return ret;
1830}
1831
1832static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1833{
1834	struct drm_pending_vblank_event *e, *t;
1835	struct timeval now;
1836	unsigned int seq;
1837
1838	assert_spin_locked(&dev->event_lock);
1839
1840	seq = drm_vblank_count_and_time(dev, pipe, &now);
1841
1842	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1843		if (e->pipe != pipe)
1844			continue;
1845		if ((seq - e->event.sequence) > (1<<23))
1846			continue;
1847
1848		DRM_DEBUG("vblank event on %d, current %d\n",
1849			  e->event.sequence, seq);
1850
1851		list_del(&e->base.link);
1852		drm_vblank_put(dev, pipe);
1853		send_vblank_event(dev, e, seq, &now);
1854	}
1855
1856	trace_drm_vblank_event(pipe, seq);
1857}
1858
1859/**
1860 * drm_handle_vblank - handle a vblank event
1861 * @dev: DRM device
1862 * @pipe: index of CRTC where this event occurred
1863 *
1864 * Drivers should call this routine in their vblank interrupt handlers to
1865 * update the vblank counter and send any signals that may be pending.
1866 *
1867 * This is the legacy version of drm_crtc_handle_vblank().
1868 */
1869bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1870{
1871	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1872	unsigned long irqflags;
1873
1874	if (WARN_ON_ONCE(!dev->num_crtcs))
1875		return false;
1876
1877	if (WARN_ON(pipe >= dev->num_crtcs))
1878		return false;
1879
1880	spin_lock_irqsave(&dev->event_lock, irqflags);
1881
1882	/* Need timestamp lock to prevent concurrent execution with
1883	 * vblank enable/disable, as this would cause inconsistent
1884	 * or corrupted timestamps and vblank counts.
1885	 */
1886	spin_lock(&dev->vblank_time_lock);
1887
1888	/* Vblank irq handling disabled. Nothing to do. */
1889	if (!vblank->enabled) {
1890		spin_unlock(&dev->vblank_time_lock);
1891		spin_unlock_irqrestore(&dev->event_lock, irqflags);
1892		return false;
1893	}
1894
1895	drm_update_vblank_count(dev, pipe, DRM_CALLED_FROM_VBLIRQ);
1896
1897	spin_unlock(&dev->vblank_time_lock);
1898
1899	wake_up(&vblank->queue);
1900	drm_handle_vblank_events(dev, pipe);
1901
1902	spin_unlock_irqrestore(&dev->event_lock, irqflags);
1903
1904	return true;
1905}
1906EXPORT_SYMBOL(drm_handle_vblank);
1907
1908/**
1909 * drm_crtc_handle_vblank - handle a vblank event
1910 * @crtc: where this event occurred
1911 *
1912 * Drivers should call this routine in their vblank interrupt handlers to
1913 * update the vblank counter and send any signals that may be pending.
1914 *
1915 * This is the native KMS version of drm_handle_vblank().
1916 *
1917 * Returns:
1918 * True if the event was successfully handled, false on failure.
1919 */
1920bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1921{
1922	return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1923}
1924EXPORT_SYMBOL(drm_crtc_handle_vblank);
1925
1926/**
1927 * drm_vblank_no_hw_counter - "No hw counter" implementation of .get_vblank_counter()
1928 * @dev: DRM device
1929 * @pipe: CRTC for which to read the counter
1930 *
1931 * Drivers can plug this into the .get_vblank_counter() function if
1932 * there is no useable hardware frame counter available.
1933 *
1934 * Returns:
1935 * 0
1936 */
1937u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
1938{
1939	return 0;
1940}
1941EXPORT_SYMBOL(drm_vblank_no_hw_counter);