1/*
   2 * drm_irq.c IRQ and vblank support
   3 *
   4 * \author Rickard E. (Rik) Faith <faith@valinux.com>
   5 * \author Gareth Hughes <gareth@valinux.com>
   6 *
   7 * Permission is hereby granted, free of charge, to any person obtaining a
   8 * copy of this software and associated documentation files (the "Software"),
   9 * to deal in the Software without restriction, including without limitation
  10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  11 * and/or sell copies of the Software, and to permit persons to whom the
  12 * Software is furnished to do so, subject to the following conditions:
  13 *
  14 * The above copyright notice and this permission notice (including the next
  15 * paragraph) shall be included in all copies or substantial portions of the
  16 * Software.
  17 *
  18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  21 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  24 * OTHER DEALINGS IN THE SOFTWARE.
  25 */
  26
  27#include <linux/export.h>
  28#include <linux/kthread.h>
  29#include <linux/moduleparam.h>
  30
  31#include <drm/drm_crtc.h>
  32#include <drm/drm_drv.h>
  33#include <drm/drm_framebuffer.h>
  34#include <drm/drm_managed.h>
  35#include <drm/drm_modeset_helper_vtables.h>
  36#include <drm/drm_print.h>
  37#include <drm/drm_vblank.h>
  38
  39#include "drm_internal.h"
  40#include "drm_trace.h"
  41
  42/**
  43 * DOC: vblank handling
  44 *
  45 * From the computer's perspective, every time the monitor displays
  46 * a new frame the scanout engine has "scanned out" the display image
  47 * from top to bottom, one row of pixels at a time. The current row
  48 * of pixels is referred to as the current scanline.
  49 *
  50 * In addition to the display's visible area, there's usually a couple of
  51 * extra scanlines which aren't actually displayed on the screen.
  52 * These extra scanlines don't contain image data and are occasionally used
  53 * for features like audio and infoframes. The region made up of these
  54 * scanlines is referred to as the vertical blanking region, or vblank for
  55 * short.
  56 *
  57 * For historical reference, the vertical blanking period was designed to
  58 * give the electron gun (on CRTs) enough time to move back to the top of
  59 * the screen to start scanning out the next frame. Similar for horizontal
  60 * blanking periods. They were designed to give the electron gun enough
  61 * time to move back to the other side of the screen to start scanning the
  62 * next scanline.
  63 *
  64 * ::
  65 *
  66 *
  67 *    physical →   ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
  68 *    top of      |                                        |
  69 *    display     |                                        |
  70 *                |               New frame                |
  71 *                |                                        |
  72 *                |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|
  73 *                |~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~| ← Scanline,
  74 *                |↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓|   updates the
  75 *                |                                        |   frame as it
  76 *                |                                        |   travels down
  77 *                |                                        |   ("scan out")
  78 *                |               Old frame                |
  79 *                |                                        |
  80 *                |                                        |
  81 *                |                                        |
  82 *                |                                        |   physical
  83 *                |                                        |   bottom of
  84 *    vertical    |⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽| ← display
  85 *    blanking    ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  86 *    region   →  ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  87 *                ┆xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx┆
  88 *    start of →   ⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽⎽
  89 *    new frame
  90 *
  91 * "Physical top of display" is the reference point for the high-precision/
  92 * corrected timestamp.
  93 *
  94 * On a lot of display hardware, programming needs to take effect during the
  95 * vertical blanking period so that settings like gamma, the image buffer
  96 * buffer to be scanned out, etc. can safely be changed without showing
  97 * any visual artifacts on the screen. In some unforgiving hardware, some of
  98 * this programming has to both start and end in the same vblank. To help
  99 * with the timing of the hardware programming, an interrupt is usually
 100 * available to notify the driver when it can start the updating of registers.
 101 * The interrupt is in this context named the vblank interrupt.
 102 *
 103 * The vblank interrupt may be fired at different points depending on the
 104 * hardware. Some hardware implementations will fire the interrupt when the
 105 * new frame start, other implementations will fire the interrupt at different
 106 * points in time.
 107 *
 108 * Vertical blanking plays a major role in graphics rendering. To achieve
 109 * tear-free display, users must synchronize page flips and/or rendering to
 110 * vertical blanking. The DRM API offers ioctls to perform page flips
 111 * synchronized to vertical blanking and wait for vertical blanking.
 112 *
 113 * The DRM core handles most of the vertical blanking management logic, which
 114 * involves filtering out spurious interrupts, keeping race-free blanking
 115 * counters, coping with counter wrap-around and resets and keeping use counts.
 116 * It relies on the driver to generate vertical blanking interrupts and
 117 * optionally provide a hardware vertical blanking counter.
 118 *
 119 * Drivers must initialize the vertical blanking handling core with a call to
 120 * drm_vblank_init(). Minimally, a driver needs to implement
 121 * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
 122 * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
 123 * support.
 124 *
 125 * Vertical blanking interrupts can be enabled by the DRM core or by drivers
 126 * themselves (for instance to handle page flipping operations).  The DRM core
 127 * maintains a vertical blanking use count to ensure that the interrupts are not
 128 * disabled while a user still needs them. To increment the use count, drivers
 129 * call drm_crtc_vblank_get() and release the vblank reference again with
 130 * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
 131 * guaranteed to be enabled.
 132 *
 133 * On many hardware disabling the vblank interrupt cannot be done in a race-free
 134 * manner, see &drm_vblank_crtc_config.disable_immediate and
 135 * &drm_driver.max_vblank_count. In that case the vblank core only disables the
 136 * vblanks after a timer has expired, which can be configured through the
 137 * ``vblankoffdelay`` module parameter.
 138 *
 139 * Drivers for hardware without support for vertical-blanking interrupts
 140 * must not call drm_vblank_init(). For such drivers, atomic helpers will
 141 * automatically generate fake vblank events as part of the display update.
 142 * This functionality also can be controlled by the driver by enabling and
 143 * disabling struct drm_crtc_state.no_vblank.
 144 */
 145
 146/* Retry timestamp calculation up to 3 times to satisfy
 147 * drm_timestamp_precision before giving up.
 148 */
 149#define DRM_TIMESTAMP_MAXRETRIES 3
 150
 151/* Threshold in nanoseconds for detection of redundant
 152 * vblank irq in drm_handle_vblank(). 1 msec should be ok.
 153 */
 154#define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
 155
 156static bool
 157drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 158			  ktime_t *tvblank, bool in_vblank_irq);
 159
 160static unsigned int drm_timestamp_precision = 20;  /* Default to 20 usecs. */
 161
 162static int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
 163
 164module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
 165module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
 166MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
 167MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
 168
 169static struct drm_vblank_crtc *
 170drm_vblank_crtc(struct drm_device *dev, unsigned int pipe)
 171{
 172	return &dev->vblank[pipe];
 173}
 174
 175struct drm_vblank_crtc *
 176drm_crtc_vblank_crtc(struct drm_crtc *crtc)
 177{
 178	return drm_vblank_crtc(crtc->dev, drm_crtc_index(crtc));
 179}
 180EXPORT_SYMBOL(drm_crtc_vblank_crtc);
 181
 182static void store_vblank(struct drm_device *dev, unsigned int pipe,
 183			 u32 vblank_count_inc,
 184			 ktime_t t_vblank, u32 last)
 185{
 186	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 187
 188	assert_spin_locked(&dev->vblank_time_lock);
 189
 190	vblank->last = last;
 191
 192	write_seqlock(&vblank->seqlock);
 193	vblank->time = t_vblank;
 194	atomic64_add(vblank_count_inc, &vblank->count);
 195	write_sequnlock(&vblank->seqlock);
 196}
 197
 198static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
 199{
 200	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 201
 202	return vblank->max_vblank_count ?: dev->max_vblank_count;
 203}
 204
 205/*
 206 * "No hw counter" fallback implementation of .get_vblank_counter() hook,
 207 * if there is no usable hardware frame counter available.
 208 */
 209static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
 210{
 211	drm_WARN_ON_ONCE(dev, drm_max_vblank_count(dev, pipe) != 0);
 212	return 0;
 213}
 214
 215static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 216{
 217	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 218		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 219
 220		if (drm_WARN_ON(dev, !crtc))
 221			return 0;
 222
 223		if (crtc->funcs->get_vblank_counter)
 224			return crtc->funcs->get_vblank_counter(crtc);
 225	}
 226
 227	return drm_vblank_no_hw_counter(dev, pipe);
 228}
 229
 230/*
 231 * Reset the stored timestamp for the current vblank count to correspond
 232 * to the last vblank occurred.
 233 *
 234 * Only to be called from drm_crtc_vblank_on().
 235 *
 236 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
 237 * device vblank fields.
 238 */
 239static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
 240{
 241	u32 cur_vblank;
 242	bool rc;
 243	ktime_t t_vblank;
 244	int count = DRM_TIMESTAMP_MAXRETRIES;
 245
 246	spin_lock(&dev->vblank_time_lock);
 247
 248	/*
 249	 * sample the current counter to avoid random jumps
 250	 * when drm_vblank_enable() applies the diff
 251	 */
 252	do {
 253		cur_vblank = __get_vblank_counter(dev, pipe);
 254		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
 255	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
 256
 257	/*
 258	 * Only reinitialize corresponding vblank timestamp if high-precision query
 259	 * available and didn't fail. Otherwise reinitialize delayed at next vblank
 260	 * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
 261	 */
 262	if (!rc)
 263		t_vblank = 0;
 264
 265	/*
 266	 * +1 to make sure user will never see the same
 267	 * vblank counter value before and after a modeset
 268	 */
 269	store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
 270
 271	spin_unlock(&dev->vblank_time_lock);
 272}
 273
 274/*
 275 * Call back into the driver to update the appropriate vblank counter
 276 * (specified by @pipe).  Deal with wraparound, if it occurred, and
 277 * update the last read value so we can deal with wraparound on the next
 278 * call if necessary.
 279 *
 280 * Only necessary when going from off->on, to account for frames we
 281 * didn't get an interrupt for.
 282 *
 283 * Note: caller must hold &drm_device.vbl_lock since this reads & writes
 284 * device vblank fields.
 285 */
 286static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
 287				    bool in_vblank_irq)
 288{
 289	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 290	u32 cur_vblank, diff;
 291	bool rc;
 292	ktime_t t_vblank;
 293	int count = DRM_TIMESTAMP_MAXRETRIES;
 294	int framedur_ns = vblank->framedur_ns;
 295	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
 296
 297	/*
 298	 * Interrupts were disabled prior to this call, so deal with counter
 299	 * wrap if needed.
 300	 * NOTE!  It's possible we lost a full dev->max_vblank_count + 1 events
 301	 * here if the register is small or we had vblank interrupts off for
 302	 * a long time.
 303	 *
 304	 * We repeat the hardware vblank counter & timestamp query until
 305	 * we get consistent results. This to prevent races between gpu
 306	 * updating its hardware counter while we are retrieving the
 307	 * corresponding vblank timestamp.
 308	 */
 309	do {
 310		cur_vblank = __get_vblank_counter(dev, pipe);
 311		rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
 312	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
 313
 314	if (max_vblank_count) {
 315		/* trust the hw counter when it's around */
 316		diff = (cur_vblank - vblank->last) & max_vblank_count;
 317	} else if (rc && framedur_ns) {
 318		u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
 319
 320		/*
 321		 * Figure out how many vblanks we've missed based
 322		 * on the difference in the timestamps and the
 323		 * frame/field duration.
 324		 */
 325
 326		drm_dbg_vbl(dev, "crtc %u: Calculating number of vblanks."
 327			    " diff_ns = %lld, framedur_ns = %d)\n",
 328			    pipe, (long long)diff_ns, framedur_ns);
 329
 330		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
 331
 332		if (diff == 0 && in_vblank_irq)
 333			drm_dbg_vbl(dev, "crtc %u: Redundant vblirq ignored\n",
 334				    pipe);
 335	} else {
 336		/* some kind of default for drivers w/o accurate vbl timestamping */
 337		diff = in_vblank_irq ? 1 : 0;
 338	}
 339
 340	/*
 341	 * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
 342	 * interval? If so then vblank irqs keep running and it will likely
 343	 * happen that the hardware vblank counter is not trustworthy as it
 344	 * might reset at some point in that interval and vblank timestamps
 345	 * are not trustworthy either in that interval. Iow. this can result
 346	 * in a bogus diff >> 1 which must be avoided as it would cause
 347	 * random large forward jumps of the software vblank counter.
 348	 */
 349	if (diff > 1 && (vblank->inmodeset & 0x2)) {
 350		drm_dbg_vbl(dev,
 351			    "clamping vblank bump to 1 on crtc %u: diffr=%u"
 352			    " due to pre-modeset.\n", pipe, diff);
 353		diff = 1;
 354	}
 355
 356	drm_dbg_vbl(dev, "updating vblank count on crtc %u:"
 357		    " current=%llu, diff=%u, hw=%u hw_last=%u\n",
 358		    pipe, (unsigned long long)atomic64_read(&vblank->count),
 359		    diff, cur_vblank, vblank->last);
 360
 361	if (diff == 0) {
 362		drm_WARN_ON_ONCE(dev, cur_vblank != vblank->last);
 363		return;
 364	}
 365
 366	/*
 367	 * Only reinitialize corresponding vblank timestamp if high-precision query
 368	 * available and didn't fail, or we were called from the vblank interrupt.
 369	 * Otherwise reinitialize delayed at next vblank interrupt and assign 0
 370	 * for now, to mark the vblanktimestamp as invalid.
 371	 */
 372	if (!rc && !in_vblank_irq)
 373		t_vblank = 0;
 374
 375	store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
 376}
 377
 378u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
 379{
 380	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 381	u64 count;
 382
 383	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
 384		return 0;
 385
 386	count = atomic64_read(&vblank->count);
 387
 388	/*
 389	 * This read barrier corresponds to the implicit write barrier of the
 390	 * write seqlock in store_vblank(). Note that this is the only place
 391	 * where we need an explicit barrier, since all other access goes
 392	 * through drm_vblank_count_and_time(), which already has the required
 393	 * read barrier curtesy of the read seqlock.
 394	 */
 395	smp_rmb();
 396
 397	return count;
 398}
 399
 400/**
 401 * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
 402 * @crtc: which counter to retrieve
 403 *
 404 * This function is similar to drm_crtc_vblank_count() but this function
 405 * interpolates to handle a race with vblank interrupts using the high precision
 406 * timestamping support.
 407 *
 408 * This is mostly useful for hardware that can obtain the scanout position, but
 409 * doesn't have a hardware frame counter.
 410 */
 411u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
 412{
 413	struct drm_device *dev = crtc->dev;
 414	unsigned int pipe = drm_crtc_index(crtc);
 415	u64 vblank;
 416	unsigned long flags;
 417
 418	drm_WARN_ONCE(dev, drm_debug_enabled(DRM_UT_VBL) &&
 419		      !crtc->funcs->get_vblank_timestamp,
 420		      "This function requires support for accurate vblank timestamps.");
 421
 422	spin_lock_irqsave(&dev->vblank_time_lock, flags);
 423
 424	drm_update_vblank_count(dev, pipe, false);
 425	vblank = drm_vblank_count(dev, pipe);
 426
 427	spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
 428
 429	return vblank;
 430}
 431EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
 432
 433static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
 434{
 435	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
 436		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 437
 438		if (drm_WARN_ON(dev, !crtc))
 439			return;
 440
 441		if (crtc->funcs->disable_vblank)
 442			crtc->funcs->disable_vblank(crtc);
 443	}
 444}
 445
 446/*
 447 * Disable vblank irq's on crtc, make sure that last vblank count
 448 * of hardware and corresponding consistent software vblank counter
 449 * are preserved, even if there are any spurious vblank irq's after
 450 * disable.
 451 */
 452void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
 453{
 454	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 455	unsigned long irqflags;
 456
 457	assert_spin_locked(&dev->vbl_lock);
 458
 459	/* Prevent vblank irq processing while disabling vblank irqs,
 460	 * so no updates of timestamps or count can happen after we've
 461	 * disabled. Needed to prevent races in case of delayed irq's.
 462	 */
 463	spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
 464
 465	/*
 466	 * Update vblank count and disable vblank interrupts only if the
 467	 * interrupts were enabled. This avoids calling the ->disable_vblank()
 468	 * operation in atomic context with the hardware potentially runtime
 469	 * suspended.
 470	 */
 471	if (!vblank->enabled)
 472		goto out;
 473
 474	/*
 475	 * Update the count and timestamp to maintain the
 476	 * appearance that the counter has been ticking all along until
 477	 * this time. This makes the count account for the entire time
 478	 * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
 479	 */
 480	drm_update_vblank_count(dev, pipe, false);
 481	__disable_vblank(dev, pipe);
 482	vblank->enabled = false;
 483
 484out:
 485	spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
 486}
 487
 488static void vblank_disable_fn(struct timer_list *t)
 489{
 490	struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
 491	struct drm_device *dev = vblank->dev;
 492	unsigned int pipe = vblank->pipe;
 493	unsigned long irqflags;
 494
 495	spin_lock_irqsave(&dev->vbl_lock, irqflags);
 496	if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
 497		drm_dbg_core(dev, "disabling vblank on crtc %u\n", pipe);
 498		drm_vblank_disable_and_save(dev, pipe);
 499	}
 500	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 501}
 502
 503static void drm_vblank_init_release(struct drm_device *dev, void *ptr)
 504{
 505	struct drm_vblank_crtc *vblank = ptr;
 506
 507	drm_WARN_ON(dev, READ_ONCE(vblank->enabled) &&
 508		    drm_core_check_feature(dev, DRIVER_MODESET));
 509
 510	drm_vblank_destroy_worker(vblank);
 511	del_timer_sync(&vblank->disable_timer);
 512}
 513
 514/**
 515 * drm_vblank_init - initialize vblank support
 516 * @dev: DRM device
 517 * @num_crtcs: number of CRTCs supported by @dev
 518 *
 519 * This function initializes vblank support for @num_crtcs display pipelines.
 520 * Cleanup is handled automatically through a cleanup function added with
 521 * drmm_add_action_or_reset().
 522 *
 523 * Returns:
 524 * Zero on success or a negative error code on failure.
 525 */
 526int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
 527{
 528	int ret;
 529	unsigned int i;
 530
 531	spin_lock_init(&dev->vbl_lock);
 532	spin_lock_init(&dev->vblank_time_lock);
 533
 534	dev->vblank = drmm_kcalloc(dev, num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
 535	if (!dev->vblank)
 536		return -ENOMEM;
 537
 538	dev->num_crtcs = num_crtcs;
 539
 540	for (i = 0; i < num_crtcs; i++) {
 541		struct drm_vblank_crtc *vblank = &dev->vblank[i];
 542
 543		vblank->dev = dev;
 544		vblank->pipe = i;
 545		init_waitqueue_head(&vblank->queue);
 546		timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
 547		seqlock_init(&vblank->seqlock);
 548
 549		ret = drmm_add_action_or_reset(dev, drm_vblank_init_release,
 550					       vblank);
 551		if (ret)
 552			return ret;
 553
 554		ret = drm_vblank_worker_init(vblank);
 555		if (ret)
 556			return ret;
 557	}
 558
 559	return 0;
 560}
 561EXPORT_SYMBOL(drm_vblank_init);
 562
 563/**
 564 * drm_dev_has_vblank - test if vblanking has been initialized for
 565 *                      a device
 566 * @dev: the device
 567 *
 568 * Drivers may call this function to test if vblank support is
 569 * initialized for a device. For most hardware this means that vblanking
 570 * can also be enabled.
 571 *
 572 * Atomic helpers use this function to initialize
 573 * &drm_crtc_state.no_vblank. See also drm_atomic_helper_check_modeset().
 574 *
 575 * Returns:
 576 * True if vblanking has been initialized for the given device, false
 577 * otherwise.
 578 */
 579bool drm_dev_has_vblank(const struct drm_device *dev)
 580{
 581	return dev->num_crtcs != 0;
 582}
 583EXPORT_SYMBOL(drm_dev_has_vblank);
 584
 585/**
 586 * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
 587 * @crtc: which CRTC's vblank waitqueue to retrieve
 588 *
 589 * This function returns a pointer to the vblank waitqueue for the CRTC.
 590 * Drivers can use this to implement vblank waits using wait_event() and related
 591 * functions.
 592 */
 593wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
 594{
 595	return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
 596}
 597EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
 598
 599
 600/**
 601 * drm_calc_timestamping_constants - calculate vblank timestamp constants
 602 * @crtc: drm_crtc whose timestamp constants should be updated.
 603 * @mode: display mode containing the scanout timings
 604 *
 605 * Calculate and store various constants which are later needed by vblank and
 606 * swap-completion timestamping, e.g, by
 607 * drm_crtc_vblank_helper_get_vblank_timestamp(). They are derived from
 608 * CRTC's true scanout timing, so they take things like panel scaling or
 609 * other adjustments into account.
 610 */
 611void drm_calc_timestamping_constants(struct drm_crtc *crtc,
 612				     const struct drm_display_mode *mode)
 613{
 614	struct drm_device *dev = crtc->dev;
 615	unsigned int pipe = drm_crtc_index(crtc);
 616	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 617	int linedur_ns = 0, framedur_ns = 0;
 618	int dotclock = mode->crtc_clock;
 619
 620	if (!drm_dev_has_vblank(dev))
 621		return;
 622
 623	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
 624		return;
 625
 626	/* Valid dotclock? */
 627	if (dotclock > 0) {
 628		int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
 629
 630		/*
 631		 * Convert scanline length in pixels and video
 632		 * dot clock to line duration and frame duration
 633		 * in nanoseconds:
 634		 */
 635		linedur_ns  = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
 636		framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
 637
 638		/*
 639		 * Fields of interlaced scanout modes are only half a frame duration.
 640		 */
 641		if (mode->flags & DRM_MODE_FLAG_INTERLACE)
 642			framedur_ns /= 2;
 643	} else {
 644		drm_err(dev, "crtc %u: Can't calculate constants, dotclock = 0!\n",
 645			crtc->base.id);
 646	}
 647
 648	vblank->linedur_ns  = linedur_ns;
 649	vblank->framedur_ns = framedur_ns;
 650	drm_mode_copy(&vblank->hwmode, mode);
 651
 652	drm_dbg_core(dev,
 653		     "crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
 654		     crtc->base.id, mode->crtc_htotal,
 655		     mode->crtc_vtotal, mode->crtc_vdisplay);
 656	drm_dbg_core(dev, "crtc %u: clock %d kHz framedur %d linedur %d\n",
 657		     crtc->base.id, dotclock, framedur_ns, linedur_ns);
 658}
 659EXPORT_SYMBOL(drm_calc_timestamping_constants);
 660
 661/**
 662 * drm_crtc_vblank_helper_get_vblank_timestamp_internal - precise vblank
 663 *                                                        timestamp helper
 664 * @crtc: CRTC whose vblank timestamp to retrieve
 665 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
 666 *             On return contains true maximum error of timestamp
 667 * @vblank_time: Pointer to time which should receive the timestamp
 668 * @in_vblank_irq:
 669 *     True when called from drm_crtc_handle_vblank().  Some drivers
 670 *     need to apply some workarounds for gpu-specific vblank irq quirks
 671 *     if flag is set.
 672 * @get_scanout_position:
 673 *     Callback function to retrieve the scanout position. See
 674 *     @struct drm_crtc_helper_funcs.get_scanout_position.
 675 *
 676 * Implements calculation of exact vblank timestamps from given drm_display_mode
 677 * timings and current video scanout position of a CRTC.
 678 *
 679 * The current implementation only handles standard video modes. For double scan
 680 * and interlaced modes the driver is supposed to adjust the hardware mode
 681 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
 682 * match the scanout position reported.
 683 *
 684 * Note that atomic drivers must call drm_calc_timestamping_constants() before
 685 * enabling a CRTC. The atomic helpers already take care of that in
 686 * drm_atomic_helper_calc_timestamping_constants().
 687 *
 688 * Returns:
 
 689 * Returns true on success, and false on failure, i.e. when no accurate
 690 * timestamp could be acquired.
 691 */
 692bool
 693drm_crtc_vblank_helper_get_vblank_timestamp_internal(
 694	struct drm_crtc *crtc, int *max_error, ktime_t *vblank_time,
 695	bool in_vblank_irq,
 696	drm_vblank_get_scanout_position_func get_scanout_position)
 697{
 698	struct drm_device *dev = crtc->dev;
 699	unsigned int pipe = crtc->index;
 700	struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 701	struct timespec64 ts_etime, ts_vblank_time;
 702	ktime_t stime, etime;
 703	bool vbl_status;
 704	const struct drm_display_mode *mode;
 705	int vpos, hpos, i;
 706	int delta_ns, duration_ns;
 707
 708	if (pipe >= dev->num_crtcs) {
 709		drm_err(dev, "Invalid crtc %u\n", pipe);
 710		return false;
 711	}
 712
 713	/* Scanout position query not supported? Should not happen. */
 714	if (!get_scanout_position) {
 715		drm_err(dev, "Called from CRTC w/o get_scanout_position()!?\n");
 716		return false;
 717	}
 718
 719	if (drm_drv_uses_atomic_modeset(dev))
 720		mode = &vblank->hwmode;
 721	else
 722		mode = &crtc->hwmode;
 723
 724	/* If mode timing undefined, just return as no-op:
 725	 * Happens during initial modesetting of a crtc.
 726	 */
 727	if (mode->crtc_clock == 0) {
 728		drm_dbg_core(dev, "crtc %u: Noop due to uninitialized mode.\n",
 729			     pipe);
 730		drm_WARN_ON_ONCE(dev, drm_drv_uses_atomic_modeset(dev));
 731		return false;
 732	}
 733
 734	/* Get current scanout position with system timestamp.
 735	 * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
 736	 * if single query takes longer than max_error nanoseconds.
 737	 *
 738	 * This guarantees a tight bound on maximum error if
 739	 * code gets preempted or delayed for some reason.
 740	 */
 741	for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
 742		/*
 743		 * Get vertical and horizontal scanout position vpos, hpos,
 744		 * and bounding timestamps stime, etime, pre/post query.
 745		 */
 746		vbl_status = get_scanout_position(crtc, in_vblank_irq,
 747						  &vpos, &hpos,
 748						  &stime, &etime,
 749						  mode);
 750
 751		/* Return as no-op if scanout query unsupported or failed. */
 752		if (!vbl_status) {
 753			drm_dbg_core(dev,
 754				     "crtc %u : scanoutpos query failed.\n",
 755				     pipe);
 756			return false;
 757		}
 758
 759		/* Compute uncertainty in timestamp of scanout position query. */
 760		duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
 761
 762		/* Accept result with <  max_error nsecs timing uncertainty. */
 763		if (duration_ns <= *max_error)
 764			break;
 765	}
 766
 767	/* Noisy system timing? */
 768	if (i == DRM_TIMESTAMP_MAXRETRIES) {
 769		drm_dbg_core(dev,
 770			     "crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
 771			     pipe, duration_ns / 1000, *max_error / 1000, i);
 772	}
 773
 774	/* Return upper bound of timestamp precision error. */
 775	*max_error = duration_ns;
 776
 777	/* Convert scanout position into elapsed time at raw_time query
 778	 * since start of scanout at first display scanline. delta_ns
 779	 * can be negative if start of scanout hasn't happened yet.
 780	 */
 781	delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
 782			   mode->crtc_clock);
 783
 784	/* Subtract time delta from raw timestamp to get final
 785	 * vblank_time timestamp for end of vblank.
 786	 */
 787	*vblank_time = ktime_sub_ns(etime, delta_ns);
 788
 789	if (!drm_debug_enabled(DRM_UT_VBL))
 790		return true;
 791
 792	ts_etime = ktime_to_timespec64(etime);
 793	ts_vblank_time = ktime_to_timespec64(*vblank_time);
 794
 795	drm_dbg_vbl(dev,
 796		    "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
 797		    pipe, hpos, vpos,
 798		    (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
 799		    (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
 800		    duration_ns / 1000, i);
 801
 802	return true;
 803}
 804EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp_internal);
 805
 806/**
 807 * drm_crtc_vblank_helper_get_vblank_timestamp - precise vblank timestamp
 808 *                                               helper
 809 * @crtc: CRTC whose vblank timestamp to retrieve
 810 * @max_error: Desired maximum allowable error in timestamps (nanosecs)
 811 *             On return contains true maximum error of timestamp
 812 * @vblank_time: Pointer to time which should receive the timestamp
 813 * @in_vblank_irq:
 814 *     True when called from drm_crtc_handle_vblank().  Some drivers
 815 *     need to apply some workarounds for gpu-specific vblank irq quirks
 816 *     if flag is set.
 817 *
 818 * Implements calculation of exact vblank timestamps from given drm_display_mode
 819 * timings and current video scanout position of a CRTC. This can be directly
 820 * used as the &drm_crtc_funcs.get_vblank_timestamp implementation of a kms
 821 * driver if &drm_crtc_helper_funcs.get_scanout_position is implemented.
 822 *
 823 * The current implementation only handles standard video modes. For double scan
 824 * and interlaced modes the driver is supposed to adjust the hardware mode
 825 * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
 826 * match the scanout position reported.
 827 *
 828 * Note that atomic drivers must call drm_calc_timestamping_constants() before
 829 * enabling a CRTC. The atomic helpers already take care of that in
 830 * drm_atomic_helper_calc_timestamping_constants().
 831 *
 832 * Returns:
 
 833 * Returns true on success, and false on failure, i.e. when no accurate
 834 * timestamp could be acquired.
 835 */
 836bool drm_crtc_vblank_helper_get_vblank_timestamp(struct drm_crtc *crtc,
 837						 int *max_error,
 838						 ktime_t *vblank_time,
 839						 bool in_vblank_irq)
 840{
 841	return drm_crtc_vblank_helper_get_vblank_timestamp_internal(
 842		crtc, max_error, vblank_time, in_vblank_irq,
 843		crtc->helper_private->get_scanout_position);
 844}
 845EXPORT_SYMBOL(drm_crtc_vblank_helper_get_vblank_timestamp);
 846
 847/**
 848 * drm_crtc_get_last_vbltimestamp - retrieve raw timestamp for the most
 849 *                                  recent vblank interval
 850 * @crtc: CRTC whose vblank timestamp to retrieve
 851 * @tvblank: Pointer to target time which should receive the timestamp
 852 * @in_vblank_irq:
 853 *     True when called from drm_crtc_handle_vblank().  Some drivers
 854 *     need to apply some workarounds for gpu-specific vblank irq quirks
 855 *     if flag is set.
 856 *
 857 * Fetches the system timestamp corresponding to the time of the most recent
 858 * vblank interval on specified CRTC. May call into kms-driver to
 859 * compute the timestamp with a high-precision GPU specific method.
 860 *
 861 * Returns zero if timestamp originates from uncorrected do_gettimeofday()
 862 * call, i.e., it isn't very precisely locked to the true vblank.
 863 *
 864 * Returns:
 865 * True if timestamp is considered to be very precise, false otherwise.
 866 */
 867static bool
 868drm_crtc_get_last_vbltimestamp(struct drm_crtc *crtc, ktime_t *tvblank,
 869			       bool in_vblank_irq)
 870{
 871	bool ret = false;
 872
 873	/* Define requested maximum error on timestamps (nanoseconds). */
 874	int max_error = (int) drm_timestamp_precision * 1000;
 875
 876	/* Query driver if possible and precision timestamping enabled. */
 877	if (crtc && crtc->funcs->get_vblank_timestamp && max_error > 0) {
 878		ret = crtc->funcs->get_vblank_timestamp(crtc, &max_error,
 879							tvblank, in_vblank_irq);
 880	}
 881
 882	/* GPU high precision timestamp query unsupported or failed.
 883	 * Return current monotonic/gettimeofday timestamp as best estimate.
 884	 */
 885	if (!ret)
 886		*tvblank = ktime_get();
 887
 888	return ret;
 889}
 890
 891static bool
 892drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
 893			  ktime_t *tvblank, bool in_vblank_irq)
 894{
 895	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
 896
 897	return drm_crtc_get_last_vbltimestamp(crtc, tvblank, in_vblank_irq);
 898}
 899
 900/**
 901 * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
 902 * @crtc: which counter to retrieve
 903 *
 904 * Fetches the "cooked" vblank count value that represents the number of
 905 * vblank events since the system was booted, including lost events due to
 906 * modesetting activity. Note that this timer isn't correct against a racing
 907 * vblank interrupt (since it only reports the software vblank counter), see
 908 * drm_crtc_accurate_vblank_count() for such use-cases.
 909 *
 910 * Note that for a given vblank counter value drm_crtc_handle_vblank()
 911 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
 912 * provide a barrier: Any writes done before calling
 913 * drm_crtc_handle_vblank() will be visible to callers of the later
 914 * functions, if the vblank count is the same or a later one.
 915 *
 916 * See also &drm_vblank_crtc.count.
 917 *
 918 * Returns:
 919 * The software vblank counter.
 920 */
 921u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
 922{
 923	return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
 924}
 925EXPORT_SYMBOL(drm_crtc_vblank_count);
 926
 927/**
 928 * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
 929 *     system timestamp corresponding to that vblank counter value.
 930 * @dev: DRM device
 931 * @pipe: index of CRTC whose counter to retrieve
 932 * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
 933 *
 934 * Fetches the "cooked" vblank count value that represents the number of
 935 * vblank events since the system was booted, including lost events due to
 936 * modesetting activity. Returns corresponding system timestamp of the time
 937 * of the vblank interval that corresponds to the current vblank counter value.
 938 *
 939 * This is the legacy version of drm_crtc_vblank_count_and_time().
 940 */
 941static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
 942				     ktime_t *vblanktime)
 943{
 944	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
 945	u64 vblank_count;
 946	unsigned int seq;
 947
 948	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs)) {
 949		*vblanktime = 0;
 950		return 0;
 951	}
 952
 953	do {
 954		seq = read_seqbegin(&vblank->seqlock);
 955		vblank_count = atomic64_read(&vblank->count);
 956		*vblanktime = vblank->time;
 957	} while (read_seqretry(&vblank->seqlock, seq));
 958
 959	return vblank_count;
 960}
 961
 962/**
 963 * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
 964 *     and the system timestamp corresponding to that vblank counter value
 965 * @crtc: which counter to retrieve
 966 * @vblanktime: Pointer to time to receive the vblank timestamp.
 967 *
 968 * Fetches the "cooked" vblank count value that represents the number of
 969 * vblank events since the system was booted, including lost events due to
 970 * modesetting activity. Returns corresponding system timestamp of the time
 971 * of the vblank interval that corresponds to the current vblank counter value.
 972 *
 973 * Note that for a given vblank counter value drm_crtc_handle_vblank()
 974 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
 975 * provide a barrier: Any writes done before calling
 976 * drm_crtc_handle_vblank() will be visible to callers of the later
 977 * functions, if the vblank count is the same or a later one.
 978 *
 979 * See also &drm_vblank_crtc.count.
 980 */
 981u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
 982				   ktime_t *vblanktime)
 983{
 984	return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
 985					 vblanktime);
 986}
 987EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
 988
 989/**
 990 * drm_crtc_next_vblank_start - calculate the time of the next vblank
 991 * @crtc: the crtc for which to calculate next vblank time
 992 * @vblanktime: pointer to time to receive the next vblank timestamp.
 993 *
 994 * Calculate the expected time of the start of the next vblank period,
 995 * based on time of previous vblank and frame duration
 996 */
 997int drm_crtc_next_vblank_start(struct drm_crtc *crtc, ktime_t *vblanktime)
 998{
 
 999	struct drm_vblank_crtc *vblank;
1000	struct drm_display_mode *mode;
1001	u64 vblank_start;
1002
1003	if (!drm_dev_has_vblank(crtc->dev))
1004		return -EINVAL;
1005
1006	vblank = drm_crtc_vblank_crtc(crtc);
1007	mode = &vblank->hwmode;
1008
1009	if (!vblank->framedur_ns || !vblank->linedur_ns)
1010		return -EINVAL;
1011
1012	if (!drm_crtc_get_last_vbltimestamp(crtc, vblanktime, false))
1013		return -EINVAL;
1014
1015	vblank_start = DIV_ROUND_DOWN_ULL(
1016			(u64)vblank->framedur_ns * mode->crtc_vblank_start,
1017			mode->crtc_vtotal);
1018	*vblanktime  = ktime_add(*vblanktime, ns_to_ktime(vblank_start));
1019
1020	return 0;
1021}
1022EXPORT_SYMBOL(drm_crtc_next_vblank_start);
1023
1024static void send_vblank_event(struct drm_device *dev,
1025		struct drm_pending_vblank_event *e,
1026		u64 seq, ktime_t now)
1027{
1028	struct timespec64 tv;
1029
1030	switch (e->event.base.type) {
1031	case DRM_EVENT_VBLANK:
1032	case DRM_EVENT_FLIP_COMPLETE:
1033		tv = ktime_to_timespec64(now);
1034		e->event.vbl.sequence = seq;
1035		/*
1036		 * e->event is a user space structure, with hardcoded unsigned
1037		 * 32-bit seconds/microseconds. This is safe as we always use
1038		 * monotonic timestamps since linux-4.15
1039		 */
1040		e->event.vbl.tv_sec = tv.tv_sec;
1041		e->event.vbl.tv_usec = tv.tv_nsec / 1000;
1042		break;
1043	case DRM_EVENT_CRTC_SEQUENCE:
1044		if (seq)
1045			e->event.seq.sequence = seq;
1046		e->event.seq.time_ns = ktime_to_ns(now);
1047		break;
1048	}
1049	trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
1050	/*
1051	 * Use the same timestamp for any associated fence signal to avoid
1052	 * mismatch in timestamps for vsync & fence events triggered by the
1053	 * same HW event. Frameworks like SurfaceFlinger in Android expects the
1054	 * retire-fence timestamp to match exactly with HW vsync as it uses it
1055	 * for its software vsync modeling.
1056	 */
1057	drm_send_event_timestamp_locked(dev, &e->base, now);
1058}
1059
1060/**
1061 * drm_crtc_arm_vblank_event - arm vblank event after pageflip
1062 * @crtc: the source CRTC of the vblank event
1063 * @e: the event to send
1064 *
1065 * A lot of drivers need to generate vblank events for the very next vblank
1066 * interrupt. For example when the page flip interrupt happens when the page
1067 * flip gets armed, but not when it actually executes within the next vblank
1068 * period. This helper function implements exactly the required vblank arming
1069 * behaviour.
1070 *
1071 * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
1072 * atomic commit must ensure that the next vblank happens at exactly the same
1073 * time as the atomic commit is committed to the hardware. This function itself
1074 * does **not** protect against the next vblank interrupt racing with either this
1075 * function call or the atomic commit operation. A possible sequence could be:
1076 *
1077 * 1. Driver commits new hardware state into vblank-synchronized registers.
1078 * 2. A vblank happens, committing the hardware state. Also the corresponding
1079 *    vblank interrupt is fired off and fully processed by the interrupt
1080 *    handler.
1081 * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
1082 * 4. The event is only send out for the next vblank, which is wrong.
1083 *
1084 * An equivalent race can happen when the driver calls
1085 * drm_crtc_arm_vblank_event() before writing out the new hardware state.
1086 *
1087 * The only way to make this work safely is to prevent the vblank from firing
1088 * (and the hardware from committing anything else) until the entire atomic
1089 * commit sequence has run to completion. If the hardware does not have such a
1090 * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
1091 * Instead drivers need to manually send out the event from their interrupt
1092 * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
1093 * possible race with the hardware committing the atomic update.
1094 *
1095 * Caller must hold a vblank reference for the event @e acquired by a
1096 * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
1097 */
1098void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
1099			       struct drm_pending_vblank_event *e)
1100{
1101	struct drm_device *dev = crtc->dev;
1102	unsigned int pipe = drm_crtc_index(crtc);
1103
1104	assert_spin_locked(&dev->event_lock);
1105
1106	e->pipe = pipe;
1107	e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
1108	list_add_tail(&e->base.link, &dev->vblank_event_list);
1109}
1110EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
1111
1112/**
1113 * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
1114 * @crtc: the source CRTC of the vblank event
1115 * @e: the event to send
1116 *
1117 * Updates sequence # and timestamp on event for the most recently processed
1118 * vblank, and sends it to userspace.  Caller must hold event lock.
1119 *
1120 * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
1121 * situation, especially to send out events for atomic commit operations.
1122 */
1123void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
1124				struct drm_pending_vblank_event *e)
1125{
1126	struct drm_device *dev = crtc->dev;
1127	u64 seq;
1128	unsigned int pipe = drm_crtc_index(crtc);
1129	ktime_t now;
1130
1131	if (drm_dev_has_vblank(dev)) {
1132		seq = drm_vblank_count_and_time(dev, pipe, &now);
1133	} else {
1134		seq = 0;
1135
1136		now = ktime_get();
1137	}
1138	e->pipe = pipe;
1139	send_vblank_event(dev, e, seq, now);
1140}
1141EXPORT_SYMBOL(drm_crtc_send_vblank_event);
1142
1143static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
1144{
1145	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1146		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1147
1148		if (drm_WARN_ON(dev, !crtc))
1149			return 0;
1150
1151		if (crtc->funcs->enable_vblank)
1152			return crtc->funcs->enable_vblank(crtc);
1153	}
1154
1155	return -EINVAL;
1156}
1157
1158static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
1159{
1160	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1161	int ret = 0;
1162
1163	assert_spin_locked(&dev->vbl_lock);
1164
1165	spin_lock(&dev->vblank_time_lock);
1166
1167	if (!vblank->enabled) {
1168		/*
1169		 * Enable vblank irqs under vblank_time_lock protection.
1170		 * All vblank count & timestamp updates are held off
1171		 * until we are done reinitializing master counter and
1172		 * timestamps. Filtercode in drm_handle_vblank() will
1173		 * prevent double-accounting of same vblank interval.
1174		 */
1175		ret = __enable_vblank(dev, pipe);
1176		drm_dbg_core(dev, "enabling vblank on crtc %u, ret: %d\n",
1177			     pipe, ret);
1178		if (ret) {
1179			atomic_dec(&vblank->refcount);
1180		} else {
1181			drm_update_vblank_count(dev, pipe, 0);
1182			/* drm_update_vblank_count() includes a wmb so we just
1183			 * need to ensure that the compiler emits the write
1184			 * to mark the vblank as enabled after the call
1185			 * to drm_update_vblank_count().
1186			 */
1187			WRITE_ONCE(vblank->enabled, true);
1188		}
1189	}
1190
1191	spin_unlock(&dev->vblank_time_lock);
1192
1193	return ret;
1194}
1195
1196int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1197{
1198	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1199	unsigned long irqflags;
1200	int ret = 0;
1201
1202	if (!drm_dev_has_vblank(dev))
1203		return -EINVAL;
1204
1205	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1206		return -EINVAL;
1207
1208	spin_lock_irqsave(&dev->vbl_lock, irqflags);
1209	/* Going from 0->1 means we have to enable interrupts again */
1210	if (atomic_add_return(1, &vblank->refcount) == 1) {
1211		ret = drm_vblank_enable(dev, pipe);
1212	} else {
1213		if (!vblank->enabled) {
1214			atomic_dec(&vblank->refcount);
1215			ret = -EINVAL;
1216		}
1217	}
1218	spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1219
1220	return ret;
1221}
1222
1223/**
1224 * drm_crtc_vblank_get - get a reference count on vblank events
1225 * @crtc: which CRTC to own
1226 *
1227 * Acquire a reference count on vblank events to avoid having them disabled
1228 * while in use.
1229 *
1230 * Returns:
1231 * Zero on success or a negative error code on failure.
1232 */
1233int drm_crtc_vblank_get(struct drm_crtc *crtc)
1234{
1235	return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1236}
1237EXPORT_SYMBOL(drm_crtc_vblank_get);
1238
1239void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1240{
1241	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1242	int vblank_offdelay = vblank->config.offdelay_ms;
1243
1244	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1245		return;
1246
1247	if (drm_WARN_ON(dev, atomic_read(&vblank->refcount) == 0))
1248		return;
1249
1250	/* Last user schedules interrupt disable */
1251	if (atomic_dec_and_test(&vblank->refcount)) {
1252		if (!vblank_offdelay)
1253			return;
1254		else if (vblank_offdelay < 0)
1255			vblank_disable_fn(&vblank->disable_timer);
1256		else if (!vblank->config.disable_immediate)
1257			mod_timer(&vblank->disable_timer,
1258				  jiffies + ((vblank_offdelay * HZ) / 1000));
1259	}
1260}
1261
1262/**
1263 * drm_crtc_vblank_put - give up ownership of vblank events
1264 * @crtc: which counter to give up
1265 *
1266 * Release ownership of a given vblank counter, turning off interrupts
1267 * if possible. Disable interrupts after &drm_vblank_crtc_config.offdelay_ms
1268 * milliseconds.
1269 */
1270void drm_crtc_vblank_put(struct drm_crtc *crtc)
1271{
1272	drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1273}
1274EXPORT_SYMBOL(drm_crtc_vblank_put);
1275
1276/**
1277 * drm_wait_one_vblank - wait for one vblank
1278 * @dev: DRM device
1279 * @pipe: CRTC index
1280 *
1281 * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1282 * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1283 * due to lack of driver support or because the crtc is off.
1284 *
1285 * This is the legacy version of drm_crtc_wait_one_vblank().
1286 */
1287void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1288{
1289	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1290	int ret;
1291	u64 last;
1292
1293	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1294		return;
1295
1296	ret = drm_vblank_get(dev, pipe);
1297	if (drm_WARN(dev, ret, "vblank not available on crtc %i, ret=%i\n",
1298		     pipe, ret))
1299		return;
1300
1301	last = drm_vblank_count(dev, pipe);
1302
1303	ret = wait_event_timeout(vblank->queue,
1304				 last != drm_vblank_count(dev, pipe),
1305				 msecs_to_jiffies(100));
1306
1307	drm_WARN(dev, ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1308
1309	drm_vblank_put(dev, pipe);
1310}
1311EXPORT_SYMBOL(drm_wait_one_vblank);
1312
1313/**
1314 * drm_crtc_wait_one_vblank - wait for one vblank
1315 * @crtc: DRM crtc
1316 *
1317 * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1318 * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1319 * due to lack of driver support or because the crtc is off.
1320 */
1321void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1322{
1323	drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1324}
1325EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1326
1327/**
1328 * drm_crtc_vblank_off - disable vblank events on a CRTC
1329 * @crtc: CRTC in question
1330 *
1331 * Drivers can use this function to shut down the vblank interrupt handling when
1332 * disabling a crtc. This function ensures that the latest vblank frame count is
1333 * stored so that drm_vblank_on can restore it again.
1334 *
1335 * Drivers must use this function when the hardware vblank counter can get
1336 * reset, e.g. when suspending or disabling the @crtc in general.
1337 */
1338void drm_crtc_vblank_off(struct drm_crtc *crtc)
1339{
1340	struct drm_device *dev = crtc->dev;
1341	unsigned int pipe = drm_crtc_index(crtc);
1342	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
1343	struct drm_pending_vblank_event *e, *t;
1344	ktime_t now;
1345	u64 seq;
1346
1347	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1348		return;
1349
1350	/*
1351	 * Grab event_lock early to prevent vblank work from being scheduled
1352	 * while we're in the middle of shutting down vblank interrupts
1353	 */
1354	spin_lock_irq(&dev->event_lock);
1355
1356	spin_lock(&dev->vbl_lock);
1357	drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1358		    pipe, vblank->enabled, vblank->inmodeset);
1359
1360	/* Avoid redundant vblank disables without previous
1361	 * drm_crtc_vblank_on(). */
1362	if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1363		drm_vblank_disable_and_save(dev, pipe);
1364
1365	wake_up(&vblank->queue);
1366
1367	/*
1368	 * Prevent subsequent drm_vblank_get() from re-enabling
1369	 * the vblank interrupt by bumping the refcount.
1370	 */
1371	if (!vblank->inmodeset) {
1372		atomic_inc(&vblank->refcount);
1373		vblank->inmodeset = 1;
1374	}
1375	spin_unlock(&dev->vbl_lock);
1376
1377	/* Send any queued vblank events, lest the natives grow disquiet */
1378	seq = drm_vblank_count_and_time(dev, pipe, &now);
1379
1380	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1381		if (e->pipe != pipe)
1382			continue;
1383		drm_dbg_core(dev, "Sending premature vblank event on disable: "
1384			     "wanted %llu, current %llu\n",
1385			     e->sequence, seq);
1386		list_del(&e->base.link);
1387		drm_vblank_put(dev, pipe);
1388		send_vblank_event(dev, e, seq, now);
1389	}
1390
1391	/* Cancel any leftover pending vblank work */
1392	drm_vblank_cancel_pending_works(vblank);
1393
1394	spin_unlock_irq(&dev->event_lock);
1395
1396	/* Will be reset by the modeset helpers when re-enabling the crtc by
1397	 * calling drm_calc_timestamping_constants(). */
1398	vblank->hwmode.crtc_clock = 0;
1399
1400	/* Wait for any vblank work that's still executing to finish */
1401	drm_vblank_flush_worker(vblank);
1402}
1403EXPORT_SYMBOL(drm_crtc_vblank_off);
1404
1405/**
1406 * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1407 * @crtc: CRTC in question
1408 *
1409 * Drivers can use this function to reset the vblank state to off at load time.
1410 * Drivers should use this together with the drm_crtc_vblank_off() and
1411 * drm_crtc_vblank_on() functions. The difference compared to
1412 * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1413 * and hence doesn't need to call any driver hooks.
1414 *
1415 * This is useful for recovering driver state e.g. on driver load, or on resume.
1416 */
1417void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1418{
1419	struct drm_device *dev = crtc->dev;
1420	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 
1421
1422	spin_lock_irq(&dev->vbl_lock);
1423	/*
1424	 * Prevent subsequent drm_vblank_get() from enabling the vblank
1425	 * interrupt by bumping the refcount.
1426	 */
1427	if (!vblank->inmodeset) {
1428		atomic_inc(&vblank->refcount);
1429		vblank->inmodeset = 1;
1430	}
1431	spin_unlock_irq(&dev->vbl_lock);
1432
1433	drm_WARN_ON(dev, !list_empty(&dev->vblank_event_list));
1434	drm_WARN_ON(dev, !list_empty(&vblank->pending_work));
1435}
1436EXPORT_SYMBOL(drm_crtc_vblank_reset);
1437
1438/**
1439 * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1440 * @crtc: CRTC in question
1441 * @max_vblank_count: max hardware vblank counter value
1442 *
1443 * Update the maximum hardware vblank counter value for @crtc
1444 * at runtime. Useful for hardware where the operation of the
1445 * hardware vblank counter depends on the currently active
1446 * display configuration.
1447 *
1448 * For example, if the hardware vblank counter does not work
1449 * when a specific connector is active the maximum can be set
1450 * to zero. And when that specific connector isn't active the
1451 * maximum can again be set to the appropriate non-zero value.
1452 *
1453 * If used, must be called before drm_vblank_on().
1454 */
1455void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1456				   u32 max_vblank_count)
1457{
1458	struct drm_device *dev = crtc->dev;
1459	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
 
1460
1461	drm_WARN_ON(dev, dev->max_vblank_count);
1462	drm_WARN_ON(dev, !READ_ONCE(vblank->inmodeset));
1463
1464	vblank->max_vblank_count = max_vblank_count;
1465}
1466EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1467
1468/**
1469 * drm_crtc_vblank_on_config - enable vblank events on a CRTC with custom
1470 *     configuration options
1471 * @crtc: CRTC in question
1472 * @config: Vblank configuration value
1473 *
1474 * See drm_crtc_vblank_on(). In addition, this function allows you to provide a
1475 * custom vblank configuration for a given CRTC.
1476 *
1477 * Note that @config is copied, the pointer does not need to stay valid beyond
1478 * this function call. For details of the parameters see
1479 * struct drm_vblank_crtc_config.
 
 
1480 */
1481void drm_crtc_vblank_on_config(struct drm_crtc *crtc,
1482			       const struct drm_vblank_crtc_config *config)
1483{
1484	struct drm_device *dev = crtc->dev;
1485	unsigned int pipe = drm_crtc_index(crtc);
1486	struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(crtc);
1487
1488	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1489		return;
1490
1491	spin_lock_irq(&dev->vbl_lock);
1492	drm_dbg_vbl(dev, "crtc %d, vblank enabled %d, inmodeset %d\n",
1493		    pipe, vblank->enabled, vblank->inmodeset);
1494
1495	vblank->config = *config;
1496
1497	/* Drop our private "prevent drm_vblank_get" refcount */
1498	if (vblank->inmodeset) {
1499		atomic_dec(&vblank->refcount);
1500		vblank->inmodeset = 0;
1501	}
1502
1503	drm_reset_vblank_timestamp(dev, pipe);
1504
1505	/*
1506	 * re-enable interrupts if there are users left, or the
1507	 * user wishes vblank interrupts to be enabled all the time.
1508	 */
1509	if (atomic_read(&vblank->refcount) != 0 || !vblank->config.offdelay_ms)
1510		drm_WARN_ON(dev, drm_vblank_enable(dev, pipe));
1511	spin_unlock_irq(&dev->vbl_lock);
1512}
1513EXPORT_SYMBOL(drm_crtc_vblank_on_config);
1514
1515/**
1516 * drm_crtc_vblank_on - enable vblank events on a CRTC
1517 * @crtc: CRTC in question
1518 *
1519 * This functions restores the vblank interrupt state captured with
1520 * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1521 * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1522 * unbalanced and so can also be unconditionally called in driver load code to
1523 * reflect the current hardware state of the crtc.
1524 *
1525 * Note that unlike in drm_crtc_vblank_on_config(), default values are used.
1526 */
1527void drm_crtc_vblank_on(struct drm_crtc *crtc)
1528{
1529	const struct drm_vblank_crtc_config config = {
1530		.offdelay_ms = drm_vblank_offdelay,
1531		.disable_immediate = crtc->dev->vblank_disable_immediate
1532	};
1533
1534	drm_crtc_vblank_on_config(crtc, &config);
1535}
1536EXPORT_SYMBOL(drm_crtc_vblank_on);
1537
1538static void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1539{
1540	ktime_t t_vblank;
1541	struct drm_vblank_crtc *vblank;
1542	int framedur_ns;
1543	u64 diff_ns;
1544	u32 cur_vblank, diff = 1;
1545	int count = DRM_TIMESTAMP_MAXRETRIES;
1546	u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
1547
1548	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1549		return;
1550
1551	assert_spin_locked(&dev->vbl_lock);
1552	assert_spin_locked(&dev->vblank_time_lock);
1553
1554	vblank = drm_vblank_crtc(dev, pipe);
1555	drm_WARN_ONCE(dev,
1556		      drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
1557		      "Cannot compute missed vblanks without frame duration\n");
1558	framedur_ns = vblank->framedur_ns;
1559
1560	do {
1561		cur_vblank = __get_vblank_counter(dev, pipe);
1562		drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1563	} while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1564
1565	diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1566	if (framedur_ns)
1567		diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1568
1569
1570	drm_dbg_vbl(dev,
1571		    "missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1572		    diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1573	vblank->last = (cur_vblank - diff) & max_vblank_count;
1574}
1575
1576/**
1577 * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1578 * @crtc: CRTC in question
1579 *
1580 * Power manamement features can cause frame counter resets between vblank
1581 * disable and enable. Drivers can use this function in their
1582 * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1583 * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1584 * vblank counter.
1585 *
1586 * Note that drivers must have race-free high-precision timestamping support,
1587 * i.e.  &drm_crtc_funcs.get_vblank_timestamp must be hooked up and
1588 * &drm_vblank_crtc_config.disable_immediate must be set to indicate the
1589 * time-stamping functions are race-free against vblank hardware counter
1590 * increments.
1591 */
1592void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1593{
1594	struct drm_device *dev = crtc->dev;
1595	unsigned int pipe = drm_crtc_index(crtc);
1596	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1597
1598	drm_WARN_ON_ONCE(dev, !crtc->funcs->get_vblank_timestamp);
1599	drm_WARN_ON_ONCE(dev, vblank->inmodeset);
1600	drm_WARN_ON_ONCE(dev, !vblank->config.disable_immediate);
1601
1602	drm_vblank_restore(dev, pipe);
1603}
1604EXPORT_SYMBOL(drm_crtc_vblank_restore);
1605
1606static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1607				  u64 req_seq,
1608				  union drm_wait_vblank *vblwait,
1609				  struct drm_file *file_priv)
1610{
1611	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1612	struct drm_pending_vblank_event *e;
1613	ktime_t now;
1614	u64 seq;
1615	int ret;
1616
1617	e = kzalloc(sizeof(*e), GFP_KERNEL);
1618	if (e == NULL) {
1619		ret = -ENOMEM;
1620		goto err_put;
1621	}
1622
1623	e->pipe = pipe;
1624	e->event.base.type = DRM_EVENT_VBLANK;
1625	e->event.base.length = sizeof(e->event.vbl);
1626	e->event.vbl.user_data = vblwait->request.signal;
1627	e->event.vbl.crtc_id = 0;
1628	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1629		struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1630
1631		if (crtc)
1632			e->event.vbl.crtc_id = crtc->base.id;
1633	}
1634
1635	spin_lock_irq(&dev->event_lock);
1636
1637	/*
1638	 * drm_crtc_vblank_off() might have been called after we called
1639	 * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1640	 * vblank disable, so no need for further locking.  The reference from
1641	 * drm_vblank_get() protects against vblank disable from another source.
1642	 */
1643	if (!READ_ONCE(vblank->enabled)) {
1644		ret = -EINVAL;
1645		goto err_unlock;
1646	}
1647
1648	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1649					    &e->event.base);
1650
1651	if (ret)
1652		goto err_unlock;
1653
1654	seq = drm_vblank_count_and_time(dev, pipe, &now);
1655
1656	drm_dbg_core(dev, "event on vblank count %llu, current %llu, crtc %u\n",
1657		     req_seq, seq, pipe);
1658
1659	trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1660
1661	e->sequence = req_seq;
1662	if (drm_vblank_passed(seq, req_seq)) {
1663		drm_vblank_put(dev, pipe);
1664		send_vblank_event(dev, e, seq, now);
1665		vblwait->reply.sequence = seq;
1666	} else {
1667		/* drm_handle_vblank_events will call drm_vblank_put */
1668		list_add_tail(&e->base.link, &dev->vblank_event_list);
1669		vblwait->reply.sequence = req_seq;
1670	}
1671
1672	spin_unlock_irq(&dev->event_lock);
1673
1674	return 0;
1675
1676err_unlock:
1677	spin_unlock_irq(&dev->event_lock);
1678	kfree(e);
1679err_put:
1680	drm_vblank_put(dev, pipe);
1681	return ret;
1682}
1683
1684static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1685{
1686	if (vblwait->request.sequence)
1687		return false;
1688
1689	return _DRM_VBLANK_RELATIVE ==
1690		(vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1691					  _DRM_VBLANK_EVENT |
1692					  _DRM_VBLANK_NEXTONMISS));
1693}
1694
1695/*
1696 * Widen a 32-bit param to 64-bits.
1697 *
1698 * \param narrow 32-bit value (missing upper 32 bits)
1699 * \param near 64-bit value that should be 'close' to near
1700 *
1701 * This function returns a 64-bit value using the lower 32-bits from
1702 * 'narrow' and constructing the upper 32-bits so that the result is
1703 * as close as possible to 'near'.
1704 */
1705
1706static u64 widen_32_to_64(u32 narrow, u64 near)
1707{
1708	return near + (s32) (narrow - near);
1709}
1710
1711static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1712				  struct drm_wait_vblank_reply *reply)
1713{
1714	ktime_t now;
1715	struct timespec64 ts;
1716
1717	/*
1718	 * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1719	 * to store the seconds. This is safe as we always use monotonic
1720	 * timestamps since linux-4.15.
1721	 */
1722	reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1723	ts = ktime_to_timespec64(now);
1724	reply->tval_sec = (u32)ts.tv_sec;
1725	reply->tval_usec = ts.tv_nsec / 1000;
1726}
1727
1728static bool drm_wait_vblank_supported(struct drm_device *dev)
1729{
1730	return drm_dev_has_vblank(dev);
1731}
1732
1733int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1734			  struct drm_file *file_priv)
1735{
1736	struct drm_crtc *crtc;
1737	struct drm_vblank_crtc *vblank;
1738	union drm_wait_vblank *vblwait = data;
1739	int ret;
1740	u64 req_seq, seq;
1741	unsigned int pipe_index;
1742	unsigned int flags, pipe, high_pipe;
1743
1744	if (!drm_wait_vblank_supported(dev))
1745		return -EOPNOTSUPP;
1746
1747	if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1748		return -EINVAL;
1749
1750	if (vblwait->request.type &
1751	    ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1752	      _DRM_VBLANK_HIGH_CRTC_MASK)) {
1753		drm_dbg_core(dev,
1754			     "Unsupported type value 0x%x, supported mask 0x%x\n",
1755			     vblwait->request.type,
1756			     (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1757			      _DRM_VBLANK_HIGH_CRTC_MASK));
1758		return -EINVAL;
1759	}
1760
1761	flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1762	high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1763	if (high_pipe)
1764		pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1765	else
1766		pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1767
1768	/* Convert lease-relative crtc index into global crtc index */
1769	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1770		pipe = 0;
1771		drm_for_each_crtc(crtc, dev) {
1772			if (drm_lease_held(file_priv, crtc->base.id)) {
1773				if (pipe_index == 0)
1774					break;
1775				pipe_index--;
1776			}
1777			pipe++;
1778		}
1779	} else {
1780		pipe = pipe_index;
1781	}
1782
1783	if (pipe >= dev->num_crtcs)
1784		return -EINVAL;
1785
1786	vblank = &dev->vblank[pipe];
1787
1788	/* If the counter is currently enabled and accurate, short-circuit
1789	 * queries to return the cached timestamp of the last vblank.
1790	 */
1791	if (vblank->config.disable_immediate &&
1792	    drm_wait_vblank_is_query(vblwait) &&
1793	    READ_ONCE(vblank->enabled)) {
1794		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1795		return 0;
1796	}
1797
1798	ret = drm_vblank_get(dev, pipe);
1799	if (ret) {
1800		drm_dbg_core(dev,
1801			     "crtc %d failed to acquire vblank counter, %d\n",
1802			     pipe, ret);
1803		return ret;
1804	}
1805	seq = drm_vblank_count(dev, pipe);
1806
1807	switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1808	case _DRM_VBLANK_RELATIVE:
1809		req_seq = seq + vblwait->request.sequence;
1810		vblwait->request.sequence = req_seq;
1811		vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1812		break;
1813	case _DRM_VBLANK_ABSOLUTE:
1814		req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1815		break;
1816	default:
1817		ret = -EINVAL;
1818		goto done;
1819	}
1820
1821	if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1822	    drm_vblank_passed(seq, req_seq)) {
1823		req_seq = seq + 1;
1824		vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1825		vblwait->request.sequence = req_seq;
1826	}
1827
1828	if (flags & _DRM_VBLANK_EVENT) {
1829		/* must hold on to the vblank ref until the event fires
1830		 * drm_vblank_put will be called asynchronously
1831		 */
1832		return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1833	}
1834
1835	if (req_seq != seq) {
1836		int wait;
1837
1838		drm_dbg_core(dev, "waiting on vblank count %llu, crtc %u\n",
1839			     req_seq, pipe);
1840		wait = wait_event_interruptible_timeout(vblank->queue,
1841			drm_vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1842				      !READ_ONCE(vblank->enabled),
1843			msecs_to_jiffies(3000));
1844
1845		switch (wait) {
1846		case 0:
1847			/* timeout */
1848			ret = -EBUSY;
1849			break;
1850		case -ERESTARTSYS:
1851			/* interrupted by signal */
1852			ret = -EINTR;
1853			break;
1854		default:
1855			ret = 0;
1856			break;
1857		}
1858	}
1859
1860	if (ret != -EINTR) {
1861		drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1862
1863		drm_dbg_core(dev, "crtc %d returning %u to client\n",
1864			     pipe, vblwait->reply.sequence);
1865	} else {
1866		drm_dbg_core(dev, "crtc %d vblank wait interrupted by signal\n",
1867			     pipe);
1868	}
1869
1870done:
1871	drm_vblank_put(dev, pipe);
1872	return ret;
1873}
1874
1875static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1876{
1877	struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1878	bool high_prec = false;
1879	struct drm_pending_vblank_event *e, *t;
1880	ktime_t now;
1881	u64 seq;
1882
1883	assert_spin_locked(&dev->event_lock);
1884
1885	seq = drm_vblank_count_and_time(dev, pipe, &now);
1886
1887	list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1888		if (e->pipe != pipe)
1889			continue;
1890		if (!drm_vblank_passed(seq, e->sequence))
1891			continue;
1892
1893		drm_dbg_core(dev, "vblank event on %llu, current %llu\n",
1894			     e->sequence, seq);
1895
1896		list_del(&e->base.link);
1897		drm_vblank_put(dev, pipe);
1898		send_vblank_event(dev, e, seq, now);
1899	}
1900
1901	if (crtc && crtc->funcs->get_vblank_timestamp)
1902		high_prec = true;
1903
1904	trace_drm_vblank_event(pipe, seq, now, high_prec);
1905}
1906
1907/**
1908 * drm_handle_vblank - handle a vblank event
1909 * @dev: DRM device
1910 * @pipe: index of CRTC where this event occurred
1911 *
1912 * Drivers should call this routine in their vblank interrupt handlers to
1913 * update the vblank counter and send any signals that may be pending.
1914 *
1915 * This is the legacy version of drm_crtc_handle_vblank().
1916 */
1917bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1918{
1919	struct drm_vblank_crtc *vblank = drm_vblank_crtc(dev, pipe);
1920	unsigned long irqflags;
1921	bool disable_irq;
1922
1923	if (drm_WARN_ON_ONCE(dev, !drm_dev_has_vblank(dev)))
1924		return false;
1925
1926	if (drm_WARN_ON(dev, pipe >= dev->num_crtcs))
1927		return false;
1928
1929	spin_lock_irqsave(&dev->event_lock, irqflags);
1930
1931	/* Need timestamp lock to prevent concurrent execution with
1932	 * vblank enable/disable, as this would cause inconsistent
1933	 * or corrupted timestamps and vblank counts.
1934	 */
1935	spin_lock(&dev->vblank_time_lock);
1936
1937	/* Vblank irq handling disabled. Nothing to do. */
1938	if (!vblank->enabled) {
1939		spin_unlock(&dev->vblank_time_lock);
1940		spin_unlock_irqrestore(&dev->event_lock, irqflags);
1941		return false;
1942	}
1943
1944	drm_update_vblank_count(dev, pipe, true);
1945
1946	spin_unlock(&dev->vblank_time_lock);
1947
1948	wake_up(&vblank->queue);
1949
1950	/* With instant-off, we defer disabling the interrupt until after
1951	 * we finish processing the following vblank after all events have
1952	 * been signaled. The disable has to be last (after
1953	 * drm_handle_vblank_events) so that the timestamp is always accurate.
1954	 */
1955	disable_irq = (vblank->config.disable_immediate &&
1956		       vblank->config.offdelay_ms > 0 &&
1957		       !atomic_read(&vblank->refcount));
1958
1959	drm_handle_vblank_events(dev, pipe);
1960	drm_handle_vblank_works(vblank);
1961
1962	spin_unlock_irqrestore(&dev->event_lock, irqflags);
1963
1964	if (disable_irq)
1965		vblank_disable_fn(&vblank->disable_timer);
1966
1967	return true;
1968}
1969EXPORT_SYMBOL(drm_handle_vblank);
1970
1971/**
1972 * drm_crtc_handle_vblank - handle a vblank event
1973 * @crtc: where this event occurred
1974 *
1975 * Drivers should call this routine in their vblank interrupt handlers to
1976 * update the vblank counter and send any signals that may be pending.
1977 *
1978 * This is the native KMS version of drm_handle_vblank().
1979 *
1980 * Note that for a given vblank counter value drm_crtc_handle_vblank()
1981 * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
1982 * provide a barrier: Any writes done before calling
1983 * drm_crtc_handle_vblank() will be visible to callers of the later
1984 * functions, if the vblank count is the same or a later one.
1985 *
1986 * See also &drm_vblank_crtc.count.
1987 *
1988 * Returns:
1989 * True if the event was successfully handled, false on failure.
1990 */
1991bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1992{
1993	return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1994}
1995EXPORT_SYMBOL(drm_crtc_handle_vblank);
1996
1997/*
1998 * Get crtc VBLANK count.
1999 *
2000 * \param dev DRM device
2001 * \param data user argument, pointing to a drm_crtc_get_sequence structure.
2002 * \param file_priv drm file private for the user's open file descriptor
2003 */
2004
2005int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
2006				struct drm_file *file_priv)
2007{
2008	struct drm_crtc *crtc;
2009	struct drm_vblank_crtc *vblank;
2010	int pipe;
2011	struct drm_crtc_get_sequence *get_seq = data;
2012	ktime_t now;
2013	bool vblank_enabled;
2014	int ret;
2015
2016	if (!drm_core_check_feature(dev, DRIVER_MODESET))
2017		return -EOPNOTSUPP;
2018
2019	if (!drm_dev_has_vblank(dev))
2020		return -EOPNOTSUPP;
2021
2022	crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
2023	if (!crtc)
2024		return -ENOENT;
2025
2026	pipe = drm_crtc_index(crtc);
2027
2028	vblank = drm_crtc_vblank_crtc(crtc);
2029	vblank_enabled = READ_ONCE(vblank->config.disable_immediate) &&
2030		READ_ONCE(vblank->enabled);
2031
2032	if (!vblank_enabled) {
2033		ret = drm_crtc_vblank_get(crtc);
2034		if (ret) {
2035			drm_dbg_core(dev,
2036				     "crtc %d failed to acquire vblank counter, %d\n",
2037				     pipe, ret);
2038			return ret;
2039		}
2040	}
2041	drm_modeset_lock(&crtc->mutex, NULL);
2042	if (crtc->state)
2043		get_seq->active = crtc->state->enable;
2044	else
2045		get_seq->active = crtc->enabled;
2046	drm_modeset_unlock(&crtc->mutex);
2047	get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
2048	get_seq->sequence_ns = ktime_to_ns(now);
2049	if (!vblank_enabled)
2050		drm_crtc_vblank_put(crtc);
2051	return 0;
2052}
2053
2054/*
2055 * Queue a event for VBLANK sequence
2056 *
2057 * \param dev DRM device
2058 * \param data user argument, pointing to a drm_crtc_queue_sequence structure.
2059 * \param file_priv drm file private for the user's open file descriptor
2060 */
2061
2062int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
2063				  struct drm_file *file_priv)
2064{
2065	struct drm_crtc *crtc;
2066	struct drm_vblank_crtc *vblank;
2067	int pipe;
2068	struct drm_crtc_queue_sequence *queue_seq = data;
2069	ktime_t now;
2070	struct drm_pending_vblank_event *e;
2071	u32 flags;
2072	u64 seq;
2073	u64 req_seq;
2074	int ret;
2075
2076	if (!drm_core_check_feature(dev, DRIVER_MODESET))
2077		return -EOPNOTSUPP;
2078
2079	if (!drm_dev_has_vblank(dev))
2080		return -EOPNOTSUPP;
2081
2082	crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
2083	if (!crtc)
2084		return -ENOENT;
2085
2086	flags = queue_seq->flags;
2087	/* Check valid flag bits */
2088	if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
2089		      DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
2090		return -EINVAL;
2091
2092	pipe = drm_crtc_index(crtc);
2093
2094	vblank = drm_crtc_vblank_crtc(crtc);
2095
2096	e = kzalloc(sizeof(*e), GFP_KERNEL);
2097	if (e == NULL)
2098		return -ENOMEM;
2099
2100	ret = drm_crtc_vblank_get(crtc);
2101	if (ret) {
2102		drm_dbg_core(dev,
2103			     "crtc %d failed to acquire vblank counter, %d\n",
2104			     pipe, ret);
2105		goto err_free;
2106	}
2107
2108	seq = drm_vblank_count_and_time(dev, pipe, &now);
2109	req_seq = queue_seq->sequence;
2110
2111	if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
2112		req_seq += seq;
2113
2114	if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && drm_vblank_passed(seq, req_seq))
2115		req_seq = seq + 1;
2116
2117	e->pipe = pipe;
2118	e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
2119	e->event.base.length = sizeof(e->event.seq);
2120	e->event.seq.user_data = queue_seq->user_data;
2121
2122	spin_lock_irq(&dev->event_lock);
2123
2124	/*
2125	 * drm_crtc_vblank_off() might have been called after we called
2126	 * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
2127	 * vblank disable, so no need for further locking.  The reference from
2128	 * drm_crtc_vblank_get() protects against vblank disable from another source.
2129	 */
2130	if (!READ_ONCE(vblank->enabled)) {
2131		ret = -EINVAL;
2132		goto err_unlock;
2133	}
2134
2135	ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
2136					    &e->event.base);
2137
2138	if (ret)
2139		goto err_unlock;
2140
2141	e->sequence = req_seq;
2142
2143	if (drm_vblank_passed(seq, req_seq)) {
2144		drm_crtc_vblank_put(crtc);
2145		send_vblank_event(dev, e, seq, now);
2146		queue_seq->sequence = seq;
2147	} else {
2148		/* drm_handle_vblank_events will call drm_vblank_put */
2149		list_add_tail(&e->base.link, &dev->vblank_event_list);
2150		queue_seq->sequence = req_seq;
2151	}
2152
2153	spin_unlock_irq(&dev->event_lock);
2154	return 0;
2155
2156err_unlock:
2157	spin_unlock_irq(&dev->event_lock);
2158	drm_crtc_vblank_put(crtc);
2159err_free:
2160	kfree(e);
2161	return ret;
2162}
2163