1// SPDX-License-Identifier: GPL-2.0
  2/*
  3 * Generic userspace implementations of gettimeofday() and similar.
  4 */
  5#include <linux/compiler.h>
  6#include <linux/math64.h>
  7#include <linux/time.h>
  8#include <linux/kernel.h>
  9#include <linux/hrtimer_defs.h>
 10#include <vdso/datapage.h>
 11#include <vdso/helpers.h>
 12
 13/*
 14 * The generic vDSO implementation requires that gettimeofday.h
 15 * provides:
 16 * - __arch_get_vdso_data(): to get the vdso datapage.
 17 * - __arch_get_hw_counter(): to get the hw counter based on the
 18 *   clock_mode.
 19 * - gettimeofday_fallback(): fallback for gettimeofday.
 20 * - clock_gettime_fallback(): fallback for clock_gettime.
 21 * - clock_getres_fallback(): fallback for clock_getres.
 22 */
 23#ifdef ENABLE_COMPAT_VDSO
 24#include <asm/vdso/compat_gettimeofday.h>
 25#else
 26#include <asm/vdso/gettimeofday.h>
 27#endif /* ENABLE_COMPAT_VDSO */
 28
 29#ifndef vdso_calc_delta
 30/*
 31 * Default implementation which works for all sane clocksources. That
 32 * obviously excludes x86/TSC.
 33 */
 34static __always_inline
 35u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
 36{
 37	return ((cycles - last) & mask) * mult;
 38}
 39#endif
 40
 41static int do_hres(const struct vdso_data *vd, clockid_t clk,
 42		   struct __kernel_timespec *ts)
 43{
 44	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
 45	u64 cycles, last, sec, ns;
 46	u32 seq;
 47
 48	do {
 49		seq = vdso_read_begin(vd);
 50		cycles = __arch_get_hw_counter(vd->clock_mode);
 51		ns = vdso_ts->nsec;
 52		last = vd->cycle_last;
 53		if (unlikely((s64)cycles < 0))
 54			return -1;
 55
 56		ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
 57		ns >>= vd->shift;
 58		sec = vdso_ts->sec;
 59	} while (unlikely(vdso_read_retry(vd, seq)));
 60
 61	/*
 62	 * Do this outside the loop: a race inside the loop could result
 63	 * in __iter_div_u64_rem() being extremely slow.
 64	 */
 65	ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
 66	ts->tv_nsec = ns;
 67
 68	return 0;
 69}
 70
 71static void do_coarse(const struct vdso_data *vd, clockid_t clk,
 72		      struct __kernel_timespec *ts)
 73{
 74	const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
 75	u32 seq;
 76
 77	do {
 78		seq = vdso_read_begin(vd);
 79		ts->tv_sec = vdso_ts->sec;
 80		ts->tv_nsec = vdso_ts->nsec;
 81	} while (unlikely(vdso_read_retry(vd, seq)));
 82}
 83
 84static __maybe_unused int
 85__cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
 86{
 87	const struct vdso_data *vd = __arch_get_vdso_data();
 88	u32 msk;
 89
 90	/* Check for negative values or invalid clocks */
 91	if (unlikely((u32) clock >= MAX_CLOCKS))
 92		return -1;
 93
 94	/*
 95	 * Convert the clockid to a bitmask and use it to check which
 96	 * clocks are handled in the VDSO directly.
 97	 */
 98	msk = 1U << clock;
 99	if (likely(msk & VDSO_HRES)) {
100		return do_hres(&vd[CS_HRES_COARSE], clock, ts);
101	} else if (msk & VDSO_COARSE) {
102		do_coarse(&vd[CS_HRES_COARSE], clock, ts);
103		return 0;
104	} else if (msk & VDSO_RAW) {
105		return do_hres(&vd[CS_RAW], clock, ts);
106	}
107	return -1;
108}
109
110static __maybe_unused int
111__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
112{
113	int ret = __cvdso_clock_gettime_common(clock, ts);
114
115	if (unlikely(ret))
116		return clock_gettime_fallback(clock, ts);
117	return 0;
118}
119
120static __maybe_unused int
121__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
122{
123	struct __kernel_timespec ts;
124	int ret;
125
126	ret = __cvdso_clock_gettime_common(clock, &ts);
127
128#ifdef VDSO_HAS_32BIT_FALLBACK
129	if (unlikely(ret))
130		return clock_gettime32_fallback(clock, res);
131#else
132	if (unlikely(ret))
133		ret = clock_gettime_fallback(clock, &ts);
134#endif
135
136	if (likely(!ret)) {
137		res->tv_sec = ts.tv_sec;
138		res->tv_nsec = ts.tv_nsec;
139	}
140	return ret;
141}
142
143static __maybe_unused int
144__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
145{
146	const struct vdso_data *vd = __arch_get_vdso_data();
147
148	if (likely(tv != NULL)) {
149		struct __kernel_timespec ts;
150
151		if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
152			return gettimeofday_fallback(tv, tz);
153
154		tv->tv_sec = ts.tv_sec;
155		tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
156	}
157
158	if (unlikely(tz != NULL)) {
159		tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
160		tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
161	}
162
163	return 0;
164}
165
166#ifdef VDSO_HAS_TIME
167static __maybe_unused time_t __cvdso_time(time_t *time)
168{
169	const struct vdso_data *vd = __arch_get_vdso_data();
170	time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
171
172	if (time)
173		*time = t;
174
175	return t;
176}
177#endif /* VDSO_HAS_TIME */
178
179#ifdef VDSO_HAS_CLOCK_GETRES
180static __maybe_unused
181int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
182{
183	const struct vdso_data *vd = __arch_get_vdso_data();
184	u64 hrtimer_res;
185	u32 msk;
186	u64 ns;
187
188	/* Check for negative values or invalid clocks */
189	if (unlikely((u32) clock >= MAX_CLOCKS))
190		return -1;
191
192	hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
193	/*
194	 * Convert the clockid to a bitmask and use it to check which
195	 * clocks are handled in the VDSO directly.
196	 */
197	msk = 1U << clock;
198	if (msk & VDSO_HRES) {
199		/*
200		 * Preserves the behaviour of posix_get_hrtimer_res().
201		 */
202		ns = hrtimer_res;
203	} else if (msk & VDSO_COARSE) {
204		/*
205		 * Preserves the behaviour of posix_get_coarse_res().
206		 */
207		ns = LOW_RES_NSEC;
208	} else if (msk & VDSO_RAW) {
209		/*
210		 * Preserves the behaviour of posix_get_hrtimer_res().
211		 */
212		ns = hrtimer_res;
213	} else {
214		return -1;
215	}
216
217	if (likely(res)) {
218		res->tv_sec = 0;
219		res->tv_nsec = ns;
220	}
221	return 0;
222}
223
224int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
225{
226	int ret = __cvdso_clock_getres_common(clock, res);
227
228	if (unlikely(ret))
229		return clock_getres_fallback(clock, res);
230	return 0;
231}
232
233static __maybe_unused int
234__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
235{
236	struct __kernel_timespec ts;
237	int ret;
238
239	ret = __cvdso_clock_getres_common(clock, &ts);
240
241#ifdef VDSO_HAS_32BIT_FALLBACK
242	if (unlikely(ret))
243		return clock_getres32_fallback(clock, res);
244#else
245	if (unlikely(ret))
246		ret = clock_getres_fallback(clock, &ts);
247#endif
248
249	if (likely(!ret && res)) {
250		res->tv_sec = ts.tv_sec;
251		res->tv_nsec = ts.tv_nsec;
252	}
253	return ret;
254}
255#endif /* VDSO_HAS_CLOCK_GETRES */