Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Generic userspace implementations of gettimeofday() and similar.
4 */
5#include <vdso/datapage.h>
6#include <vdso/helpers.h>
7
8#ifndef vdso_calc_ns
9
10#ifdef VDSO_DELTA_NOMASK
11# define VDSO_DELTA_MASK(vd) ULLONG_MAX
12#else
13# define VDSO_DELTA_MASK(vd) (vd->mask)
14#endif
15
16#ifdef CONFIG_GENERIC_VDSO_OVERFLOW_PROTECT
17static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
18{
19 return delta < vd->max_cycles;
20}
21#else
22static __always_inline bool vdso_delta_ok(const struct vdso_data *vd, u64 delta)
23{
24 return true;
25}
26#endif
27
28#ifndef vdso_shift_ns
29static __always_inline u64 vdso_shift_ns(u64 ns, u32 shift)
30{
31 return ns >> shift;
32}
33#endif
34
35/*
36 * Default implementation which works for all sane clocksources. That
37 * obviously excludes x86/TSC.
38 */
39static __always_inline u64 vdso_calc_ns(const struct vdso_data *vd, u64 cycles, u64 base)
40{
41 u64 delta = (cycles - vd->cycle_last) & VDSO_DELTA_MASK(vd);
42
43 if (likely(vdso_delta_ok(vd, delta)))
44 return vdso_shift_ns((delta * vd->mult) + base, vd->shift);
45
46 return mul_u64_u32_add_u64_shr(delta, vd->mult, base, vd->shift);
47}
48#endif /* vdso_calc_ns */
49
50#ifndef __arch_vdso_hres_capable
51static inline bool __arch_vdso_hres_capable(void)
52{
53 return true;
54}
55#endif
56
57#ifndef vdso_clocksource_ok
58static inline bool vdso_clocksource_ok(const struct vdso_data *vd)
59{
60 return vd->clock_mode != VDSO_CLOCKMODE_NONE;
61}
62#endif
63
64#ifndef vdso_cycles_ok
65static inline bool vdso_cycles_ok(u64 cycles)
66{
67 return true;
68}
69#endif
70
71#ifdef CONFIG_TIME_NS
72static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
73 struct __kernel_timespec *ts)
74{
75 const struct timens_offset *offs = &vdns->offset[clk];
76 const struct vdso_timestamp *vdso_ts;
77 const struct vdso_data *vd;
78 u64 cycles, ns;
79 u32 seq;
80 s64 sec;
81
82 vd = vdns - (clk == CLOCK_MONOTONIC_RAW ? CS_RAW : CS_HRES_COARSE);
83 vd = __arch_get_timens_vdso_data(vd);
84 if (clk != CLOCK_MONOTONIC_RAW)
85 vd = &vd[CS_HRES_COARSE];
86 else
87 vd = &vd[CS_RAW];
88 vdso_ts = &vd->basetime[clk];
89
90 do {
91 seq = vdso_read_begin(vd);
92
93 if (unlikely(!vdso_clocksource_ok(vd)))
94 return -1;
95
96 cycles = __arch_get_hw_counter(vd->clock_mode, vd);
97 if (unlikely(!vdso_cycles_ok(cycles)))
98 return -1;
99 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
100 sec = vdso_ts->sec;
101 } while (unlikely(vdso_read_retry(vd, seq)));
102
103 /* Add the namespace offset */
104 sec += offs->sec;
105 ns += offs->nsec;
106
107 /*
108 * Do this outside the loop: a race inside the loop could result
109 * in __iter_div_u64_rem() being extremely slow.
110 */
111 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
112 ts->tv_nsec = ns;
113
114 return 0;
115}
116#else
117static __always_inline
118const struct vdso_data *__arch_get_timens_vdso_data(const struct vdso_data *vd)
119{
120 return NULL;
121}
122
123static __always_inline int do_hres_timens(const struct vdso_data *vdns, clockid_t clk,
124 struct __kernel_timespec *ts)
125{
126 return -EINVAL;
127}
128#endif
129
130static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
131 struct __kernel_timespec *ts)
132{
133 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
134 u64 cycles, sec, ns;
135 u32 seq;
136
137 /* Allows to compile the high resolution parts out */
138 if (!__arch_vdso_hres_capable())
139 return -1;
140
141 do {
142 /*
143 * Open coded function vdso_read_begin() to handle
144 * VDSO_CLOCKMODE_TIMENS. Time namespace enabled tasks have a
145 * special VVAR page installed which has vd->seq set to 1 and
146 * vd->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non time
147 * namespace affected tasks this does not affect performance
148 * because if vd->seq is odd, i.e. a concurrent update is in
149 * progress the extra check for vd->clock_mode is just a few
150 * extra instructions while spin waiting for vd->seq to become
151 * even again.
152 */
153 while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) {
154 if (IS_ENABLED(CONFIG_TIME_NS) &&
155 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
156 return do_hres_timens(vd, clk, ts);
157 cpu_relax();
158 }
159 smp_rmb();
160
161 if (unlikely(!vdso_clocksource_ok(vd)))
162 return -1;
163
164 cycles = __arch_get_hw_counter(vd->clock_mode, vd);
165 if (unlikely(!vdso_cycles_ok(cycles)))
166 return -1;
167 ns = vdso_calc_ns(vd, cycles, vdso_ts->nsec);
168 sec = vdso_ts->sec;
169 } while (unlikely(vdso_read_retry(vd, seq)));
170
171 /*
172 * Do this outside the loop: a race inside the loop could result
173 * in __iter_div_u64_rem() being extremely slow.
174 */
175 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
176 ts->tv_nsec = ns;
177
178 return 0;
179}
180
181#ifdef CONFIG_TIME_NS
182static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
183 struct __kernel_timespec *ts)
184{
185 const struct vdso_data *vd = __arch_get_timens_vdso_data(vdns);
186 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
187 const struct timens_offset *offs = &vdns->offset[clk];
188 u64 nsec;
189 s64 sec;
190 s32 seq;
191
192 do {
193 seq = vdso_read_begin(vd);
194 sec = vdso_ts->sec;
195 nsec = vdso_ts->nsec;
196 } while (unlikely(vdso_read_retry(vd, seq)));
197
198 /* Add the namespace offset */
199 sec += offs->sec;
200 nsec += offs->nsec;
201
202 /*
203 * Do this outside the loop: a race inside the loop could result
204 * in __iter_div_u64_rem() being extremely slow.
205 */
206 ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
207 ts->tv_nsec = nsec;
208 return 0;
209}
210#else
211static __always_inline int do_coarse_timens(const struct vdso_data *vdns, clockid_t clk,
212 struct __kernel_timespec *ts)
213{
214 return -1;
215}
216#endif
217
218static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
219 struct __kernel_timespec *ts)
220{
221 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
222 u32 seq;
223
224 do {
225 /*
226 * Open coded function vdso_read_begin() to handle
227 * VDSO_CLOCK_TIMENS. See comment in do_hres().
228 */
229 while ((seq = READ_ONCE(vd->seq)) & 1) {
230 if (IS_ENABLED(CONFIG_TIME_NS) &&
231 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
232 return do_coarse_timens(vd, clk, ts);
233 cpu_relax();
234 }
235 smp_rmb();
236
237 ts->tv_sec = vdso_ts->sec;
238 ts->tv_nsec = vdso_ts->nsec;
239 } while (unlikely(vdso_read_retry(vd, seq)));
240
241 return 0;
242}
243
244static __always_inline int
245__cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock,
246 struct __kernel_timespec *ts)
247{
248 u32 msk;
249
250 /* Check for negative values or invalid clocks */
251 if (unlikely((u32) clock >= MAX_CLOCKS))
252 return -1;
253
254 /*
255 * Convert the clockid to a bitmask and use it to check which
256 * clocks are handled in the VDSO directly.
257 */
258 msk = 1U << clock;
259 if (likely(msk & VDSO_HRES))
260 vd = &vd[CS_HRES_COARSE];
261 else if (msk & VDSO_COARSE)
262 return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
263 else if (msk & VDSO_RAW)
264 vd = &vd[CS_RAW];
265 else
266 return -1;
267
268 return do_hres(vd, clock, ts);
269}
270
271static __maybe_unused int
272__cvdso_clock_gettime_data(const struct vdso_data *vd, clockid_t clock,
273 struct __kernel_timespec *ts)
274{
275 int ret = __cvdso_clock_gettime_common(vd, clock, ts);
276
277 if (unlikely(ret))
278 return clock_gettime_fallback(clock, ts);
279 return 0;
280}
281
282static __maybe_unused int
283__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
284{
285 return __cvdso_clock_gettime_data(__arch_get_vdso_data(), clock, ts);
286}
287
288#ifdef BUILD_VDSO32
289static __maybe_unused int
290__cvdso_clock_gettime32_data(const struct vdso_data *vd, clockid_t clock,
291 struct old_timespec32 *res)
292{
293 struct __kernel_timespec ts;
294 int ret;
295
296 ret = __cvdso_clock_gettime_common(vd, clock, &ts);
297
298 if (unlikely(ret))
299 return clock_gettime32_fallback(clock, res);
300
301 /* For ret == 0 */
302 res->tv_sec = ts.tv_sec;
303 res->tv_nsec = ts.tv_nsec;
304
305 return ret;
306}
307
308static __maybe_unused int
309__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
310{
311 return __cvdso_clock_gettime32_data(__arch_get_vdso_data(), clock, res);
312}
313#endif /* BUILD_VDSO32 */
314
315static __maybe_unused int
316__cvdso_gettimeofday_data(const struct vdso_data *vd,
317 struct __kernel_old_timeval *tv, struct timezone *tz)
318{
319
320 if (likely(tv != NULL)) {
321 struct __kernel_timespec ts;
322
323 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
324 return gettimeofday_fallback(tv, tz);
325
326 tv->tv_sec = ts.tv_sec;
327 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
328 }
329
330 if (unlikely(tz != NULL)) {
331 if (IS_ENABLED(CONFIG_TIME_NS) &&
332 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
333 vd = __arch_get_timens_vdso_data(vd);
334
335 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
336 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
337 }
338
339 return 0;
340}
341
342static __maybe_unused int
343__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
344{
345 return __cvdso_gettimeofday_data(__arch_get_vdso_data(), tv, tz);
346}
347
348#ifdef VDSO_HAS_TIME
349static __maybe_unused __kernel_old_time_t
350__cvdso_time_data(const struct vdso_data *vd, __kernel_old_time_t *time)
351{
352 __kernel_old_time_t t;
353
354 if (IS_ENABLED(CONFIG_TIME_NS) &&
355 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
356 vd = __arch_get_timens_vdso_data(vd);
357
358 t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
359
360 if (time)
361 *time = t;
362
363 return t;
364}
365
366static __maybe_unused __kernel_old_time_t __cvdso_time(__kernel_old_time_t *time)
367{
368 return __cvdso_time_data(__arch_get_vdso_data(), time);
369}
370#endif /* VDSO_HAS_TIME */
371
372#ifdef VDSO_HAS_CLOCK_GETRES
373static __maybe_unused
374int __cvdso_clock_getres_common(const struct vdso_data *vd, clockid_t clock,
375 struct __kernel_timespec *res)
376{
377 u32 msk;
378 u64 ns;
379
380 /* Check for negative values or invalid clocks */
381 if (unlikely((u32) clock >= MAX_CLOCKS))
382 return -1;
383
384 if (IS_ENABLED(CONFIG_TIME_NS) &&
385 vd->clock_mode == VDSO_CLOCKMODE_TIMENS)
386 vd = __arch_get_timens_vdso_data(vd);
387
388 /*
389 * Convert the clockid to a bitmask and use it to check which
390 * clocks are handled in the VDSO directly.
391 */
392 msk = 1U << clock;
393 if (msk & (VDSO_HRES | VDSO_RAW)) {
394 /*
395 * Preserves the behaviour of posix_get_hrtimer_res().
396 */
397 ns = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
398 } else if (msk & VDSO_COARSE) {
399 /*
400 * Preserves the behaviour of posix_get_coarse_res().
401 */
402 ns = LOW_RES_NSEC;
403 } else {
404 return -1;
405 }
406
407 if (likely(res)) {
408 res->tv_sec = 0;
409 res->tv_nsec = ns;
410 }
411 return 0;
412}
413
414static __maybe_unused
415int __cvdso_clock_getres_data(const struct vdso_data *vd, clockid_t clock,
416 struct __kernel_timespec *res)
417{
418 int ret = __cvdso_clock_getres_common(vd, clock, res);
419
420 if (unlikely(ret))
421 return clock_getres_fallback(clock, res);
422 return 0;
423}
424
425static __maybe_unused
426int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
427{
428 return __cvdso_clock_getres_data(__arch_get_vdso_data(), clock, res);
429}
430
431#ifdef BUILD_VDSO32
432static __maybe_unused int
433__cvdso_clock_getres_time32_data(const struct vdso_data *vd, clockid_t clock,
434 struct old_timespec32 *res)
435{
436 struct __kernel_timespec ts;
437 int ret;
438
439 ret = __cvdso_clock_getres_common(vd, clock, &ts);
440
441 if (unlikely(ret))
442 return clock_getres32_fallback(clock, res);
443
444 if (likely(res)) {
445 res->tv_sec = ts.tv_sec;
446 res->tv_nsec = ts.tv_nsec;
447 }
448 return ret;
449}
450
451static __maybe_unused int
452__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
453{
454 return __cvdso_clock_getres_time32_data(__arch_get_vdso_data(),
455 clock, res);
456}
457#endif /* BUILD_VDSO32 */
458#endif /* VDSO_HAS_CLOCK_GETRES */
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Generic userspace implementations of gettimeofday() and similar.
4 */
5#include <linux/compiler.h>
6#include <linux/math64.h>
7#include <linux/time.h>
8#include <linux/kernel.h>
9#include <linux/hrtimer_defs.h>
10#include <vdso/datapage.h>
11#include <vdso/helpers.h>
12
13/*
14 * The generic vDSO implementation requires that gettimeofday.h
15 * provides:
16 * - __arch_get_vdso_data(): to get the vdso datapage.
17 * - __arch_get_hw_counter(): to get the hw counter based on the
18 * clock_mode.
19 * - gettimeofday_fallback(): fallback for gettimeofday.
20 * - clock_gettime_fallback(): fallback for clock_gettime.
21 * - clock_getres_fallback(): fallback for clock_getres.
22 */
23#ifdef ENABLE_COMPAT_VDSO
24#include <asm/vdso/compat_gettimeofday.h>
25#else
26#include <asm/vdso/gettimeofday.h>
27#endif /* ENABLE_COMPAT_VDSO */
28
29#ifndef vdso_calc_delta
30/*
31 * Default implementation which works for all sane clocksources. That
32 * obviously excludes x86/TSC.
33 */
34static __always_inline
35u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
36{
37 return ((cycles - last) & mask) * mult;
38}
39#endif
40
41static int do_hres(const struct vdso_data *vd, clockid_t clk,
42 struct __kernel_timespec *ts)
43{
44 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
45 u64 cycles, last, sec, ns;
46 u32 seq;
47
48 do {
49 seq = vdso_read_begin(vd);
50 cycles = __arch_get_hw_counter(vd->clock_mode);
51 ns = vdso_ts->nsec;
52 last = vd->cycle_last;
53 if (unlikely((s64)cycles < 0))
54 return -1;
55
56 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
57 ns >>= vd->shift;
58 sec = vdso_ts->sec;
59 } while (unlikely(vdso_read_retry(vd, seq)));
60
61 /*
62 * Do this outside the loop: a race inside the loop could result
63 * in __iter_div_u64_rem() being extremely slow.
64 */
65 ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
66 ts->tv_nsec = ns;
67
68 return 0;
69}
70
71static void do_coarse(const struct vdso_data *vd, clockid_t clk,
72 struct __kernel_timespec *ts)
73{
74 const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
75 u32 seq;
76
77 do {
78 seq = vdso_read_begin(vd);
79 ts->tv_sec = vdso_ts->sec;
80 ts->tv_nsec = vdso_ts->nsec;
81 } while (unlikely(vdso_read_retry(vd, seq)));
82}
83
84static __maybe_unused int
85__cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
86{
87 const struct vdso_data *vd = __arch_get_vdso_data();
88 u32 msk;
89
90 /* Check for negative values or invalid clocks */
91 if (unlikely((u32) clock >= MAX_CLOCKS))
92 return -1;
93
94 /*
95 * Convert the clockid to a bitmask and use it to check which
96 * clocks are handled in the VDSO directly.
97 */
98 msk = 1U << clock;
99 if (likely(msk & VDSO_HRES)) {
100 return do_hres(&vd[CS_HRES_COARSE], clock, ts);
101 } else if (msk & VDSO_COARSE) {
102 do_coarse(&vd[CS_HRES_COARSE], clock, ts);
103 return 0;
104 } else if (msk & VDSO_RAW) {
105 return do_hres(&vd[CS_RAW], clock, ts);
106 }
107 return -1;
108}
109
110static __maybe_unused int
111__cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
112{
113 int ret = __cvdso_clock_gettime_common(clock, ts);
114
115 if (unlikely(ret))
116 return clock_gettime_fallback(clock, ts);
117 return 0;
118}
119
120static __maybe_unused int
121__cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
122{
123 struct __kernel_timespec ts;
124 int ret;
125
126 ret = __cvdso_clock_gettime_common(clock, &ts);
127
128#ifdef VDSO_HAS_32BIT_FALLBACK
129 if (unlikely(ret))
130 return clock_gettime32_fallback(clock, res);
131#else
132 if (unlikely(ret))
133 ret = clock_gettime_fallback(clock, &ts);
134#endif
135
136 if (likely(!ret)) {
137 res->tv_sec = ts.tv_sec;
138 res->tv_nsec = ts.tv_nsec;
139 }
140 return ret;
141}
142
143static __maybe_unused int
144__cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
145{
146 const struct vdso_data *vd = __arch_get_vdso_data();
147
148 if (likely(tv != NULL)) {
149 struct __kernel_timespec ts;
150
151 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
152 return gettimeofday_fallback(tv, tz);
153
154 tv->tv_sec = ts.tv_sec;
155 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
156 }
157
158 if (unlikely(tz != NULL)) {
159 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
160 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
161 }
162
163 return 0;
164}
165
166#ifdef VDSO_HAS_TIME
167static __maybe_unused time_t __cvdso_time(time_t *time)
168{
169 const struct vdso_data *vd = __arch_get_vdso_data();
170 time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
171
172 if (time)
173 *time = t;
174
175 return t;
176}
177#endif /* VDSO_HAS_TIME */
178
179#ifdef VDSO_HAS_CLOCK_GETRES
180static __maybe_unused
181int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
182{
183 const struct vdso_data *vd = __arch_get_vdso_data();
184 u64 hrtimer_res;
185 u32 msk;
186 u64 ns;
187
188 /* Check for negative values or invalid clocks */
189 if (unlikely((u32) clock >= MAX_CLOCKS))
190 return -1;
191
192 hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
193 /*
194 * Convert the clockid to a bitmask and use it to check which
195 * clocks are handled in the VDSO directly.
196 */
197 msk = 1U << clock;
198 if (msk & VDSO_HRES) {
199 /*
200 * Preserves the behaviour of posix_get_hrtimer_res().
201 */
202 ns = hrtimer_res;
203 } else if (msk & VDSO_COARSE) {
204 /*
205 * Preserves the behaviour of posix_get_coarse_res().
206 */
207 ns = LOW_RES_NSEC;
208 } else if (msk & VDSO_RAW) {
209 /*
210 * Preserves the behaviour of posix_get_hrtimer_res().
211 */
212 ns = hrtimer_res;
213 } else {
214 return -1;
215 }
216
217 if (likely(res)) {
218 res->tv_sec = 0;
219 res->tv_nsec = ns;
220 }
221 return 0;
222}
223
224int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
225{
226 int ret = __cvdso_clock_getres_common(clock, res);
227
228 if (unlikely(ret))
229 return clock_getres_fallback(clock, res);
230 return 0;
231}
232
233static __maybe_unused int
234__cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
235{
236 struct __kernel_timespec ts;
237 int ret;
238
239 ret = __cvdso_clock_getres_common(clock, &ts);
240
241#ifdef VDSO_HAS_32BIT_FALLBACK
242 if (unlikely(ret))
243 return clock_getres32_fallback(clock, res);
244#else
245 if (unlikely(ret))
246 ret = clock_getres_fallback(clock, &ts);
247#endif
248
249 if (likely(!ret && res)) {
250 res->tv_sec = ts.tv_sec;
251 res->tv_nsec = ts.tv_nsec;
252 }
253 return ret;
254}
255#endif /* VDSO_HAS_CLOCK_GETRES */