1/* SPDX-License-Identifier: GPL-2.0 */
 2#ifndef __VDSO_HELPERS_H
 3#define __VDSO_HELPERS_H
 4
 5#ifndef __ASSEMBLY__
 6
 
 7#include <vdso/datapage.h>
 8
 9static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
10{
11	u32 seq;
12
13	while ((seq = READ_ONCE(vd->seq)) & 1)
14		cpu_relax();
15
16	smp_rmb();
17	return seq;
18}
19
20static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
21					   u32 start)
22{
23	u32 seq;
24
25	smp_rmb();
26	seq = READ_ONCE(vd->seq);
27	return seq != start;
28}
29
30static __always_inline void vdso_write_begin(struct vdso_data *vd)
31{
32	/*
33	 * WRITE_ONCE it is required otherwise the compiler can validly tear
34	 * updates to vd[x].seq and it is possible that the value seen by the
35	 * reader it is inconsistent.
36	 */
37	WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
38	WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
39	smp_wmb();
40}
41
42static __always_inline void vdso_write_end(struct vdso_data *vd)
43{
44	smp_wmb();
45	/*
46	 * WRITE_ONCE it is required otherwise the compiler can validly tear
47	 * updates to vd[x].seq and it is possible that the value seen by the
48	 * reader it is inconsistent.
49	 */
50	WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
51	WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
52}
53
54#endif /* !__ASSEMBLY__ */
55
56#endif /* __VDSO_HELPERS_H */

 1/* SPDX-License-Identifier: GPL-2.0 */
 2#ifndef __VDSO_HELPERS_H
 3#define __VDSO_HELPERS_H
 4
 5#ifndef __ASSEMBLY__
 6
 7#include <asm/barrier.h>
 8#include <vdso/datapage.h>
 9
10static __always_inline u32 vdso_read_begin(const struct vdso_data *vd)
11{
12	u32 seq;
13
14	while (unlikely((seq = READ_ONCE(vd->seq)) & 1))
15		cpu_relax();
16
17	smp_rmb();
18	return seq;
19}
20
21static __always_inline u32 vdso_read_retry(const struct vdso_data *vd,
22					   u32 start)
23{
24	u32 seq;
25
26	smp_rmb();
27	seq = READ_ONCE(vd->seq);
28	return seq != start;
29}
30
31static __always_inline void vdso_write_begin(struct vdso_data *vd)
32{
33	/*
34	 * WRITE_ONCE() is required otherwise the compiler can validly tear
35	 * updates to vd[x].seq and it is possible that the value seen by the
36	 * reader is inconsistent.
37	 */
38	WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
39	WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
40	smp_wmb();
41}
42
43static __always_inline void vdso_write_end(struct vdso_data *vd)
44{
45	smp_wmb();
46	/*
47	 * WRITE_ONCE() is required otherwise the compiler can validly tear
48	 * updates to vd[x].seq and it is possible that the value seen by the
49	 * reader is inconsistent.
50	 */
51	WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1);
52	WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1);
53}
54
55#endif /* !__ASSEMBLY__ */
56
57#endif /* __VDSO_HELPERS_H */