1/*
  2 * sched_clock.h: support for extending counters to full 64-bit ns counter
  3 *
  4 * This program is free software; you can redistribute it and/or modify
  5 * it under the terms of the GNU General Public License version 2 as
  6 * published by the Free Software Foundation.
  7 */
  8#ifndef ASM_SCHED_CLOCK
  9#define ASM_SCHED_CLOCK
 10
 11#include <linux/kernel.h>
 12#include <linux/types.h>
 13
 14struct clock_data {
 15	u64 epoch_ns;
 16	u32 epoch_cyc;
 17	u32 epoch_cyc_copy;
 18	u32 mult;
 19	u32 shift;
 20};
 21
 22#define DEFINE_CLOCK_DATA(name)	struct clock_data name
 23
 24static inline u64 cyc_to_ns(u64 cyc, u32 mult, u32 shift)
 25{
 26	return (cyc * mult) >> shift;
 27}
 28
 29/*
 30 * Atomically update the sched_clock epoch.  Your update callback will
 31 * be called from a timer before the counter wraps - read the current
 32 * counter value, and call this function to safely move the epochs
 33 * forward.  Only use this from the update callback.
 34 */
 35static inline void update_sched_clock(struct clock_data *cd, u32 cyc, u32 mask)
 36{
 37	unsigned long flags;
 38	u64 ns = cd->epoch_ns +
 39		cyc_to_ns((cyc - cd->epoch_cyc) & mask, cd->mult, cd->shift);
 40
 41	/*
 42	 * Write epoch_cyc and epoch_ns in a way that the update is
 43	 * detectable in cyc_to_fixed_sched_clock().
 44	 */
 45	raw_local_irq_save(flags);
 46	cd->epoch_cyc = cyc;
 47	smp_wmb();
 48	cd->epoch_ns = ns;
 49	smp_wmb();
 50	cd->epoch_cyc_copy = cyc;
 51	raw_local_irq_restore(flags);
 52}
 53
 54/*
 55 * If your clock rate is known at compile time, using this will allow
 56 * you to optimize the mult/shift loads away.  This is paired with
 57 * init_fixed_sched_clock() to ensure that your mult/shift are correct.
 58 */
 59static inline unsigned long long cyc_to_fixed_sched_clock(struct clock_data *cd,
 60	u32 cyc, u32 mask, u32 mult, u32 shift)
 61{
 62	u64 epoch_ns;
 63	u32 epoch_cyc;
 64
 65	/*
 66	 * Load the epoch_cyc and epoch_ns atomically.  We do this by
 67	 * ensuring that we always write epoch_cyc, epoch_ns and
 68	 * epoch_cyc_copy in strict order, and read them in strict order.
 69	 * If epoch_cyc and epoch_cyc_copy are not equal, then we're in
 70	 * the middle of an update, and we should repeat the load.
 71	 */
 72	do {
 73		epoch_cyc = cd->epoch_cyc;
 74		smp_rmb();
 75		epoch_ns = cd->epoch_ns;
 76		smp_rmb();
 77	} while (epoch_cyc != cd->epoch_cyc_copy);
 78
 79	return epoch_ns + cyc_to_ns((cyc - epoch_cyc) & mask, mult, shift);
 80}
 81
 82/*
 83 * Otherwise, you need to use this, which will obtain the mult/shift
 84 * from the clock_data structure.  Use init_sched_clock() with this.
 85 */
 86static inline unsigned long long cyc_to_sched_clock(struct clock_data *cd,
 87	u32 cyc, u32 mask)
 88{
 89	return cyc_to_fixed_sched_clock(cd, cyc, mask, cd->mult, cd->shift);
 90}
 91
 92/*
 93 * Initialize the clock data - calculate the appropriate multiplier
 94 * and shift.  Also setup a timer to ensure that the epoch is refreshed
 95 * at the appropriate time interval, which will call your update
 96 * handler.
 97 */
 98void init_sched_clock(struct clock_data *, void (*)(void),
 99	unsigned int, unsigned long);
100
101/*
102 * Use this initialization function rather than init_sched_clock() if
103 * you're using cyc_to_fixed_sched_clock, which will warn if your
104 * constants are incorrect.
105 */
106static inline void init_fixed_sched_clock(struct clock_data *cd,
107	void (*update)(void), unsigned int bits, unsigned long rate,
108	u32 mult, u32 shift)
109{
110	init_sched_clock(cd, update, bits, rate);
111	if (cd->mult != mult || cd->shift != shift) {
112		pr_crit("sched_clock: wrong multiply/shift: %u>>%u vs calculated %u>>%u\n"
113			"sched_clock: fix multiply/shift to avoid scheduler hiccups\n",
114			mult, shift, cd->mult, cd->shift);
115	}
116}
117
118extern void sched_clock_postinit(void);
119
120#endif