blob: 2b021b0e8507e7e4f9951780c88877bb2d4d56bd [file] [log] [blame]
john stultz85240702007-05-08 00:27:59 -07001/*
2 * linux/kernel/time/timekeeping.c
3 *
4 * Kernel timekeeping code and accessor functions
5 *
6 * This code was moved from linux/kernel/timer.c.
7 * Please see that file for copyright and history logs.
8 *
9 */
10
11#include <linux/module.h>
12#include <linux/interrupt.h>
13#include <linux/percpu.h>
14#include <linux/init.h>
15#include <linux/mm.h>
Alexey Dobriyand43c36d2009-10-07 17:09:06 +040016#include <linux/sched.h>
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +010017#include <linux/syscore_ops.h>
john stultz85240702007-05-08 00:27:59 -070018#include <linux/clocksource.h>
19#include <linux/jiffies.h>
20#include <linux/time.h>
21#include <linux/tick.h>
Martin Schwidefsky75c51582009-08-14 15:47:30 +020022#include <linux/stop_machine.h>
john stultz85240702007-05-08 00:27:59 -070023
Martin Schwidefsky155ec602009-08-14 15:47:26 +020024/* Structure holding internal timekeeping values. */
25struct timekeeper {
26 /* Current clocksource used for timekeeping. */
27 struct clocksource *clock;
Martin Schwidefsky23ce7212009-08-14 15:47:27 +020028 /* The shift value of the current clocksource. */
29 int shift;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020030
31 /* Number of clock cycles in one NTP interval. */
32 cycle_t cycle_interval;
33 /* Number of clock shifted nano seconds in one NTP interval. */
34 u64 xtime_interval;
Kasper Pedersena386b5a2010-10-20 15:55:15 -070035 /* shifted nano seconds left over when rounding cycle_interval */
36 s64 xtime_remainder;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020037 /* Raw nano seconds accumulated per NTP interval. */
38 u32 raw_interval;
39
40 /* Clock shifted nano seconds remainder not stored in xtime.tv_nsec. */
41 u64 xtime_nsec;
42 /* Difference between accumulated time and NTP time in ntp
43 * shifted nano seconds. */
44 s64 ntp_error;
Martin Schwidefsky23ce7212009-08-14 15:47:27 +020045 /* Shift conversion between clock shifted nano seconds and
46 * ntp shifted nano seconds. */
47 int ntp_error_shift;
Martin Schwidefsky0a544192009-08-14 15:47:28 +020048 /* NTP adjusted clock multiplier */
49 u32 mult;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020050};
51
H Hartley Sweetenafa14e72011-01-11 17:59:38 -060052static struct timekeeper timekeeper;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020053
54/**
55 * timekeeper_setup_internals - Set up internals to use clocksource clock.
56 *
57 * @clock: Pointer to clocksource.
58 *
59 * Calculates a fixed cycle/nsec interval for a given clocksource/adjustment
60 * pair and interval request.
61 *
62 * Unless you're the timekeeping code, you should not be using this!
63 */
64static void timekeeper_setup_internals(struct clocksource *clock)
65{
66 cycle_t interval;
Kasper Pedersena386b5a2010-10-20 15:55:15 -070067 u64 tmp, ntpinterval;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020068
69 timekeeper.clock = clock;
70 clock->cycle_last = clock->read(clock);
71
72 /* Do the ns -> cycle conversion first, using original mult */
73 tmp = NTP_INTERVAL_LENGTH;
74 tmp <<= clock->shift;
Kasper Pedersena386b5a2010-10-20 15:55:15 -070075 ntpinterval = tmp;
Martin Schwidefsky0a544192009-08-14 15:47:28 +020076 tmp += clock->mult/2;
77 do_div(tmp, clock->mult);
Martin Schwidefsky155ec602009-08-14 15:47:26 +020078 if (tmp == 0)
79 tmp = 1;
80
81 interval = (cycle_t) tmp;
82 timekeeper.cycle_interval = interval;
83
84 /* Go back from cycles -> shifted ns */
85 timekeeper.xtime_interval = (u64) interval * clock->mult;
Kasper Pedersena386b5a2010-10-20 15:55:15 -070086 timekeeper.xtime_remainder = ntpinterval - timekeeper.xtime_interval;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020087 timekeeper.raw_interval =
Martin Schwidefsky0a544192009-08-14 15:47:28 +020088 ((u64) interval * clock->mult) >> clock->shift;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020089
90 timekeeper.xtime_nsec = 0;
Martin Schwidefsky23ce7212009-08-14 15:47:27 +020091 timekeeper.shift = clock->shift;
Martin Schwidefsky155ec602009-08-14 15:47:26 +020092
93 timekeeper.ntp_error = 0;
Martin Schwidefsky23ce7212009-08-14 15:47:27 +020094 timekeeper.ntp_error_shift = NTP_SCALE_SHIFT - clock->shift;
Martin Schwidefsky0a544192009-08-14 15:47:28 +020095
96 /*
97 * The timekeeper keeps its own mult values for the currently
98 * active clocksource. These value will be adjusted via NTP
99 * to counteract clock drifting.
100 */
101 timekeeper.mult = clock->mult;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200102}
john stultz85240702007-05-08 00:27:59 -0700103
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200104/* Timekeeper helper functions. */
105static inline s64 timekeeping_get_ns(void)
106{
107 cycle_t cycle_now, cycle_delta;
108 struct clocksource *clock;
109
110 /* read clocksource: */
111 clock = timekeeper.clock;
112 cycle_now = clock->read(clock);
113
114 /* calculate the delta since the last update_wall_time: */
115 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
116
117 /* return delta convert to nanoseconds using ntp adjusted mult. */
118 return clocksource_cyc2ns(cycle_delta, timekeeper.mult,
119 timekeeper.shift);
120}
121
122static inline s64 timekeeping_get_ns_raw(void)
123{
124 cycle_t cycle_now, cycle_delta;
125 struct clocksource *clock;
126
127 /* read clocksource: */
128 clock = timekeeper.clock;
129 cycle_now = clock->read(clock);
130
131 /* calculate the delta since the last update_wall_time: */
132 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
133
134 /* return delta convert to nanoseconds using ntp adjusted mult. */
135 return clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
136}
137
john stultz85240702007-05-08 00:27:59 -0700138/*
139 * This read-write spinlock protects us from races in SMP while
Thomas Gleixnerdce48a82009-04-11 10:43:41 +0200140 * playing with xtime.
john stultz85240702007-05-08 00:27:59 -0700141 */
Adrian Bunkba2a6312007-10-16 23:27:16 -0700142__cacheline_aligned_in_smp DEFINE_SEQLOCK(xtime_lock);
john stultz85240702007-05-08 00:27:59 -0700143
144
145/*
146 * The current time
147 * wall_to_monotonic is what we need to add to xtime (or xtime corrected
148 * for sub jiffie times) to get to monotonic time. Monotonic is pegged
149 * at zero at system boot time, so wall_to_monotonic will be negative,
150 * however, we will ALWAYS keep the tv_nsec part positive so we can use
151 * the usual normalization.
Tomas Janousek7c3f1a52007-07-15 23:39:41 -0700152 *
153 * wall_to_monotonic is moved after resume from suspend for the monotonic
154 * time not to jump. We need to add total_sleep_time to wall_to_monotonic
155 * to get the real boot based time offset.
156 *
157 * - wall_to_monotonic is no longer the boot time, getboottime must be
158 * used instead.
john stultz85240702007-05-08 00:27:59 -0700159 */
John Stultz0fb86b02010-07-13 17:56:26 -0700160static struct timespec xtime __attribute__ ((aligned (16)));
161static struct timespec wall_to_monotonic __attribute__ ((aligned (16)));
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200162static struct timespec total_sleep_time;
john stultz85240702007-05-08 00:27:59 -0700163
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200164/*
165 * The raw monotonic time for the CLOCK_MONOTONIC_RAW posix clock.
166 */
H Hartley Sweetenafa14e72011-01-11 17:59:38 -0600167static struct timespec raw_time;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200168
Thomas Gleixner1c5745a2008-12-22 23:05:28 +0100169/* flag for if timekeeping is suspended */
170int __read_mostly timekeeping_suspended;
171
John Stultz31089c12009-08-14 15:47:18 +0200172/* must hold xtime_lock */
173void timekeeping_leap_insert(int leapsecond)
174{
175 xtime.tv_sec += leapsecond;
176 wall_to_monotonic.tv_sec -= leapsecond;
John Stultz76158562010-07-13 17:56:23 -0700177 update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
178 timekeeper.mult);
John Stultz31089c12009-08-14 15:47:18 +0200179}
john stultz85240702007-05-08 00:27:59 -0700180
john stultz85240702007-05-08 00:27:59 -0700181/**
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200182 * timekeeping_forward_now - update clock to the current time
john stultz85240702007-05-08 00:27:59 -0700183 *
Roman Zippel9a055112008-08-20 16:37:28 -0700184 * Forward the current clock to update its state since the last call to
185 * update_wall_time(). This is useful before significant clock changes,
186 * as it avoids having to deal with this time offset explicitly.
john stultz85240702007-05-08 00:27:59 -0700187 */
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200188static void timekeeping_forward_now(void)
john stultz85240702007-05-08 00:27:59 -0700189{
190 cycle_t cycle_now, cycle_delta;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200191 struct clocksource *clock;
Roman Zippel9a055112008-08-20 16:37:28 -0700192 s64 nsec;
john stultz85240702007-05-08 00:27:59 -0700193
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200194 clock = timekeeper.clock;
Martin Schwidefskya0f7d482009-08-14 15:47:19 +0200195 cycle_now = clock->read(clock);
john stultz85240702007-05-08 00:27:59 -0700196 cycle_delta = (cycle_now - clock->cycle_last) & clock->mask;
Roman Zippel9a055112008-08-20 16:37:28 -0700197 clock->cycle_last = cycle_now;
john stultz85240702007-05-08 00:27:59 -0700198
Martin Schwidefsky0a544192009-08-14 15:47:28 +0200199 nsec = clocksource_cyc2ns(cycle_delta, timekeeper.mult,
200 timekeeper.shift);
john stultz7d275582009-05-01 13:10:26 -0700201
202 /* If arch requires, add in gettimeoffset() */
203 nsec += arch_gettimeoffset();
204
Roman Zippel9a055112008-08-20 16:37:28 -0700205 timespec_add_ns(&xtime, nsec);
John Stultz2d422442008-08-20 16:37:30 -0700206
Martin Schwidefsky0a544192009-08-14 15:47:28 +0200207 nsec = clocksource_cyc2ns(cycle_delta, clock->mult, clock->shift);
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200208 timespec_add_ns(&raw_time, nsec);
john stultz85240702007-05-08 00:27:59 -0700209}
210
211/**
Geert Uytterhoevenefd9ac82008-01-30 13:30:01 +0100212 * getnstimeofday - Returns the time of day in a timespec
john stultz85240702007-05-08 00:27:59 -0700213 * @ts: pointer to the timespec to be set
214 *
Geert Uytterhoevenefd9ac82008-01-30 13:30:01 +0100215 * Returns the time of day in a timespec.
john stultz85240702007-05-08 00:27:59 -0700216 */
Geert Uytterhoevenefd9ac82008-01-30 13:30:01 +0100217void getnstimeofday(struct timespec *ts)
john stultz85240702007-05-08 00:27:59 -0700218{
219 unsigned long seq;
220 s64 nsecs;
221
Thomas Gleixner1c5745a2008-12-22 23:05:28 +0100222 WARN_ON(timekeeping_suspended);
223
john stultz85240702007-05-08 00:27:59 -0700224 do {
225 seq = read_seqbegin(&xtime_lock);
226
227 *ts = xtime;
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200228 nsecs = timekeeping_get_ns();
john stultz85240702007-05-08 00:27:59 -0700229
john stultz7d275582009-05-01 13:10:26 -0700230 /* If arch requires, add in gettimeoffset() */
231 nsecs += arch_gettimeoffset();
232
john stultz85240702007-05-08 00:27:59 -0700233 } while (read_seqretry(&xtime_lock, seq));
234
235 timespec_add_ns(ts, nsecs);
236}
237
john stultz85240702007-05-08 00:27:59 -0700238EXPORT_SYMBOL(getnstimeofday);
239
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200240ktime_t ktime_get(void)
241{
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200242 unsigned int seq;
243 s64 secs, nsecs;
244
245 WARN_ON(timekeeping_suspended);
246
247 do {
248 seq = read_seqbegin(&xtime_lock);
249 secs = xtime.tv_sec + wall_to_monotonic.tv_sec;
250 nsecs = xtime.tv_nsec + wall_to_monotonic.tv_nsec;
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200251 nsecs += timekeeping_get_ns();
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200252
253 } while (read_seqretry(&xtime_lock, seq));
254 /*
255 * Use ktime_set/ktime_add_ns to create a proper ktime on
256 * 32-bit architectures without CONFIG_KTIME_SCALAR.
257 */
258 return ktime_add_ns(ktime_set(secs, 0), nsecs);
259}
260EXPORT_SYMBOL_GPL(ktime_get);
261
262/**
263 * ktime_get_ts - get the monotonic clock in timespec format
264 * @ts: pointer to timespec variable
265 *
266 * The function calculates the monotonic clock from the realtime
267 * clock and the wall_to_monotonic offset and stores the result
268 * in normalized timespec format in the variable pointed to by @ts.
269 */
270void ktime_get_ts(struct timespec *ts)
271{
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200272 struct timespec tomono;
273 unsigned int seq;
274 s64 nsecs;
275
276 WARN_ON(timekeeping_suspended);
277
278 do {
279 seq = read_seqbegin(&xtime_lock);
280 *ts = xtime;
281 tomono = wall_to_monotonic;
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200282 nsecs = timekeeping_get_ns();
Martin Schwidefsky951ed4d2009-07-07 11:27:28 +0200283
284 } while (read_seqretry(&xtime_lock, seq));
285
286 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec,
287 ts->tv_nsec + tomono.tv_nsec + nsecs);
288}
289EXPORT_SYMBOL_GPL(ktime_get_ts);
290
Alexander Gordeeve2c18e42011-01-12 17:00:57 -0800291#ifdef CONFIG_NTP_PPS
292
293/**
294 * getnstime_raw_and_real - get day and raw monotonic time in timespec format
295 * @ts_raw: pointer to the timespec to be set to raw monotonic time
296 * @ts_real: pointer to the timespec to be set to the time of day
297 *
298 * This function reads both the time of day and raw monotonic time at the
299 * same time atomically and stores the resulting timestamps in timespec
300 * format.
301 */
302void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real)
303{
304 unsigned long seq;
305 s64 nsecs_raw, nsecs_real;
306
307 WARN_ON_ONCE(timekeeping_suspended);
308
309 do {
310 u32 arch_offset;
311
312 seq = read_seqbegin(&xtime_lock);
313
314 *ts_raw = raw_time;
315 *ts_real = xtime;
316
317 nsecs_raw = timekeeping_get_ns_raw();
318 nsecs_real = timekeeping_get_ns();
319
320 /* If arch requires, add in gettimeoffset() */
321 arch_offset = arch_gettimeoffset();
322 nsecs_raw += arch_offset;
323 nsecs_real += arch_offset;
324
325 } while (read_seqretry(&xtime_lock, seq));
326
327 timespec_add_ns(ts_raw, nsecs_raw);
328 timespec_add_ns(ts_real, nsecs_real);
329}
330EXPORT_SYMBOL(getnstime_raw_and_real);
331
332#endif /* CONFIG_NTP_PPS */
333
john stultz85240702007-05-08 00:27:59 -0700334/**
335 * do_gettimeofday - Returns the time of day in a timeval
336 * @tv: pointer to the timeval to be set
337 *
Geert Uytterhoevenefd9ac82008-01-30 13:30:01 +0100338 * NOTE: Users should be converted to using getnstimeofday()
john stultz85240702007-05-08 00:27:59 -0700339 */
340void do_gettimeofday(struct timeval *tv)
341{
342 struct timespec now;
343
Geert Uytterhoevenefd9ac82008-01-30 13:30:01 +0100344 getnstimeofday(&now);
john stultz85240702007-05-08 00:27:59 -0700345 tv->tv_sec = now.tv_sec;
346 tv->tv_usec = now.tv_nsec/1000;
347}
348
349EXPORT_SYMBOL(do_gettimeofday);
350/**
351 * do_settimeofday - Sets the time of day
352 * @tv: pointer to the timespec variable containing the new time
353 *
354 * Sets the time of day to the new time and update NTP and notify hrtimers
355 */
Richard Cochran1e6d7672011-02-01 13:50:58 +0000356int do_settimeofday(const struct timespec *tv)
john stultz85240702007-05-08 00:27:59 -0700357{
Roman Zippel9a055112008-08-20 16:37:28 -0700358 struct timespec ts_delta;
john stultz85240702007-05-08 00:27:59 -0700359 unsigned long flags;
john stultz85240702007-05-08 00:27:59 -0700360
361 if ((unsigned long)tv->tv_nsec >= NSEC_PER_SEC)
362 return -EINVAL;
363
364 write_seqlock_irqsave(&xtime_lock, flags);
365
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200366 timekeeping_forward_now();
john stultz85240702007-05-08 00:27:59 -0700367
Roman Zippel9a055112008-08-20 16:37:28 -0700368 ts_delta.tv_sec = tv->tv_sec - xtime.tv_sec;
369 ts_delta.tv_nsec = tv->tv_nsec - xtime.tv_nsec;
370 wall_to_monotonic = timespec_sub(wall_to_monotonic, ts_delta);
john stultz85240702007-05-08 00:27:59 -0700371
Roman Zippel9a055112008-08-20 16:37:28 -0700372 xtime = *tv;
373
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200374 timekeeper.ntp_error = 0;
john stultz85240702007-05-08 00:27:59 -0700375 ntp_clear();
376
John Stultz76158562010-07-13 17:56:23 -0700377 update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
378 timekeeper.mult);
john stultz85240702007-05-08 00:27:59 -0700379
380 write_sequnlock_irqrestore(&xtime_lock, flags);
381
382 /* signal hrtimers about time change */
383 clock_was_set();
384
385 return 0;
386}
387
388EXPORT_SYMBOL(do_settimeofday);
389
John Stultzc528f7c2011-02-01 13:52:17 +0000390
391/**
392 * timekeeping_inject_offset - Adds or subtracts from the current time.
393 * @tv: pointer to the timespec variable containing the offset
394 *
395 * Adds or subtracts an offset value from the current time.
396 */
397int timekeeping_inject_offset(struct timespec *ts)
398{
399 unsigned long flags;
400
401 if ((unsigned long)ts->tv_nsec >= NSEC_PER_SEC)
402 return -EINVAL;
403
404 write_seqlock_irqsave(&xtime_lock, flags);
405
406 timekeeping_forward_now();
407
408 xtime = timespec_add(xtime, *ts);
409 wall_to_monotonic = timespec_sub(wall_to_monotonic, *ts);
410
411 timekeeper.ntp_error = 0;
412 ntp_clear();
413
414 update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
415 timekeeper.mult);
416
417 write_sequnlock_irqrestore(&xtime_lock, flags);
418
419 /* signal hrtimers about time change */
420 clock_was_set();
421
422 return 0;
423}
424EXPORT_SYMBOL(timekeeping_inject_offset);
425
john stultz85240702007-05-08 00:27:59 -0700426/**
427 * change_clocksource - Swaps clocksources if a new one is available
428 *
429 * Accumulates current time interval and initializes new clocksource
430 */
Martin Schwidefsky75c51582009-08-14 15:47:30 +0200431static int change_clocksource(void *data)
john stultz85240702007-05-08 00:27:59 -0700432{
Magnus Damm4614e6a2009-04-21 12:24:02 -0700433 struct clocksource *new, *old;
john stultz85240702007-05-08 00:27:59 -0700434
Martin Schwidefsky75c51582009-08-14 15:47:30 +0200435 new = (struct clocksource *) data;
john stultz85240702007-05-08 00:27:59 -0700436
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200437 timekeeping_forward_now();
Martin Schwidefsky75c51582009-08-14 15:47:30 +0200438 if (!new->enable || new->enable(new) == 0) {
439 old = timekeeper.clock;
440 timekeeper_setup_internals(new);
441 if (old->disable)
442 old->disable(old);
443 }
444 return 0;
445}
john stultz85240702007-05-08 00:27:59 -0700446
Martin Schwidefsky75c51582009-08-14 15:47:30 +0200447/**
448 * timekeeping_notify - Install a new clock source
449 * @clock: pointer to the clock source
450 *
451 * This function is called from clocksource.c after a new, better clock
452 * source has been registered. The caller holds the clocksource_mutex.
453 */
454void timekeeping_notify(struct clocksource *clock)
455{
456 if (timekeeper.clock == clock)
Magnus Damm4614e6a2009-04-21 12:24:02 -0700457 return;
Martin Schwidefsky75c51582009-08-14 15:47:30 +0200458 stop_machine(change_clocksource, clock, NULL);
john stultz85240702007-05-08 00:27:59 -0700459 tick_clock_notify();
john stultz85240702007-05-08 00:27:59 -0700460}
Martin Schwidefsky75c51582009-08-14 15:47:30 +0200461
Thomas Gleixnera40f2622009-07-07 13:00:31 +0200462/**
463 * ktime_get_real - get the real (wall-) time in ktime_t format
464 *
465 * returns the time in ktime_t format
466 */
467ktime_t ktime_get_real(void)
468{
469 struct timespec now;
470
471 getnstimeofday(&now);
472
473 return timespec_to_ktime(now);
474}
475EXPORT_SYMBOL_GPL(ktime_get_real);
john stultz85240702007-05-08 00:27:59 -0700476
477/**
John Stultz2d422442008-08-20 16:37:30 -0700478 * getrawmonotonic - Returns the raw monotonic time in a timespec
479 * @ts: pointer to the timespec to be set
480 *
481 * Returns the raw monotonic time (completely un-modified by ntp)
482 */
483void getrawmonotonic(struct timespec *ts)
484{
485 unsigned long seq;
486 s64 nsecs;
John Stultz2d422442008-08-20 16:37:30 -0700487
488 do {
489 seq = read_seqbegin(&xtime_lock);
Martin Schwidefsky2ba2a302009-08-14 15:47:29 +0200490 nsecs = timekeeping_get_ns_raw();
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200491 *ts = raw_time;
John Stultz2d422442008-08-20 16:37:30 -0700492
493 } while (read_seqretry(&xtime_lock, seq));
494
495 timespec_add_ns(ts, nsecs);
496}
497EXPORT_SYMBOL(getrawmonotonic);
498
499
500/**
Li Zefancf4fc6c2008-02-08 04:19:24 -0800501 * timekeeping_valid_for_hres - Check if timekeeping is suitable for hres
john stultz85240702007-05-08 00:27:59 -0700502 */
Li Zefancf4fc6c2008-02-08 04:19:24 -0800503int timekeeping_valid_for_hres(void)
john stultz85240702007-05-08 00:27:59 -0700504{
505 unsigned long seq;
506 int ret;
507
508 do {
509 seq = read_seqbegin(&xtime_lock);
510
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200511 ret = timekeeper.clock->flags & CLOCK_SOURCE_VALID_FOR_HRES;
john stultz85240702007-05-08 00:27:59 -0700512
513 } while (read_seqretry(&xtime_lock, seq));
514
515 return ret;
516}
517
518/**
Jon Hunter98962462009-08-18 12:45:10 -0500519 * timekeeping_max_deferment - Returns max time the clocksource can be deferred
520 *
521 * Caller must observe xtime_lock via read_seqbegin/read_seqretry to
522 * ensure that the clocksource does not change!
523 */
524u64 timekeeping_max_deferment(void)
525{
526 return timekeeper.clock->max_idle_ns;
527}
528
529/**
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200530 * read_persistent_clock - Return time from the persistent clock.
john stultz85240702007-05-08 00:27:59 -0700531 *
532 * Weak dummy function for arches that do not yet support it.
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200533 * Reads the time from the battery backed persistent clock.
534 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
john stultz85240702007-05-08 00:27:59 -0700535 *
536 * XXX - Do be sure to remove it once all arches implement it.
537 */
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200538void __attribute__((weak)) read_persistent_clock(struct timespec *ts)
john stultz85240702007-05-08 00:27:59 -0700539{
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200540 ts->tv_sec = 0;
541 ts->tv_nsec = 0;
john stultz85240702007-05-08 00:27:59 -0700542}
543
Martin Schwidefsky23970e32009-08-14 15:47:32 +0200544/**
545 * read_boot_clock - Return time of the system start.
546 *
547 * Weak dummy function for arches that do not yet support it.
548 * Function to read the exact time the system has been started.
549 * Returns a timespec with tv_sec=0 and tv_nsec=0 if unsupported.
550 *
551 * XXX - Do be sure to remove it once all arches implement it.
552 */
553void __attribute__((weak)) read_boot_clock(struct timespec *ts)
554{
555 ts->tv_sec = 0;
556 ts->tv_nsec = 0;
557}
558
john stultz85240702007-05-08 00:27:59 -0700559/*
560 * timekeeping_init - Initializes the clocksource and common timekeeping values
561 */
562void __init timekeeping_init(void)
563{
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200564 struct clocksource *clock;
john stultz85240702007-05-08 00:27:59 -0700565 unsigned long flags;
Martin Schwidefsky23970e32009-08-14 15:47:32 +0200566 struct timespec now, boot;
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200567
568 read_persistent_clock(&now);
Martin Schwidefsky23970e32009-08-14 15:47:32 +0200569 read_boot_clock(&boot);
john stultz85240702007-05-08 00:27:59 -0700570
571 write_seqlock_irqsave(&xtime_lock, flags);
572
Roman Zippel7dffa3c2008-05-01 04:34:41 -0700573 ntp_init();
john stultz85240702007-05-08 00:27:59 -0700574
Martin Schwidefskyf1b82742009-08-14 15:47:21 +0200575 clock = clocksource_default_clock();
Martin Schwidefskya0f7d482009-08-14 15:47:19 +0200576 if (clock->enable)
577 clock->enable(clock);
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200578 timekeeper_setup_internals(clock);
john stultz85240702007-05-08 00:27:59 -0700579
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200580 xtime.tv_sec = now.tv_sec;
581 xtime.tv_nsec = now.tv_nsec;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200582 raw_time.tv_sec = 0;
583 raw_time.tv_nsec = 0;
Martin Schwidefsky23970e32009-08-14 15:47:32 +0200584 if (boot.tv_sec == 0 && boot.tv_nsec == 0) {
585 boot.tv_sec = xtime.tv_sec;
586 boot.tv_nsec = xtime.tv_nsec;
587 }
john stultz85240702007-05-08 00:27:59 -0700588 set_normalized_timespec(&wall_to_monotonic,
Martin Schwidefsky23970e32009-08-14 15:47:32 +0200589 -boot.tv_sec, -boot.tv_nsec);
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200590 total_sleep_time.tv_sec = 0;
591 total_sleep_time.tv_nsec = 0;
john stultz85240702007-05-08 00:27:59 -0700592 write_sequnlock_irqrestore(&xtime_lock, flags);
593}
594
john stultz85240702007-05-08 00:27:59 -0700595/* time in seconds when suspend began */
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200596static struct timespec timekeeping_suspend_time;
john stultz85240702007-05-08 00:27:59 -0700597
598/**
John Stultz304529b2011-04-01 14:32:09 -0700599 * __timekeeping_inject_sleeptime - Internal function to add sleep interval
600 * @delta: pointer to a timespec delta value
601 *
602 * Takes a timespec offset measuring a suspend interval and properly
603 * adds the sleep offset to the timekeeping variables.
604 */
605static void __timekeeping_inject_sleeptime(struct timespec *delta)
606{
John Stultzcb5de2f8d2011-06-01 18:18:09 -0700607 if (!timespec_valid(delta)) {
John Stultzcbaa5152011-07-20 15:42:55 -0700608 printk(KERN_WARNING "__timekeeping_inject_sleeptime: Invalid "
John Stultzcb5de2f8d2011-06-01 18:18:09 -0700609 "sleep delta value!\n");
610 return;
611 }
612
John Stultz304529b2011-04-01 14:32:09 -0700613 xtime = timespec_add(xtime, *delta);
614 wall_to_monotonic = timespec_sub(wall_to_monotonic, *delta);
615 total_sleep_time = timespec_add(total_sleep_time, *delta);
616}
617
618
619/**
620 * timekeeping_inject_sleeptime - Adds suspend interval to timeekeeping values
621 * @delta: pointer to a timespec delta value
622 *
623 * This hook is for architectures that cannot support read_persistent_clock
624 * because their RTC/persistent clock is only accessible when irqs are enabled.
625 *
626 * This function should only be called by rtc_resume(), and allows
627 * a suspend offset to be injected into the timekeeping values.
628 */
629void timekeeping_inject_sleeptime(struct timespec *delta)
630{
631 unsigned long flags;
632 struct timespec ts;
633
634 /* Make sure we don't set the clock twice */
635 read_persistent_clock(&ts);
636 if (!(ts.tv_sec == 0 && ts.tv_nsec == 0))
637 return;
638
639 write_seqlock_irqsave(&xtime_lock, flags);
640 timekeeping_forward_now();
641
642 __timekeeping_inject_sleeptime(delta);
643
644 timekeeper.ntp_error = 0;
645 ntp_clear();
646 update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
647 timekeeper.mult);
648
649 write_sequnlock_irqrestore(&xtime_lock, flags);
650
651 /* signal hrtimers about time change */
652 clock_was_set();
653}
654
655
656/**
john stultz85240702007-05-08 00:27:59 -0700657 * timekeeping_resume - Resumes the generic timekeeping subsystem.
john stultz85240702007-05-08 00:27:59 -0700658 *
659 * This is for the generic clocksource timekeeping.
660 * xtime/wall_to_monotonic/jiffies/etc are
661 * still managed by arch specific suspend/resume code.
662 */
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +0100663static void timekeeping_resume(void)
john stultz85240702007-05-08 00:27:59 -0700664{
665 unsigned long flags;
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200666 struct timespec ts;
667
668 read_persistent_clock(&ts);
john stultz85240702007-05-08 00:27:59 -0700669
Thomas Gleixnerd10ff3f2007-05-14 11:10:02 +0200670 clocksource_resume();
671
john stultz85240702007-05-08 00:27:59 -0700672 write_seqlock_irqsave(&xtime_lock, flags);
673
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200674 if (timespec_compare(&ts, &timekeeping_suspend_time) > 0) {
675 ts = timespec_sub(ts, timekeeping_suspend_time);
John Stultz304529b2011-04-01 14:32:09 -0700676 __timekeeping_inject_sleeptime(&ts);
john stultz85240702007-05-08 00:27:59 -0700677 }
678 /* re-base the last cycle value */
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200679 timekeeper.clock->cycle_last = timekeeper.clock->read(timekeeper.clock);
680 timekeeper.ntp_error = 0;
john stultz85240702007-05-08 00:27:59 -0700681 timekeeping_suspended = 0;
682 write_sequnlock_irqrestore(&xtime_lock, flags);
683
684 touch_softlockup_watchdog();
685
686 clockevents_notify(CLOCK_EVT_NOTIFY_RESUME, NULL);
687
688 /* Resume hrtimers */
Thomas Gleixnerb12a03c2011-05-02 16:48:57 +0200689 hrtimers_resume();
john stultz85240702007-05-08 00:27:59 -0700690}
691
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +0100692static int timekeeping_suspend(void)
john stultz85240702007-05-08 00:27:59 -0700693{
694 unsigned long flags;
John Stultzcb332172011-05-31 22:53:23 -0700695 struct timespec delta, delta_delta;
696 static struct timespec old_delta;
john stultz85240702007-05-08 00:27:59 -0700697
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200698 read_persistent_clock(&timekeeping_suspend_time);
Thomas Gleixner3be90952007-09-16 15:36:43 +0200699
john stultz85240702007-05-08 00:27:59 -0700700 write_seqlock_irqsave(&xtime_lock, flags);
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200701 timekeeping_forward_now();
john stultz85240702007-05-08 00:27:59 -0700702 timekeeping_suspended = 1;
John Stultzcb332172011-05-31 22:53:23 -0700703
704 /*
705 * To avoid drift caused by repeated suspend/resumes,
706 * which each can add ~1 second drift error,
707 * try to compensate so the difference in system time
708 * and persistent_clock time stays close to constant.
709 */
710 delta = timespec_sub(xtime, timekeeping_suspend_time);
711 delta_delta = timespec_sub(delta, old_delta);
712 if (abs(delta_delta.tv_sec) >= 2) {
713 /*
714 * if delta_delta is too large, assume time correction
715 * has occured and set old_delta to the current delta.
716 */
717 old_delta = delta;
718 } else {
719 /* Otherwise try to adjust old_system to compensate */
720 timekeeping_suspend_time =
721 timespec_add(timekeeping_suspend_time, delta_delta);
722 }
john stultz85240702007-05-08 00:27:59 -0700723 write_sequnlock_irqrestore(&xtime_lock, flags);
724
725 clockevents_notify(CLOCK_EVT_NOTIFY_SUSPEND, NULL);
Magnus Dammc54a42b2010-02-02 14:41:41 -0800726 clocksource_suspend();
john stultz85240702007-05-08 00:27:59 -0700727
728 return 0;
729}
730
731/* sysfs resume/suspend bits for timekeeping */
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +0100732static struct syscore_ops timekeeping_syscore_ops = {
john stultz85240702007-05-08 00:27:59 -0700733 .resume = timekeeping_resume,
734 .suspend = timekeeping_suspend,
john stultz85240702007-05-08 00:27:59 -0700735};
736
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +0100737static int __init timekeeping_init_ops(void)
john stultz85240702007-05-08 00:27:59 -0700738{
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +0100739 register_syscore_ops(&timekeeping_syscore_ops);
740 return 0;
john stultz85240702007-05-08 00:27:59 -0700741}
742
Rafael J. Wysockie1a85b22011-03-23 22:16:04 +0100743device_initcall(timekeeping_init_ops);
john stultz85240702007-05-08 00:27:59 -0700744
745/*
746 * If the error is already larger, we look ahead even further
747 * to compensate for late or lost adjustments.
748 */
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200749static __always_inline int timekeeping_bigadjust(s64 error, s64 *interval,
john stultz85240702007-05-08 00:27:59 -0700750 s64 *offset)
751{
752 s64 tick_error, i;
753 u32 look_ahead, adj;
754 s32 error2, mult;
755
756 /*
757 * Use the current error value to determine how much to look ahead.
758 * The larger the error the slower we adjust for it to avoid problems
759 * with losing too many ticks, otherwise we would overadjust and
760 * produce an even larger error. The smaller the adjustment the
761 * faster we try to adjust for it, as lost ticks can do less harm
Li Zefan3eb05672008-02-08 04:19:25 -0800762 * here. This is tuned so that an error of about 1 msec is adjusted
john stultz85240702007-05-08 00:27:59 -0700763 * within about 1 sec (or 2^20 nsec in 2^SHIFT_HZ ticks).
764 */
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200765 error2 = timekeeper.ntp_error >> (NTP_SCALE_SHIFT + 22 - 2 * SHIFT_HZ);
john stultz85240702007-05-08 00:27:59 -0700766 error2 = abs(error2);
767 for (look_ahead = 0; error2 > 0; look_ahead++)
768 error2 >>= 2;
769
770 /*
771 * Now calculate the error in (1 << look_ahead) ticks, but first
772 * remove the single look ahead already included in the error.
773 */
Martin Schwidefsky23ce7212009-08-14 15:47:27 +0200774 tick_error = tick_length >> (timekeeper.ntp_error_shift + 1);
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200775 tick_error -= timekeeper.xtime_interval >> 1;
john stultz85240702007-05-08 00:27:59 -0700776 error = ((error - tick_error) >> look_ahead) + tick_error;
777
778 /* Finally calculate the adjustment shift value. */
779 i = *interval;
780 mult = 1;
781 if (error < 0) {
782 error = -error;
783 *interval = -*interval;
784 *offset = -*offset;
785 mult = -1;
786 }
787 for (adj = 0; error > i; adj++)
788 error >>= 1;
789
790 *interval <<= adj;
791 *offset <<= adj;
792 return mult << adj;
793}
794
795/*
796 * Adjust the multiplier to reduce the error value,
797 * this is optimized for the most common adjustments of -1,0,1,
798 * for other values we can do a bit more work.
799 */
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200800static void timekeeping_adjust(s64 offset)
john stultz85240702007-05-08 00:27:59 -0700801{
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200802 s64 error, interval = timekeeper.cycle_interval;
john stultz85240702007-05-08 00:27:59 -0700803 int adj;
804
Martin Schwidefsky23ce7212009-08-14 15:47:27 +0200805 error = timekeeper.ntp_error >> (timekeeper.ntp_error_shift - 1);
john stultz85240702007-05-08 00:27:59 -0700806 if (error > interval) {
807 error >>= 2;
808 if (likely(error <= interval))
809 adj = 1;
810 else
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200811 adj = timekeeping_bigadjust(error, &interval, &offset);
john stultz85240702007-05-08 00:27:59 -0700812 } else if (error < -interval) {
813 error >>= 2;
814 if (likely(error >= -interval)) {
815 adj = -1;
816 interval = -interval;
817 offset = -offset;
818 } else
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200819 adj = timekeeping_bigadjust(error, &interval, &offset);
john stultz85240702007-05-08 00:27:59 -0700820 } else
821 return;
822
Martin Schwidefsky0a544192009-08-14 15:47:28 +0200823 timekeeper.mult += adj;
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200824 timekeeper.xtime_interval += interval;
825 timekeeper.xtime_nsec -= offset;
826 timekeeper.ntp_error -= (interval - offset) <<
Martin Schwidefsky23ce7212009-08-14 15:47:27 +0200827 timekeeper.ntp_error_shift;
john stultz85240702007-05-08 00:27:59 -0700828}
829
Linus Torvalds83f57a12009-12-22 14:10:37 -0800830
john stultz85240702007-05-08 00:27:59 -0700831/**
john stultza092ff02009-10-02 16:17:53 -0700832 * logarithmic_accumulation - shifted accumulation of cycles
833 *
834 * This functions accumulates a shifted interval of cycles into
835 * into a shifted interval nanoseconds. Allows for O(log) accumulation
836 * loop.
837 *
838 * Returns the unconsumed cycles.
839 */
840static cycle_t logarithmic_accumulation(cycle_t offset, int shift)
841{
842 u64 nsecps = (u64)NSEC_PER_SEC << timekeeper.shift;
Jason Wesseldeda2e82010-08-09 14:20:09 -0700843 u64 raw_nsecs;
john stultza092ff02009-10-02 16:17:53 -0700844
845 /* If the offset is smaller then a shifted interval, do nothing */
846 if (offset < timekeeper.cycle_interval<<shift)
847 return offset;
848
849 /* Accumulate one shifted interval */
850 offset -= timekeeper.cycle_interval << shift;
851 timekeeper.clock->cycle_last += timekeeper.cycle_interval << shift;
852
853 timekeeper.xtime_nsec += timekeeper.xtime_interval << shift;
854 while (timekeeper.xtime_nsec >= nsecps) {
855 timekeeper.xtime_nsec -= nsecps;
856 xtime.tv_sec++;
857 second_overflow();
858 }
859
Jason Wesseldeda2e82010-08-09 14:20:09 -0700860 /* Accumulate raw time */
861 raw_nsecs = timekeeper.raw_interval << shift;
862 raw_nsecs += raw_time.tv_nsec;
John Stultzc7dcf872010-08-13 11:30:58 -0700863 if (raw_nsecs >= NSEC_PER_SEC) {
864 u64 raw_secs = raw_nsecs;
865 raw_nsecs = do_div(raw_secs, NSEC_PER_SEC);
866 raw_time.tv_sec += raw_secs;
john stultza092ff02009-10-02 16:17:53 -0700867 }
Jason Wesseldeda2e82010-08-09 14:20:09 -0700868 raw_time.tv_nsec = raw_nsecs;
john stultza092ff02009-10-02 16:17:53 -0700869
870 /* Accumulate error between NTP and clock interval */
871 timekeeper.ntp_error += tick_length << shift;
Kasper Pedersena386b5a2010-10-20 15:55:15 -0700872 timekeeper.ntp_error -=
873 (timekeeper.xtime_interval + timekeeper.xtime_remainder) <<
john stultza092ff02009-10-02 16:17:53 -0700874 (timekeeper.ntp_error_shift + shift);
875
876 return offset;
877}
878
Linus Torvalds83f57a12009-12-22 14:10:37 -0800879
john stultz85240702007-05-08 00:27:59 -0700880/**
881 * update_wall_time - Uses the current clocksource to increment the wall time
882 *
883 * Called from the timer interrupt, must hold a write on xtime_lock.
884 */
Torben Hohn871cf1e2011-01-27 15:58:55 +0100885static void update_wall_time(void)
john stultz85240702007-05-08 00:27:59 -0700886{
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200887 struct clocksource *clock;
john stultz85240702007-05-08 00:27:59 -0700888 cycle_t offset;
john stultza092ff02009-10-02 16:17:53 -0700889 int shift = 0, maxshift;
john stultz85240702007-05-08 00:27:59 -0700890
891 /* Make sure we're fully resumed: */
892 if (unlikely(timekeeping_suspended))
893 return;
894
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200895 clock = timekeeper.clock;
John Stultz592913e2010-07-13 17:56:20 -0700896
897#ifdef CONFIG_ARCH_USES_GETTIMEOFFSET
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200898 offset = timekeeper.cycle_interval;
John Stultz592913e2010-07-13 17:56:20 -0700899#else
900 offset = (clock->read(clock) - clock->cycle_last) & clock->mask;
john stultz85240702007-05-08 00:27:59 -0700901#endif
Martin Schwidefsky23ce7212009-08-14 15:47:27 +0200902 timekeeper.xtime_nsec = (s64)xtime.tv_nsec << timekeeper.shift;
john stultz85240702007-05-08 00:27:59 -0700903
john stultza092ff02009-10-02 16:17:53 -0700904 /*
905 * With NO_HZ we may have to accumulate many cycle_intervals
906 * (think "ticks") worth of time at once. To do this efficiently,
907 * we calculate the largest doubling multiple of cycle_intervals
908 * that is smaller then the offset. We then accumulate that
909 * chunk in one go, and then try to consume the next smaller
910 * doubled multiple.
john stultz85240702007-05-08 00:27:59 -0700911 */
john stultza092ff02009-10-02 16:17:53 -0700912 shift = ilog2(offset) - ilog2(timekeeper.cycle_interval);
913 shift = max(0, shift);
914 /* Bound shift to one less then what overflows tick_length */
915 maxshift = (8*sizeof(tick_length) - (ilog2(tick_length)+1)) - 1;
916 shift = min(shift, maxshift);
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200917 while (offset >= timekeeper.cycle_interval) {
john stultza092ff02009-10-02 16:17:53 -0700918 offset = logarithmic_accumulation(offset, shift);
John Stultz830ec042010-03-18 14:47:30 -0700919 if(offset < timekeeper.cycle_interval<<shift)
920 shift--;
john stultz85240702007-05-08 00:27:59 -0700921 }
922
923 /* correct the clock when NTP error is too big */
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200924 timekeeping_adjust(offset);
john stultz85240702007-05-08 00:27:59 -0700925
john stultz6c9bacb2008-12-01 18:34:41 -0800926 /*
927 * Since in the loop above, we accumulate any amount of time
928 * in xtime_nsec over a second into xtime.tv_sec, its possible for
929 * xtime_nsec to be fairly small after the loop. Further, if we're
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200930 * slightly speeding the clocksource up in timekeeping_adjust(),
john stultz6c9bacb2008-12-01 18:34:41 -0800931 * its possible the required corrective factor to xtime_nsec could
932 * cause it to underflow.
933 *
934 * Now, we cannot simply roll the accumulated second back, since
935 * the NTP subsystem has been notified via second_overflow. So
936 * instead we push xtime_nsec forward by the amount we underflowed,
937 * and add that amount into the error.
938 *
939 * We'll correct this error next time through this function, when
940 * xtime_nsec is not as small.
941 */
Martin Schwidefsky155ec602009-08-14 15:47:26 +0200942 if (unlikely((s64)timekeeper.xtime_nsec < 0)) {
943 s64 neg = -(s64)timekeeper.xtime_nsec;
944 timekeeper.xtime_nsec = 0;
Martin Schwidefsky23ce7212009-08-14 15:47:27 +0200945 timekeeper.ntp_error += neg << timekeeper.ntp_error_shift;
john stultz6c9bacb2008-12-01 18:34:41 -0800946 }
947
John Stultz6a867a32010-04-06 14:30:51 -0700948
949 /*
950 * Store full nanoseconds into xtime after rounding it up and
Roman Zippel5cd1c9c2008-09-22 14:42:43 -0700951 * add the remainder to the error difference.
952 */
Martin Schwidefsky23ce7212009-08-14 15:47:27 +0200953 xtime.tv_nsec = ((s64) timekeeper.xtime_nsec >> timekeeper.shift) + 1;
954 timekeeper.xtime_nsec -= (s64) xtime.tv_nsec << timekeeper.shift;
955 timekeeper.ntp_error += timekeeper.xtime_nsec <<
956 timekeeper.ntp_error_shift;
john stultz85240702007-05-08 00:27:59 -0700957
John Stultz6a867a32010-04-06 14:30:51 -0700958 /*
959 * Finally, make sure that after the rounding
960 * xtime.tv_nsec isn't larger then NSEC_PER_SEC
961 */
962 if (unlikely(xtime.tv_nsec >= NSEC_PER_SEC)) {
963 xtime.tv_nsec -= NSEC_PER_SEC;
964 xtime.tv_sec++;
965 second_overflow();
966 }
Linus Torvalds83f57a12009-12-22 14:10:37 -0800967
john stultz85240702007-05-08 00:27:59 -0700968 /* check to see if there is a new clocksource to use */
John Stultz76158562010-07-13 17:56:23 -0700969 update_vsyscall(&xtime, &wall_to_monotonic, timekeeper.clock,
970 timekeeper.mult);
john stultz85240702007-05-08 00:27:59 -0700971}
Tomas Janousek7c3f1a52007-07-15 23:39:41 -0700972
973/**
974 * getboottime - Return the real time of system boot.
975 * @ts: pointer to the timespec to be set
976 *
John Stultzabb3a4e2011-02-14 17:52:09 -0800977 * Returns the wall-time of boot in a timespec.
Tomas Janousek7c3f1a52007-07-15 23:39:41 -0700978 *
979 * This is based on the wall_to_monotonic offset and the total suspend
980 * time. Calls to settimeofday will affect the value returned (which
981 * basically means that however wrong your real time clock is at boot time,
982 * you get the right time here).
983 */
984void getboottime(struct timespec *ts)
985{
Hiroshi Shimamoto36d47482009-08-25 15:08:30 +0900986 struct timespec boottime = {
987 .tv_sec = wall_to_monotonic.tv_sec + total_sleep_time.tv_sec,
988 .tv_nsec = wall_to_monotonic.tv_nsec + total_sleep_time.tv_nsec
989 };
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200990
Martin Schwidefskyd4f587c2009-08-14 15:47:31 +0200991 set_normalized_timespec(ts, -boottime.tv_sec, -boottime.tv_nsec);
Tomas Janousek7c3f1a52007-07-15 23:39:41 -0700992}
Jason Wangc93d89f2010-01-27 19:13:40 +0800993EXPORT_SYMBOL_GPL(getboottime);
Tomas Janousek7c3f1a52007-07-15 23:39:41 -0700994
John Stultzabb3a4e2011-02-14 17:52:09 -0800995
996/**
997 * get_monotonic_boottime - Returns monotonic time since boot
998 * @ts: pointer to the timespec to be set
999 *
1000 * Returns the monotonic time since boot in a timespec.
1001 *
1002 * This is similar to CLOCK_MONTONIC/ktime_get_ts, but also
1003 * includes the time spent in suspend.
1004 */
1005void get_monotonic_boottime(struct timespec *ts)
1006{
1007 struct timespec tomono, sleep;
1008 unsigned int seq;
1009 s64 nsecs;
1010
1011 WARN_ON(timekeeping_suspended);
1012
1013 do {
1014 seq = read_seqbegin(&xtime_lock);
1015 *ts = xtime;
1016 tomono = wall_to_monotonic;
1017 sleep = total_sleep_time;
1018 nsecs = timekeeping_get_ns();
1019
1020 } while (read_seqretry(&xtime_lock, seq));
1021
1022 set_normalized_timespec(ts, ts->tv_sec + tomono.tv_sec + sleep.tv_sec,
1023 ts->tv_nsec + tomono.tv_nsec + sleep.tv_nsec + nsecs);
1024}
1025EXPORT_SYMBOL_GPL(get_monotonic_boottime);
1026
1027/**
1028 * ktime_get_boottime - Returns monotonic time since boot in a ktime
1029 *
1030 * Returns the monotonic time since boot in a ktime
1031 *
1032 * This is similar to CLOCK_MONTONIC/ktime_get, but also
1033 * includes the time spent in suspend.
1034 */
1035ktime_t ktime_get_boottime(void)
1036{
1037 struct timespec ts;
1038
1039 get_monotonic_boottime(&ts);
1040 return timespec_to_ktime(ts);
1041}
1042EXPORT_SYMBOL_GPL(ktime_get_boottime);
1043
Tomas Janousek7c3f1a52007-07-15 23:39:41 -07001044/**
1045 * monotonic_to_bootbased - Convert the monotonic time to boot based.
1046 * @ts: pointer to the timespec to be converted
1047 */
1048void monotonic_to_bootbased(struct timespec *ts)
1049{
John Stultzce3bf7a2010-07-13 17:56:19 -07001050 *ts = timespec_add(*ts, total_sleep_time);
Tomas Janousek7c3f1a52007-07-15 23:39:41 -07001051}
Jason Wangc93d89f2010-01-27 19:13:40 +08001052EXPORT_SYMBOL_GPL(monotonic_to_bootbased);
john stultz2c6b47d2007-07-24 17:47:43 -07001053
john stultz17c38b72007-07-24 18:38:34 -07001054unsigned long get_seconds(void)
1055{
John Stultz6a867a32010-04-06 14:30:51 -07001056 return xtime.tv_sec;
john stultz17c38b72007-07-24 18:38:34 -07001057}
1058EXPORT_SYMBOL(get_seconds);
1059
john stultzda15cfd2009-08-19 19:13:34 -07001060struct timespec __current_kernel_time(void)
1061{
John Stultz6a867a32010-04-06 14:30:51 -07001062 return xtime;
john stultzda15cfd2009-08-19 19:13:34 -07001063}
john stultz17c38b72007-07-24 18:38:34 -07001064
john stultz2c6b47d2007-07-24 17:47:43 -07001065struct timespec current_kernel_time(void)
1066{
1067 struct timespec now;
1068 unsigned long seq;
1069
1070 do {
1071 seq = read_seqbegin(&xtime_lock);
Linus Torvalds83f57a12009-12-22 14:10:37 -08001072
John Stultz6a867a32010-04-06 14:30:51 -07001073 now = xtime;
john stultz2c6b47d2007-07-24 17:47:43 -07001074 } while (read_seqretry(&xtime_lock, seq));
1075
1076 return now;
1077}
john stultz2c6b47d2007-07-24 17:47:43 -07001078EXPORT_SYMBOL(current_kernel_time);
john stultzda15cfd2009-08-19 19:13:34 -07001079
1080struct timespec get_monotonic_coarse(void)
1081{
1082 struct timespec now, mono;
1083 unsigned long seq;
1084
1085 do {
1086 seq = read_seqbegin(&xtime_lock);
Linus Torvalds83f57a12009-12-22 14:10:37 -08001087
John Stultz6a867a32010-04-06 14:30:51 -07001088 now = xtime;
john stultzda15cfd2009-08-19 19:13:34 -07001089 mono = wall_to_monotonic;
1090 } while (read_seqretry(&xtime_lock, seq));
1091
1092 set_normalized_timespec(&now, now.tv_sec + mono.tv_sec,
1093 now.tv_nsec + mono.tv_nsec);
1094 return now;
1095}
Torben Hohn871cf1e2011-01-27 15:58:55 +01001096
1097/*
1098 * The 64-bit jiffies value is not atomic - you MUST NOT read it
1099 * without sampling the sequence number in xtime_lock.
1100 * jiffies is defined in the linker script...
1101 */
1102void do_timer(unsigned long ticks)
1103{
1104 jiffies_64 += ticks;
1105 update_wall_time();
1106 calc_global_load(ticks);
1107}
Torben Hohn48cf76f72011-01-27 15:59:05 +01001108
1109/**
John Stultz314ac372011-02-14 18:43:08 -08001110 * get_xtime_and_monotonic_and_sleep_offset() - get xtime, wall_to_monotonic,
1111 * and sleep offsets.
Torben Hohn48cf76f72011-01-27 15:59:05 +01001112 * @xtim: pointer to timespec to be set with xtime
1113 * @wtom: pointer to timespec to be set with wall_to_monotonic
John Stultz314ac372011-02-14 18:43:08 -08001114 * @sleep: pointer to timespec to be set with time in suspend
Torben Hohn48cf76f72011-01-27 15:59:05 +01001115 */
John Stultz314ac372011-02-14 18:43:08 -08001116void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim,
1117 struct timespec *wtom, struct timespec *sleep)
Torben Hohn48cf76f72011-01-27 15:59:05 +01001118{
1119 unsigned long seq;
1120
1121 do {
1122 seq = read_seqbegin(&xtime_lock);
1123 *xtim = xtime;
1124 *wtom = wall_to_monotonic;
John Stultz314ac372011-02-14 18:43:08 -08001125 *sleep = total_sleep_time;
Torben Hohn48cf76f72011-01-27 15:59:05 +01001126 } while (read_seqretry(&xtime_lock, seq));
1127}
Torben Hohnf0af911a92011-01-27 15:59:10 +01001128
1129/**
Thomas Gleixner99ee5312011-04-27 14:16:42 +02001130 * ktime_get_monotonic_offset() - get wall_to_monotonic in ktime_t format
1131 */
1132ktime_t ktime_get_monotonic_offset(void)
1133{
1134 unsigned long seq;
1135 struct timespec wtom;
1136
1137 do {
1138 seq = read_seqbegin(&xtime_lock);
1139 wtom = wall_to_monotonic;
1140 } while (read_seqretry(&xtime_lock, seq));
1141 return timespec_to_ktime(wtom);
1142}
1143
1144/**
Torben Hohnf0af911a92011-01-27 15:59:10 +01001145 * xtime_update() - advances the timekeeping infrastructure
1146 * @ticks: number of ticks, that have elapsed since the last call.
1147 *
1148 * Must be called with interrupts disabled.
1149 */
1150void xtime_update(unsigned long ticks)
1151{
1152 write_seqlock(&xtime_lock);
1153 do_timer(ticks);
1154 write_sequnlock(&xtime_lock);
1155}