1 /*
2 * Copyright (c) 2015 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23 #include <stddef.h>
24 #include <stdbool.h>
25 #include <sys/types.h>
26 #include <machine/cpu_capabilities.h>
27 #include <mach/mach_time.h>
28
29 __attribute__((visibility("hidden")))
30 uint64_t
_mach_continuous_time_base(void)31 _mach_continuous_time_base(void)
32 {
33 #if !defined(__x86_64__) && !defined(__arm64__)
34 // Deal with the lack of 64-bit loads on arm32 (see mach_approximate_time.s)
35 while (1) {
36 volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
37 uint64_t read1, read2;
38 read1 = *base_ptr;
39 #if defined(__arm__)
40 __asm__ volatile ("dsb sy" ::: "memory");
41 #elif defined(__i386__)
42 __asm__ volatile ("lfence" ::: "memory");
43 #else
44 #error "unsupported arch"
45 #endif
46 read2 = *base_ptr;
47
48 if (__builtin_expect((read1 == read2), 1)) {
49 return read1;
50 }
51 }
52 #else // 64-bit
53 return *(volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
54 #endif // 64-bit
55 }
56
57 #define CNTVCTSS_EL0 "S3_3_c14_c0_6"
58 #define ACNTVCT_EL0 "S3_4_c15_c10_6"
59
60 __attribute__((visibility("hidden")))
61 kern_return_t
_mach_continuous_hwclock(uint64_t * cont_time __unused)62 _mach_continuous_hwclock(uint64_t *cont_time __unused)
63 {
64 #if defined(__arm64__)
65 #define ISB_SY 0xf
66 uint8_t cont_hwclock = *((uint8_t*)_COMM_PAGE_CONT_HWCLOCK);
67 if (cont_hwclock) {
68 volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_HW_TIMEBASE;
69
70 boolean_t has_cntvctss_el0 = *((uint8_t*)_COMM_PAGE_USER_TIMEBASE) == USER_TIMEBASE_NOSPEC;
71 if (has_cntvctss_el0) {
72 *cont_time = __builtin_arm_rsr64(CNTVCTSS_EL0) + *base_ptr;
73 return KERN_SUCCESS;
74 }
75
76 boolean_t has_acntvct = *((uint8_t*)_COMM_PAGE_USER_TIMEBASE) == USER_TIMEBASE_NOSPEC_APPLE;
77 if (has_acntvct) {
78 *cont_time = __builtin_arm_rsr64(ACNTVCT_EL0) + *base_ptr;
79 return KERN_SUCCESS;
80 }
81
82 __builtin_arm_isb(ISB_SY);
83 *cont_time = __builtin_arm_rsr64("CNTVCT_EL0") + *base_ptr;
84 return KERN_SUCCESS;
85 }
86 #endif
87 return KERN_NOT_SUPPORTED;
88 }
89
90 __attribute__((visibility("hidden")))
91 kern_return_t
_mach_continuous_time(uint64_t * absolute_time,uint64_t * cont_time)92 _mach_continuous_time(uint64_t* absolute_time, uint64_t* cont_time)
93 {
94 volatile uint64_t *base_ptr = (volatile uint64_t*)_COMM_PAGE_CONT_TIMEBASE;
95 volatile uint64_t read1, read2;
96 volatile uint64_t absolute;
97
98 do {
99 read1 = *base_ptr;
100 absolute = mach_absolute_time();
101 #if defined(__arm__) || defined(__arm64__)
102 /*
103 * mach_absolute_time() contains an instruction barrier which will
104 * prevent the speculation of read2 above this point, so we don't
105 * need another barrier here.
106 */
107 #endif
108 read2 = *base_ptr;
109 } while (__builtin_expect((read1 != read2), 0));
110
111 if (absolute_time) {
112 *absolute_time = absolute;
113 }
114 if (cont_time) {
115 *cont_time = absolute + read1;
116 }
117
118 return KERN_SUCCESS;
119 }
120
121 uint64_t
mach_continuous_time(void)122 mach_continuous_time(void)
123 {
124 uint64_t cont_time;
125 if (_mach_continuous_hwclock(&cont_time) != KERN_SUCCESS) {
126 _mach_continuous_time(NULL, &cont_time);
127 }
128 return cont_time;
129 }
130
131 uint64_t
mach_continuous_approximate_time(void)132 mach_continuous_approximate_time(void)
133 {
134 /*
135 * No retry loop here because if we use a slightly too old timebase that's
136 * okay, we are approximate time anyway.
137 */
138 volatile register uint64_t time_base = _mach_continuous_time_base();
139 return time_base + mach_approximate_time();
140 }
141