1 /*
2 * Copyright (c) 2018 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #ifndef _KERN_LOCKSTAT_H
29 #define _KERN_LOCKSTAT_H
30 #include <machine/locks.h>
31 #include <machine/atomic.h>
32 #include <kern/lock_group.h>
33
34 /*
35 * N.B.: On x86, statistics are currently recorded for all indirect mutexes.
36 * Also, only the acquire attempt count (GRP_MTX_STAT_UTIL) is maintained
37 * as a 64-bit quantity (the new x86 specific statistics are also maintained
38 * as 32-bit quantities).
39 *
40 *
41 * Enable this preprocessor define to record the first miss alone
42 * By default, we count every miss, hence multiple misses may be
43 * recorded for a single lock acquire attempt via lck_mtx_lock
44 */
45 #undef LOG_FIRST_MISS_ALONE
46
47 /*
48 * This preprocessor define controls whether the R-M-W update of the
49 * per-group statistics elements are atomic (LOCK-prefixed)
50 * Enabled by default.
51 */
52 #define ATOMIC_STAT_UPDATES 1
53
54 /*
55 * DTrace lockstat probe definitions
56 *
57 */
58
59 enum lockstat_probe_id {
60 /* Spinlocks */
61 LS_LCK_SPIN_LOCK_ACQUIRE,
62 LS_LCK_SPIN_LOCK_SPIN,
63 LS_LCK_SPIN_UNLOCK_RELEASE,
64
65 /*
66 * Mutexes can also have interlock-spin events, which are
67 * unique to our lock implementation.
68 */
69 LS_LCK_MTX_LOCK_ACQUIRE,
70 LS_LCK_MTX_LOCK_BLOCK,
71 LS_LCK_MTX_LOCK_SPIN,
72 LS_LCK_MTX_LOCK_ILK_SPIN,
73 LS_LCK_MTX_TRY_LOCK_ACQUIRE,
74 LS_LCK_MTX_TRY_SPIN_LOCK_ACQUIRE,
75 LS_LCK_MTX_UNLOCK_RELEASE,
76 LS_LCK_MTX_LOCK_SPIN_ACQUIRE,
77
78 /*
79 * Provide a parallel set for indirect mutexes
80 */
81 LS_LCK_MTX_EXT_LOCK_ACQUIRE,
82 LS_LCK_MTX_EXT_LOCK_BLOCK,
83 LS_LCK_MTX_EXT_LOCK_SPIN,
84 LS_LCK_MTX_EXT_LOCK_ILK_SPIN,
85 LS_LCK_MTX_EXT_UNLOCK_RELEASE,
86
87 /*
88 * Reader-writer locks support a blocking upgrade primitive, as
89 * well as the possibility of spinning on the interlock.
90 */
91 LS_LCK_RW_LOCK_SHARED_ACQUIRE,
92 LS_LCK_RW_LOCK_SHARED_BLOCK,
93 LS_LCK_RW_LOCK_SHARED_SPIN,
94
95 LS_LCK_RW_LOCK_EXCL_ACQUIRE,
96 LS_LCK_RW_LOCK_EXCL_BLOCK,
97 LS_LCK_RW_LOCK_EXCL_SPIN,
98
99 LS_LCK_RW_DONE_RELEASE,
100
101 LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE,
102 LS_LCK_RW_TRY_LOCK_SHARED_SPIN,
103
104 LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE,
105 LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN,
106
107 LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE,
108 LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN,
109 LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK,
110
111 LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE,
112 LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN,
113
114 /* Ticket lock */
115 LS_LCK_TICKET_LOCK_ACQUIRE,
116 LS_LCK_TICKET_LOCK_RELEASE,
117 LS_LCK_TICKET_LOCK_SPIN,
118
119 LS_NPROBES
120 };
121
122 #if CONFIG_DTRACE
123 extern uint32_t lockstat_probemap[LS_NPROBES];
124 extern void dtrace_probe(uint32_t, uint64_t, uint64_t,
125 uint64_t, uint64_t, uint64_t);
126 /*
127 * Macros to record lockstat probes.
128 */
129 #define LOCKSTAT_RECORD4(probe, lp, arg0, arg1, arg2, arg3) \
130 { \
131 uint32_t id; \
132 if (__improbable(id = lockstat_probemap[(probe)])) { \
133 dtrace_probe(id, (uintptr_t)(lp), (arg0), \
134 (arg1), (arg2), (arg3)); \
135 } \
136 }
137 #define LOCKSTAT_RECORD_(probe, lp, arg0, arg1, arg2, arg3, ...) LOCKSTAT_RECORD4(probe, lp, arg0, arg1, arg2, arg3)
138 #define LOCKSTAT_RECORD__(probe, lp, arg0, arg1, arg2, arg3, ...) LOCKSTAT_RECORD_(probe, lp, arg0, arg1, arg2, arg3)
139 #define LOCKSTAT_RECORD(probe, lp, ...) LOCKSTAT_RECORD__(probe, lp, ##__VA_ARGS__, 0, 0, 0, 0)
140 #else
141 #define LOCKSTAT_RECORD()
142 #endif /* CONFIG_DTRACE */
143
144 /*
145 * Time threshold before dtrace lockstat spin
146 * probes are triggered
147 */
148 extern machine_timeout32_t dtrace_spin_threshold;
149
150 #if CONFIG_DTRACE
151 void lockprof_invoke(lck_grp_t*, lck_grp_stat_t*, uint64_t);
152 #endif /* CONFIG_DTRACE */
153
154 static inline void
lck_grp_stat_enable(lck_grp_stat_t * stat)155 lck_grp_stat_enable(lck_grp_stat_t *stat)
156 {
157 stat->lgs_enablings++;
158 }
159
160 static inline void
lck_grp_stat_disable(lck_grp_stat_t * stat)161 lck_grp_stat_disable(lck_grp_stat_t *stat)
162 {
163 stat->lgs_enablings--;
164 }
165
166 #if MACH_KERNEL_PRIVATE
167 static inline void
lck_grp_inc_stats(lck_grp_t * grp,lck_grp_stat_t * stat)168 lck_grp_inc_stats(lck_grp_t *grp, lck_grp_stat_t *stat)
169 {
170 #pragma unused(grp)
171 if (__improbable(stat->lgs_enablings)) {
172 #if ATOMIC_STAT_UPDATES
173 uint64_t val = os_atomic_inc_orig(&stat->lgs_count, relaxed);
174 #else
175 uint64_t val = stat->lgs_count++;
176 #endif /* ATOMIC_STAT_UPDATES */
177 #if CONFIG_DTRACE && LOCK_STATS
178 if (__improbable(stat->lgs_limit && (val % (stat->lgs_limit)) == 0)) {
179 lockprof_invoke(grp, stat, val);
180 }
181 #else
182 #pragma unused(val)
183 #endif /* CONFIG_DTRACE && LOCK_STATS */
184 }
185 }
186
187 #if LOCK_STATS
188 static inline void
lck_grp_inc_time_stats(lck_grp_t * grp,lck_grp_stat_t * stat,uint64_t time)189 lck_grp_inc_time_stats(lck_grp_t *grp, lck_grp_stat_t *stat, uint64_t time)
190 {
191 if (__improbable(stat->lgs_enablings)) {
192 uint64_t val = os_atomic_add_orig(&stat->lgs_count, time, relaxed);
193 #if CONFIG_DTRACE
194 if (__improbable(stat->lgs_limit)) {
195 while (__improbable(time > stat->lgs_limit)) {
196 time -= stat->lgs_limit;
197 lockprof_invoke(grp, stat, val);
198 }
199 if (__improbable(((val % stat->lgs_limit) + time) > stat->lgs_limit)) {
200 lockprof_invoke(grp, stat, val);
201 }
202 }
203 #else
204 #pragma unused(val)
205 #endif /* CONFIG_DTRACE */
206 }
207 }
208
209 #endif /* LOCK_STATS */
210
211 static inline void
lck_grp_spin_update_held(void * lock LCK_GRP_ARG (lck_grp_t * grp))212 lck_grp_spin_update_held(void *lock LCK_GRP_ARG(lck_grp_t *grp))
213 {
214 #pragma unused(lock)
215 #if CONFIG_DTRACE
216 LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, (uintptr_t)LCK_GRP_PROBEARG(grp));
217 #endif
218 #if LOCK_STATS
219 if (!grp) {
220 return;
221 }
222 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_spin_held;
223 lck_grp_inc_stats(grp, stat);
224 #endif /* LOCK_STATS */
225 }
226
227 static inline void
lck_grp_spin_update_miss(void * lock LCK_GRP_ARG (lck_grp_t * grp))228 lck_grp_spin_update_miss(void *lock LCK_GRP_ARG(lck_grp_t *grp))
229 {
230 #pragma unused(lock)
231 #if LOCK_STATS
232 if (!grp) {
233 return;
234 }
235 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_spin_miss;
236 lck_grp_inc_stats(grp, stat);
237 #endif /* LOCK_STATS */
238 }
239
240 static inline void
lck_grp_spin_update_spin(void * lock LCK_GRP_ARG (lck_grp_t * grp),uint64_t time)241 lck_grp_spin_update_spin(void *lock LCK_GRP_ARG(lck_grp_t *grp), uint64_t time)
242 {
243 #pragma unused(lock, time)
244 #if CONFIG_DTRACE
245 if (time > os_atomic_load(&dtrace_spin_threshold, relaxed)) {
246 LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_SPIN, lock, time LCK_GRP_ARG((uintptr_t)grp));
247 }
248 #endif /* CONFIG_DTRACE */
249 #if LOCK_STATS
250 if (!grp) {
251 return;
252 }
253 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_spin_spin;
254 lck_grp_inc_time_stats(grp, stat, time);
255 #endif /* LOCK_STATS */
256 }
257
258 static inline boolean_t
lck_grp_spin_spin_enabled(void * lock LCK_GRP_ARG (lck_grp_t * grp))259 lck_grp_spin_spin_enabled(void *lock LCK_GRP_ARG(lck_grp_t *grp))
260 {
261 #pragma unused(lock)
262 boolean_t enabled = FALSE;
263 #if CONFIG_DTRACE
264 enabled |= lockstat_probemap[LS_LCK_SPIN_LOCK_SPIN] != 0;
265 #endif /* CONFIG_DTRACE */
266 #if LOCK_STATS
267 enabled |= (grp && grp->lck_grp_stats.lgss_spin_spin.lgs_enablings);
268 #endif /* LOCK_STATS */
269 return enabled;
270 }
271
272 static inline void
lck_grp_ticket_update_held(void * lock LCK_GRP_ARG (lck_grp_t * grp))273 lck_grp_ticket_update_held(void *lock LCK_GRP_ARG(lck_grp_t *grp))
274 {
275 #pragma unused(lock)
276 #if CONFIG_DTRACE
277 LOCKSTAT_RECORD(LS_LCK_TICKET_LOCK_ACQUIRE, lock, (uintptr_t)LCK_GRP_PROBEARG(grp));
278 #endif
279 #if LOCK_STATS
280 if (!grp) {
281 return;
282 }
283 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_ticket_held;
284 lck_grp_inc_stats(grp, stat);
285 #endif /* LOCK_STATS */
286 }
287
288 static inline void
lck_grp_ticket_update_miss(void * lock LCK_GRP_ARG (lck_grp_t * grp))289 lck_grp_ticket_update_miss(void *lock LCK_GRP_ARG(lck_grp_t *grp))
290 {
291 #pragma unused(lock)
292 #if LOCK_STATS
293 if (!grp) {
294 return;
295 }
296 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_ticket_miss;
297 lck_grp_inc_stats(grp, stat);
298 #endif /* LOCK_STATS */
299 }
300
301 static inline boolean_t
lck_grp_ticket_spin_enabled(void * lock LCK_GRP_ARG (lck_grp_t * grp))302 lck_grp_ticket_spin_enabled(void *lock LCK_GRP_ARG(lck_grp_t *grp))
303 {
304 #pragma unused(lock)
305 boolean_t enabled = FALSE;
306 #if CONFIG_DTRACE
307 enabled |= lockstat_probemap[LS_LCK_TICKET_LOCK_SPIN] != 0;
308 #endif /* CONFIG_DTRACE */
309 #if LOCK_STATS
310 enabled |= (grp && grp->lck_grp_stats.lgss_ticket_spin.lgs_enablings);
311 #endif /* LOCK_STATS */
312 return enabled;
313 }
314
315 static inline void
lck_grp_ticket_update_spin(void * lock LCK_GRP_ARG (lck_grp_t * grp),uint64_t time)316 lck_grp_ticket_update_spin(void *lock LCK_GRP_ARG(lck_grp_t *grp), uint64_t time)
317 {
318 #pragma unused(lock, time)
319 #if CONFIG_DTRACE
320 if (time > os_atomic_load(&dtrace_spin_threshold, relaxed)) {
321 LOCKSTAT_RECORD(LS_LCK_TICKET_LOCK_SPIN, lock, time LCK_GRP_ARG((uintptr_t)grp));
322 }
323 #endif /* CONFIG_DTRACE */
324 #if LOCK_STATS
325 if (!grp) {
326 return;
327 }
328 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_ticket_spin;
329 lck_grp_inc_time_stats(grp, stat, time);
330 #endif /* LOCK_STATS */
331 }
332
333
334 static void inline
lck_grp_mtx_update_miss(struct _lck_mtx_ext_ * lock,int * first_miss)335 lck_grp_mtx_update_miss(
336 struct _lck_mtx_ext_ *lock,
337 int *first_miss)
338 {
339 #pragma unused(first_miss)
340 #if LOG_FIRST_MISS_ALONE
341 if ((*first_miss & 1) == 0) {
342 #endif /* LOG_FIRST_MISS_ALONE */
343 lck_grp_t *grp = lock->lck_mtx_grp;
344 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_mtx_miss;
345 lck_grp_inc_stats(grp, stat);
346
347 #if LOG_FIRST_MISS_ALONE
348 *first_miss |= 1;
349 }
350 #endif /* LOG_FIRST_MISS_ALONE */
351 }
352
353 static void inline
lck_grp_mtx_update_direct_wait(struct _lck_mtx_ext_ * lock)354 lck_grp_mtx_update_direct_wait(
355 struct _lck_mtx_ext_ *lock)
356 {
357 lck_grp_t *grp = lock->lck_mtx_grp;
358 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_mtx_direct_wait;
359 lck_grp_inc_stats(grp, stat);
360 }
361
362 static void inline
lck_grp_mtx_update_wait(struct _lck_mtx_ext_ * lock,int * first_miss)363 lck_grp_mtx_update_wait(
364 struct _lck_mtx_ext_ *lock,
365 int *first_miss)
366 {
367 #pragma unused(first_miss)
368 #if LOG_FIRST_MISS_ALONE
369 if ((*first_miss & 2) == 0) {
370 #endif /* LOG_FIRST_MISS_ALONE */
371 lck_grp_t *grp = lock->lck_mtx_grp;
372 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_mtx_wait;
373 lck_grp_inc_stats(grp, stat);
374 #if LOG_FIRST_MISS_ALONE
375 *first_miss |= 2;
376 }
377 #endif /* LOG_FIRST_MISS_ALONE */
378 }
379
380 static void inline
lck_grp_mtx_update_held(struct _lck_mtx_ext_ * lock)381 lck_grp_mtx_update_held(
382 struct _lck_mtx_ext_ *lock)
383 {
384 lck_grp_t *grp = lock->lck_mtx_grp;
385 lck_grp_stat_t *stat = &grp->lck_grp_stats.lgss_mtx_held;
386 lck_grp_inc_stats(grp, stat);
387 }
388
389 #endif /* MACH_KERNEL_PRIVATE */
390 #endif /* _KERN_LOCKSTAT_H */
391