1 /*
2 * Copyright (c) 2018 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 #ifndef _KERN_LOCKSTAT_H
29 #define _KERN_LOCKSTAT_H
30
31 #include <machine/locks.h>
32 #include <machine/atomic.h>
33 #include <kern/lock_group.h>
34 #include <kern/lock_mtx.h>
35
36 __BEGIN_DECLS
37 #pragma GCC visibility push(hidden)
38
39 #if XNU_KERNEL_PRIVATE
40
41 /*
42 * DTrace lockstat probe definitions
43 *
44 */
45
46 enum lockstat_probe_id {
47 LS_NO_PROBE,
48
49 /* Spinlocks */
50 LS_LCK_SPIN_LOCK_ACQUIRE,
51 LS_LCK_SPIN_LOCK_SPIN,
52 LS_LCK_SPIN_UNLOCK_RELEASE,
53
54 /*
55 * Mutexes can also have interlock-spin events, which are
56 * unique to our lock implementation.
57 */
58 LS_LCK_MTX_LOCK_ACQUIRE,
59 LS_LCK_MTX_LOCK_SPIN_ACQUIRE,
60 LS_LCK_MTX_TRY_LOCK_ACQUIRE,
61 LS_LCK_MTX_TRY_LOCK_SPIN_ACQUIRE,
62 LS_LCK_MTX_UNLOCK_RELEASE,
63
64 LS_LCK_MTX_LOCK_BLOCK,
65 LS_LCK_MTX_LOCK_ADAPTIVE_SPIN,
66 LS_LCK_MTX_LOCK_SPIN_SPIN,
67
68
69 /*
70 * Reader-writer locks support a blocking upgrade primitive, as
71 * well as the possibility of spinning on the interlock.
72 */
73 LS_LCK_RW_LOCK_SHARED_ACQUIRE,
74 LS_LCK_RW_LOCK_SHARED_BLOCK,
75 LS_LCK_RW_LOCK_SHARED_SPIN,
76
77 LS_LCK_RW_LOCK_EXCL_ACQUIRE,
78 LS_LCK_RW_LOCK_EXCL_BLOCK,
79 LS_LCK_RW_LOCK_EXCL_SPIN,
80
81 LS_LCK_RW_DONE_RELEASE,
82
83 LS_LCK_RW_TRY_LOCK_SHARED_ACQUIRE,
84 LS_LCK_RW_TRY_LOCK_SHARED_SPIN,
85
86 LS_LCK_RW_TRY_LOCK_EXCL_ACQUIRE,
87 LS_LCK_RW_TRY_LOCK_EXCL_ILK_SPIN,
88
89 LS_LCK_RW_LOCK_SHARED_TO_EXCL_UPGRADE,
90 LS_LCK_RW_LOCK_SHARED_TO_EXCL_SPIN,
91 LS_LCK_RW_LOCK_SHARED_TO_EXCL_BLOCK,
92
93 LS_LCK_RW_LOCK_EXCL_TO_SHARED_DOWNGRADE,
94 LS_LCK_RW_LOCK_EXCL_TO_SHARED_ILK_SPIN,
95
96 /* Ticket lock */
97 LS_LCK_TICKET_LOCK_ACQUIRE,
98 LS_LCK_TICKET_LOCK_RELEASE,
99 LS_LCK_TICKET_LOCK_SPIN,
100
101 LS_NPROBES
102 };
103
104 #if CONFIG_DTRACE
105 /*
106 * Time threshold before dtrace lockstat spin
107 * probes are triggered
108 */
109 extern machine_timeout_t dtrace_spin_threshold;
110 extern uint32_t lockstat_probemap[LS_NPROBES];
111
112 extern void lck_grp_stat_enable(lck_grp_stat_t *stat);
113
114 extern void lck_grp_stat_disable(lck_grp_stat_t *stat);
115
116 extern bool lck_grp_stat_enabled(lck_grp_stat_t *stat);
117
118 extern void lck_grp_stat_inc(lck_grp_t *grp, lck_grp_stat_t *stat, bool always);
119
120 #endif /* CONFIG_DTRACE */
121 #endif /* XNU_KERNEL_PRIVATE */
122 #if MACH_KERNEL_PRIVATE
123 #if CONFIG_DTRACE
124
125 extern void dtrace_probe(uint32_t, uint64_t, uint64_t, uint64_t, uint64_t, uint64_t);
126
127 static inline void
lockprof_probe(lck_grp_t * grp,lck_grp_stat_t * stat,uint64_t val)128 lockprof_probe(lck_grp_t *grp, lck_grp_stat_t *stat, uint64_t val)
129 {
130 dtrace_probe(stat->lgs_probeid, (uintptr_t)grp, val, 0, 0, 0);
131 }
132
133 __attribute__((always_inline))
134 static inline void
lockstat_probe(enum lockstat_probe_id pid,const void * lock,uint64_t arg0,uint64_t arg1,uint64_t arg2,uint64_t arg3)135 lockstat_probe(
136 enum lockstat_probe_id pid,
137 const void *lock,
138 uint64_t arg0,
139 uint64_t arg1,
140 uint64_t arg2,
141 uint64_t arg3)
142 {
143 uint32_t id = lockstat_probemap[pid];
144
145 if (__improbable(id)) {
146 dtrace_probe(id, (uintptr_t)lock, arg0, arg1, arg2, arg3);
147 }
148 }
149
150 __pure2
151 static inline uint32_t
lockstat_enabled(void)152 lockstat_enabled(void)
153 {
154 return lck_debug_state.lds_value;
155 }
156
157 /*
158 * Macros to record lockstat probes.
159 */
160 #define LOCKSTAT_RECORD_(probe, lp, arg0, arg1, arg2, arg3, ...) \
161 lockstat_probe(probe, lp, arg0, arg1, arg2, arg3)
162 #define LOCKSTAT_RECORD__(probe, lp, arg0, arg1, arg2, arg3, ...) \
163 LOCKSTAT_RECORD_(probe, lp, arg0, arg1, arg2, arg3)
164 #define LOCKSTAT_RECORD(probe, lp, ...) \
165 LOCKSTAT_RECORD__(probe, lp, ##__VA_ARGS__, 0, 0, 0, 0)
166
167 __attribute__((always_inline, overloadable))
168 static inline bool
__lck_time_stat_enabled(enum lockstat_probe_id lspid,uint32_t grp_attr_id)169 __lck_time_stat_enabled(enum lockstat_probe_id lspid, uint32_t grp_attr_id)
170 {
171 if (__improbable(grp_attr_id & LCK_GRP_ATTR_TIME_STAT)) {
172 return true;
173 }
174 if (__improbable(lspid && lockstat_probemap[lspid])) {
175 return true;
176 }
177 return false;
178 }
179
180 __attribute__((always_inline, overloadable))
181 static inline bool
__lck_time_stat_enabled(enum lockstat_probe_id lspid,lck_grp_t * grp)182 __lck_time_stat_enabled(enum lockstat_probe_id lspid, lck_grp_t *grp)
183 {
184 uint32_t grp_attr_id = grp ? grp->lck_grp_attr_id : 0;
185
186 return __lck_time_stat_enabled(lspid, grp_attr_id);
187 }
188
189 #if LOCK_STATS
190 extern void __lck_grp_spin_update_held(lck_grp_t *grp);
191 extern void __lck_grp_spin_update_miss(lck_grp_t *grp);
192 extern void __lck_grp_spin_update_spin(lck_grp_t *grp, uint64_t time);
193 extern void __lck_grp_ticket_update_held(lck_grp_t *grp);
194 extern void __lck_grp_ticket_update_miss(lck_grp_t *grp);
195 extern void __lck_grp_ticket_update_spin(lck_grp_t *grp, uint64_t time);
196 #define LOCK_STATS_CALL(fn, ...) fn(__VA_ARGS__)
197 #else
198 #define LOCK_STATS_CALL(fn, ...) ((void)0)
199 #endif
200
201 static inline enum lockstat_probe_id
lck_mtx_acquire_probe(bool spin,bool try_lock)202 lck_mtx_acquire_probe(bool spin, bool try_lock)
203 {
204 if (spin) {
205 if (try_lock) {
206 return LS_LCK_MTX_TRY_LOCK_SPIN_ACQUIRE;
207 }
208 return LS_LCK_MTX_LOCK_SPIN_ACQUIRE;
209 } else {
210 if (try_lock) {
211 return LS_LCK_MTX_TRY_LOCK_ACQUIRE;
212 }
213 return LS_LCK_MTX_LOCK_ACQUIRE;
214 }
215 }
216
217 __attribute__((cold))
218 __header_always_inline void
lck_mtx_prof_probe(enum lockstat_probe_id id,lck_mtx_t * mtx,uint32_t grp_attr_id,bool profile)219 lck_mtx_prof_probe(
220 enum lockstat_probe_id id,
221 lck_mtx_t *mtx,
222 uint32_t grp_attr_id,
223 bool profile)
224 {
225 #pragma unused(mtx)
226 if (profile) {
227 lck_grp_t *grp = LCK_GRP_NULL;
228
229 switch (id) {
230 case LS_LCK_MTX_LOCK_ACQUIRE:
231 case LS_LCK_MTX_LOCK_SPIN_ACQUIRE:
232 case LS_LCK_MTX_TRY_LOCK_ACQUIRE:
233 case LS_LCK_MTX_TRY_LOCK_SPIN_ACQUIRE:
234 grp = lck_grp_resolve(grp_attr_id);
235 __builtin_assume(grp != NULL);
236 lck_grp_stat_inc(grp, &grp->lck_grp_stats.lgss_mtx_held, true);
237 break;
238 default:
239 break;
240 }
241 }
242 LOCKSTAT_RECORD(id, mtx, (uintptr_t)lck_grp_resolve(grp_attr_id));
243 }
244
245 #define lck_mtx_time_stat_begin(id) ({ \
246 uint64_t __start = 0; \
247 if (__lck_time_stat_enabled(id, LCK_GRP_NULL)) { \
248 __start = ml_get_timebase(); \
249 __builtin_assume(__start != 0); \
250 } \
251 __start; \
252 })
253
254 extern void lck_mtx_time_stat_record(
255 enum lockstat_probe_id id,
256 lck_mtx_t *mtx,
257 uint32_t grp_attr_id,
258 uint64_t start);
259
260 /*
261 * Enable this preprocessor define to record the first miss alone
262 * By default, we count every miss, hence multiple misses may be
263 * recorded for a single lock acquire attempt via lck_mtx_lock
264 */
265 #define LCK_MTX_LOCK_FIRST_MISS_ONLY 0
266
267 static inline void
LCK_MTX_PROF_MISS(lck_mtx_t * mtx,uint32_t grp_attr_id,int * first_miss)268 LCK_MTX_PROF_MISS(lck_mtx_t *mtx, uint32_t grp_attr_id, int *first_miss)
269 {
270 lck_grp_t *grp = lck_grp_resolve(grp_attr_id);
271
272 #pragma unused(mtx, grp, first_miss)
273 #if LCK_MTX_LOCK_FIRST_MISS_ONLY
274 if (*first_miss & 1) {
275 return;
276 }
277 *first_miss |= 1;
278 #endif /* LCK_MTX_LOCK_FIRST_MISS_ONLY */
279 lck_grp_stat_inc(grp, &grp->lck_grp_stats.lgss_mtx_miss, true);
280 }
281
282 static void inline
LCK_MTX_PROF_WAIT(lck_mtx_t * mtx,uint32_t grp_attr_id,bool direct_wait,int * first_miss)283 LCK_MTX_PROF_WAIT(
284 lck_mtx_t *mtx,
285 uint32_t grp_attr_id,
286 bool direct_wait,
287 int *first_miss)
288 {
289 lck_grp_t *grp = lck_grp_resolve(grp_attr_id);
290
291 #pragma unused(mtx, first_miss)
292 #if LCK_MTX_LOCK_FIRST_MISS_ONLY
293 if (*first_miss & 2) {
294 return;
295 }
296 *first_miss |= 2;
297 #endif /* LCK_MTX_LOCK_FIRST_MISS_ONLY */
298 if (direct_wait) {
299 lck_grp_stat_inc(grp, &grp->lck_grp_stats.lgss_mtx_direct_wait, true);
300 } else {
301 lck_grp_stat_inc(grp, &grp->lck_grp_stats.lgss_mtx_wait, true);
302 }
303 }
304
305 #else /* !CONFIG_DTRACE */
306
307 #define lockstat_enabled(probe, lock, ...) 0u
308 #define LOCKSTAT_RECORD(probe, lock, ...) ((void)0)
309
310 #define __lck_time_stat_enabled(lspid, grp) false
311 #define lck_mtx_prof_probe(id, mtx, grp, profile) ((void)0)
312 #define lck_mtx_time_stat_begin(id) 0ull
313 #define lck_mtx_time_stat_record(id, lck, grp, start) ((void)(start))
314
315 #endif /* !CONFIG_DTRACE */
316
317 static inline void
lck_grp_spin_update_held(void * lock LCK_GRP_ARG (lck_grp_t * grp))318 lck_grp_spin_update_held(void *lock LCK_GRP_ARG(lck_grp_t *grp))
319 {
320 #pragma unused(lock)
321 #if CONFIG_DTRACE
322 LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_ACQUIRE, lock, (uintptr_t)LCK_GRP_PROBEARG(grp));
323 LOCK_STATS_CALL(__lck_grp_spin_update_held, grp);
324 #endif /* CONFIG_DTRACE */
325 }
326
327 static inline void
lck_grp_spin_update_miss(void * lock LCK_GRP_ARG (lck_grp_t * grp))328 lck_grp_spin_update_miss(void *lock LCK_GRP_ARG(lck_grp_t *grp))
329 {
330 #pragma unused(lock)
331 #if CONFIG_DTRACE
332 LOCK_STATS_CALL(__lck_grp_spin_update_miss, grp);
333 #endif /* CONFIG_DTRACE */
334 }
335
336 static inline void
lck_grp_spin_update_spin(void * lock LCK_GRP_ARG (lck_grp_t * grp),uint64_t time)337 lck_grp_spin_update_spin(void *lock LCK_GRP_ARG(lck_grp_t *grp), uint64_t time)
338 {
339 #pragma unused(lock, time)
340 #if CONFIG_DTRACE
341 if (time > os_atomic_load(&dtrace_spin_threshold, relaxed)) {
342 LOCKSTAT_RECORD(LS_LCK_SPIN_LOCK_SPIN, lock, time LCK_GRP_ARG((uintptr_t)grp));
343 }
344 LOCK_STATS_CALL(__lck_grp_spin_update_spin, grp, time);
345 #endif /* CONFIG_DTRACE */
346 }
347
348 static inline bool
lck_grp_spin_spin_enabled(void * lock LCK_GRP_ARG (lck_grp_t * grp))349 lck_grp_spin_spin_enabled(void *lock LCK_GRP_ARG(lck_grp_t *grp))
350 {
351 #pragma unused(lock)
352 bool enabled = __lck_time_stat_enabled(LS_LCK_SPIN_LOCK_SPIN, LCK_GRP_PROBEARG(grp));
353 #if CONFIG_DTRACE && LOCK_STATS
354 enabled |= (grp && lck_grp_stat_enabled(&grp->lck_grp_stats.lgss_spin_spin));
355 #endif /* CONFIG_DTRACE && LOCK_STATS */
356 return enabled;
357 }
358
359 static inline void
lck_grp_ticket_update_held(void * lock LCK_GRP_ARG (lck_grp_t * grp))360 lck_grp_ticket_update_held(void *lock LCK_GRP_ARG(lck_grp_t *grp))
361 {
362 #pragma unused(lock)
363 #if CONFIG_DTRACE
364 LOCKSTAT_RECORD(LS_LCK_TICKET_LOCK_ACQUIRE, lock, (uintptr_t)LCK_GRP_PROBEARG(grp));
365 LOCK_STATS_CALL(__lck_grp_ticket_update_held, grp);
366 #endif /* CONFIG_DTRACE */
367 }
368
369 static inline void
lck_grp_ticket_update_miss(void * lock LCK_GRP_ARG (lck_grp_t * grp))370 lck_grp_ticket_update_miss(void *lock LCK_GRP_ARG(lck_grp_t *grp))
371 {
372 #pragma unused(lock)
373 #if CONFIG_DTRACE
374 LOCK_STATS_CALL(__lck_grp_ticket_update_miss, grp);
375 #endif /* CONFIG_DTRACE */
376 }
377
378 static inline bool
lck_grp_ticket_spin_enabled(void * lock LCK_GRP_ARG (lck_grp_t * grp))379 lck_grp_ticket_spin_enabled(void *lock LCK_GRP_ARG(lck_grp_t *grp))
380 {
381 #pragma unused(lock)
382 bool enabled = __lck_time_stat_enabled(LS_LCK_TICKET_LOCK_SPIN, LCK_GRP_PROBEARG(grp));
383 #if CONFIG_DTRACE && LOCK_STATS
384 enabled |= (grp && lck_grp_stat_enabled(&grp->lck_grp_stats.lgss_ticket_spin));
385 #endif /* CONFIG_DTRACE && LOCK_STATS */
386 return enabled;
387 }
388
389 static inline void
lck_grp_ticket_update_spin(void * lock LCK_GRP_ARG (lck_grp_t * grp),uint64_t time)390 lck_grp_ticket_update_spin(void *lock LCK_GRP_ARG(lck_grp_t *grp), uint64_t time)
391 {
392 #pragma unused(lock, time)
393 #if CONFIG_DTRACE
394 if (time > os_atomic_load(&dtrace_spin_threshold, relaxed)) {
395 LOCKSTAT_RECORD(LS_LCK_TICKET_LOCK_SPIN, lock, time LCK_GRP_ARG((uintptr_t)grp));
396 }
397 LOCK_STATS_CALL(__lck_grp_ticket_update_spin, grp, time);
398 #endif /* CONFIG_DTRACE */
399 }
400
401 /*
402 * Mutexes
403 */
404 #define LCK_MTX_ACQUIRED(mtx, grp, spin, profile) \
405 lck_mtx_prof_probe(lck_mtx_acquire_probe(spin, false), mtx, grp, profile)
406
407 #define LCK_MTX_TRY_ACQUIRED(mtx, grp, spin, profile) \
408 lck_mtx_prof_probe(lck_mtx_acquire_probe(spin, true), mtx, grp, profile)
409
410 #define LCK_MTX_RELEASED(mtx, grp, profile) \
411 lck_mtx_prof_probe(LS_LCK_MTX_UNLOCK_RELEASE, mtx, grp, profile)
412
413 #define LCK_MTX_BLOCK_BEGIN() \
414 lck_mtx_time_stat_begin(LS_LCK_MTX_LOCK_BLOCK)
415
416 #define LCK_MTX_BLOCK_END(mtx, grp, start) \
417 lck_mtx_time_stat_record(LS_LCK_MTX_LOCK_BLOCK, mtx, grp, start)
418
419 #define LCK_MTX_ADAPTIVE_SPIN_BEGIN() \
420 lck_mtx_time_stat_begin(LS_LCK_MTX_LOCK_ADAPTIVE_SPIN)
421
422 #define LCK_MTX_ADAPTIVE_SPIN_END(mtx, grp, start) \
423 lck_mtx_time_stat_record(LS_LCK_MTX_LOCK_ADAPTIVE_SPIN, mtx, grp, start)
424
425 #define LCK_MTX_SPIN_SPIN_BEGIN() \
426 lck_mtx_time_stat_begin(LS_LCK_MTX_LOCK_SPIN_SPIN)
427
428 #define LCK_MTX_SPIN_SPIN_END(mtx, grp, start) \
429 lck_mtx_time_stat_record(LS_LCK_MTX_LOCK_SPIN_SPIN, mtx, grp, start)
430
431 #endif /* MACH_KERNEL_PRIVATE */
432
433 #pragma GCC visibility pop
434 __END_DECLS
435
436 #endif /* _KERN_LOCKSTAT_H */
437