1 /*
2 * Copyright (c) 2000-2023 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 *
31 */
32
33 #ifndef I386_CPU_DATA
34 #define I386_CPU_DATA
35
36 #include <mach_assert.h>
37 #include <machine/atomic.h>
38
39 #include <kern/assert.h>
40 #include <kern/kern_types.h>
41 #include <kern/mpqueue.h>
42 #include <kern/queue.h>
43 #include <kern/processor.h>
44 #include <kern/pms.h>
45 #include <pexpert/pexpert.h>
46 #include <mach/i386/thread_status.h>
47 #include <mach/i386/vm_param.h>
48 #include <i386/locks.h>
49 #include <i386/rtclock_protos.h>
50 #include <i386/pmCPU.h>
51 #include <i386/cpu_topology.h>
52 #include <i386/seg.h>
53 #include <i386/mp.h>
54
55 #if CONFIG_VMX
56 #include <i386/vmx/vmx_cpu.h>
57 #endif
58
59 #if MONOTONIC
60 #include <machine/monotonic.h>
61 #endif /* MONOTONIC */
62
63 #include <san/kcov_data.h>
64
65 #include <machine/pal_routines.h>
66
67 /*
68 * Data structures referenced (anonymously) from per-cpu data:
69 */
70 struct cpu_cons_buffer;
71 struct cpu_desc_table;
72 struct mca_state;
73 struct prngContext;
74
75 /*
76 * Data structures embedded in per-cpu data:
77 */
78 typedef struct rtclock_timer {
79 mpqueue_head_t queue;
80 uint64_t deadline;
81 uint64_t when_set;
82 boolean_t has_expired;
83 } rtclock_timer_t;
84
85 typedef struct {
86 /* The 'u' suffixed fields store the double-mapped descriptor addresses */
87 struct x86_64_tss *cdi_ktssu;
88 struct x86_64_tss *cdi_ktssb;
89 x86_64_desc_register_t cdi_gdtu;
90 x86_64_desc_register_t cdi_gdtb;
91 x86_64_desc_register_t cdi_idtu;
92 x86_64_desc_register_t cdi_idtb;
93 struct real_descriptor *cdi_ldtu;
94 struct real_descriptor *cdi_ldtb;
95 vm_offset_t cdi_sstku;
96 vm_offset_t cdi_sstkb;
97 } cpu_desc_index_t;
98
99 typedef enum {
100 TASK_MAP_32BIT, /* 32-bit user, compatibility mode */
101 TASK_MAP_64BIT, /* 64-bit user thread, shared space */
102 } task_map_t;
103
104
105 /*
106 * This structure is used on entry into the (uber-)kernel on syscall from
107 * a 64-bit user. It contains the address of the machine state save area
108 * for the current thread and a temporary place to save the user's rsp
109 * before loading this address into rsp.
110 */
111 typedef struct {
112 addr64_t cu_isf; /* thread->pcb->iss.isf */
113 uint64_t cu_tmp; /* temporary scratch */
114 addr64_t cu_user_gs_base;
115 } cpu_uber_t;
116
117 typedef uint16_t pcid_t;
118 typedef uint8_t pcid_ref_t;
119
120 #define CPU_RTIME_BINS (12)
121 #define CPU_ITIME_BINS (CPU_RTIME_BINS)
122
123 #define MAX_TRACE_BTFRAMES (16)
124 typedef struct {
125 boolean_t pltype;
126 int plevel;
127 uint64_t plbt[MAX_TRACE_BTFRAMES];
128 } plrecord_t;
129
130 #if DEVELOPMENT || DEBUG
131
132 typedef struct {
133 int vector; /* Vector number of interrupt */
134 thread_t curthread; /* Current thread at the time of the interrupt */
135 uint64_t interrupted_pc;
136 int curpl; /* Current preemption level */
137 int curil; /* Current interrupt level */
138 uint64_t start_time_abs;
139 uint64_t duration;
140 uint64_t backtrace[MAX_TRACE_BTFRAMES];
141 } traptrace_entry_t;
142
143 #define TRAPTRACE_INVALID_INDEX (~0U)
144 #define DEFAULT_TRAPTRACE_ENTRIES_PER_CPU (16)
145 #define TRAPTRACE_MAX_ENTRIES_PER_CPU (256)
146 extern volatile int traptrace_enabled;
147 extern uint32_t traptrace_entries_per_cpu;
148 PERCPU_DECL(uint32_t, traptrace_next);
149 PERCPU_DECL(traptrace_entry_t * __unsafe_indexable, traptrace_ring);
150 #endif /* DEVELOPMENT || DEBUG */
151
152 /*
153 * Per-cpu data.
154 *
155 * Each processor has a per-cpu data area which is dereferenced through the
156 * current_cpu_datap() macro. For speed, the %gs segment is based here, and
157 * using this, inlines provides single-instruction access to frequently used
158 * members - such as get_cpu_number()/cpu_number(), and get_active_thread()/
159 * current_thread().
160 *
161 * Cpu data owned by another processor can be accessed using the
162 * cpu_datap(cpu_number) macro which uses the cpu_data_ptr[] array of per-cpu
163 * pointers.
164 */
165 typedef struct {
166 pcid_t cpu_pcid_free_hint;
167 #define PMAP_PCID_MAX_PCID (0x800)
168 pcid_ref_t cpu_pcid_refcounts[PMAP_PCID_MAX_PCID];
169 pmap_t cpu_pcid_last_pmap_dispatched[PMAP_PCID_MAX_PCID];
170 } pcid_cdata_t;
171
172 typedef struct cpu_data {
173 struct pal_cpu_data cpu_pal_data; /* PAL-specific data */
174 #define cpu_pd cpu_pal_data /* convenience alias */
175 struct cpu_data *cpu_this; /* pointer to myself */
176 vm_offset_t cpu_pcpu_base;
177 thread_t cpu_active_thread;
178 thread_t cpu_nthread;
179 int cpu_number; /* Logical CPU */
180 void *cpu_int_state; /* interrupt state */
181 vm_offset_t cpu_active_stack; /* kernel stack base */
182 vm_offset_t cpu_kernel_stack; /* kernel stack top */
183 vm_offset_t cpu_int_stack_top;
184 volatile int cpu_signals; /* IPI events */
185 volatile int cpu_prior_signals; /* Last set of events,
186 * debugging
187 */
188 ast_t cpu_pending_ast;
189 /*
190 * Note if rearranging fields:
191 * We want cpu_preemption_level on a different
192 * cache line than cpu_active_thread
193 * for optimizing mtx_spin phase.
194 */
195 int cpu_interrupt_level;
196 volatile int cpu_preemption_level;
197 volatile int cpu_running;
198 #if !MONOTONIC
199 boolean_t cpu_fixed_pmcs_enabled;
200 #endif /* !MONOTONIC */
201 rtclock_timer_t rtclock_timer;
202 volatile addr64_t cpu_active_cr3 __attribute((aligned(64)));
203 union {
204 volatile uint32_t cpu_tlb_invalid;
205 struct {
206 volatile uint16_t cpu_tlb_invalid_local;
207 volatile uint16_t cpu_tlb_invalid_global;
208 };
209 };
210 uint64_t cpu_ip_desc[2];
211 volatile task_map_t cpu_task_map;
212 volatile addr64_t cpu_task_cr3;
213 addr64_t cpu_kernel_cr3;
214 volatile addr64_t cpu_ucr3;
215 volatile addr64_t cpu_shadowtask_cr3;
216 boolean_t cpu_pagezero_mapped;
217 cpu_uber_t cpu_uber;
218 /* Double-mapped per-CPU exception stack address */
219 uintptr_t cd_estack;
220 int cpu_xstate;
221 int cpu_curtask_has_ldt;
222 int cpu_curthread_do_segchk;
223 /* Address of shadowed, partially mirrored CPU data structures located
224 * in the double mapped PML4
225 */
226 void *cd_shadow;
227 union {
228 volatile uint32_t cpu_tlb_invalid_count;
229 struct {
230 volatile uint16_t cpu_tlb_invalid_local_count;
231 volatile uint16_t cpu_tlb_invalid_global_count;
232 };
233 };
234
235 uint16_t cpu_tlb_gen_counts_local[MAX_CPUS];
236 uint16_t cpu_tlb_gen_counts_global[MAX_CPUS];
237
238 struct processor *cpu_processor;
239 struct real_descriptor *cpu_ldtp;
240 struct cpu_desc_table *cpu_desc_tablep;
241 cpu_desc_index_t cpu_desc_index;
242 int cpu_ldt;
243
244 #define HWINTCNT_SIZE 256
245 uint32_t cpu_hwIntCnt[HWINTCNT_SIZE]; /* Interrupt counts */
246 uint64_t cpu_hwIntpexits[HWINTCNT_SIZE];
247 uint64_t cpu_dr7; /* debug control register */
248 uint64_t cpu_int_event_time; /* intr entry/exit time */
249 pal_rtc_nanotime_t *cpu_nanotime; /* Nanotime info */
250 #if KPC
251 /* double-buffered performance counter data */
252 uint64_t *cpu_kpc_buf[2];
253 /* PMC shadow and reload value buffers */
254 uint64_t *cpu_kpc_shadow;
255 uint64_t *cpu_kpc_reload;
256 #endif
257 #if MONOTONIC
258 struct mt_cpu cpu_monotonic;
259 #endif /* MONOTONIC */
260 uint32_t cpu_pmap_pcid_enabled;
261 pcid_t cpu_active_pcid;
262 pcid_t cpu_last_pcid;
263 pcid_t cpu_kernel_pcid;
264 volatile pcid_ref_t *cpu_pmap_pcid_coherentp;
265 volatile pcid_ref_t *cpu_pmap_pcid_coherentp_kernel;
266 pcid_cdata_t *cpu_pcid_data;
267 #ifdef PCID_STATS
268 uint64_t cpu_pmap_pcid_flushes;
269 uint64_t cpu_pmap_pcid_preserves;
270 #endif
271 uint64_t cpu_aperf;
272 uint64_t cpu_mperf;
273 uint64_t cpu_c3res;
274 uint64_t cpu_c6res;
275 uint64_t cpu_c7res;
276 uint64_t cpu_itime_total;
277 uint64_t cpu_rtime_total;
278 uint64_t cpu_ixtime;
279 uint64_t cpu_idle_exits;
280 /*
281 * Note that the cacheline-copy mechanism uses the cpu_rtimes field in the shadow CPU
282 * structures to temporarily stash the code cacheline that includes the instruction
283 * pointer at the time of the fault (this field is otherwise unused in the shadow
284 * CPU structures).
285 */
286 uint64_t cpu_rtimes[CPU_RTIME_BINS];
287 uint64_t cpu_itimes[CPU_ITIME_BINS];
288 #if !MONOTONIC
289 uint64_t cpu_cur_insns;
290 uint64_t cpu_cur_ucc;
291 uint64_t cpu_cur_urc;
292 #endif /* !MONOTONIC */
293 uint64_t cpu_gpmcs[4];
294 uint64_t cpu_max_observed_int_latency;
295 int cpu_max_observed_int_latency_vector;
296 volatile boolean_t cpu_NMI_acknowledged;
297 uint64_t debugger_entry_time;
298 uint64_t debugger_ipi_time;
299 /* A separate nested interrupt stack flag, to account
300 * for non-nested interrupts arriving while on the interrupt stack
301 * Currently only occurs when AICPM enables interrupts on the
302 * interrupt stack during processor offlining.
303 */
304 uint32_t cpu_nested_istack;
305 uint32_t cpu_nested_istack_events;
306 x86_saved_state64_t *cpu_fatal_trap_state;
307 x86_saved_state64_t *cpu_post_fatal_trap_state;
308 #if CONFIG_VMX
309 vmx_cpu_t cpu_vmx; /* wonderful world of virtualization */
310 #endif
311 #if CONFIG_MCA
312 struct mca_state *cpu_mca_state; /* State at MC fault */
313 #endif
314 int cpu_type;
315 int cpu_subtype;
316 int cpu_threadtype;
317 boolean_t cpu_iflag;
318 boolean_t cpu_boot_complete;
319 int cpu_hibernate;
320 #define MAX_PREEMPTION_RECORDS (8)
321 #if DEVELOPMENT || DEBUG
322 int cpu_plri;
323 plrecord_t plrecords[MAX_PREEMPTION_RECORDS];
324 #endif
325 struct x86_lcpu lcpu;
326 int cpu_phys_number; /* Physical CPU */
327 cpu_id_t cpu_id; /* Platform Expert */
328 #if DEBUG
329 uint64_t cpu_entry_cr3;
330 uint64_t cpu_exit_cr3;
331 uint64_t cpu_pcid_last_cr3;
332 #endif
333 boolean_t cpu_rendezvous_in_progress;
334 #if CST_DEMOTION_DEBUG
335 /* Count of thread wakeups issued by this processor */
336 uint64_t cpu_wakeups_issued_total;
337 #endif
338 #if DEBUG || DEVELOPMENT
339 uint64_t tsc_sync_delta;
340 #endif
341 uint32_t cpu_soft_apic_lvt_timer;
342 #if CONFIG_KCOV
343 kcov_cpu_data_t cpu_kcov_data;
344 #endif
345 } cpu_data_t;
346
347 extern cpu_data_t *__single cpu_data_ptr[MAX_CPUS];
348
349 /*
350 * __SEG_GS marks %gs-relative operations:
351 * https://clang.llvm.org/docs/LanguageExtensions.html#memory-references-to-specified-segments
352 * https://gcc.gnu.org/onlinedocs/gcc/Named-Address-Spaces.html#x86-Named-Address-Spaces
353 */
354 #if defined(__SEG_GS)
355 // __seg_gs exists
356 #elif defined(__clang__)
357 #define __seg_gs __attribute__((address_space(256)))
358 #else
359 #error use a compiler that supports address spaces or __seg_gs
360 #endif
361
362 #define CPU_DATA() ((cpu_data_t __seg_gs *)0UL)
363
364 /*
365 * Everyone within the osfmk part of the kernel can use the fast
366 * inline versions of these routines. Everyone outside, must call
367 * the real thing,
368 */
369
370
371 /*
372 * The "volatile" flavor of current_thread() is intended for use by
373 * scheduler code which may need to update the thread pointer in the
374 * course of a context switch. Any call to current_thread() made
375 * prior to the thread pointer update should be safe to optimize away
376 * as it should be consistent with that thread's state to the extent
377 * the compiler can reason about it. Likewise, the context switch
378 * path will eventually result in an arbitrary branch to the new
379 * thread's pc, about which the compiler won't be able to reason.
380 * Thus any compile-time optimization of current_thread() calls made
381 * within the new thread should be safely encapsulated in its
382 * register/stack state. The volatile form therefore exists to cover
383 * the window between the thread pointer update and the branch to
384 * the new pc.
385 */
386 static inline thread_t
get_active_thread_volatile(void)387 get_active_thread_volatile(void)
388 {
389 return CPU_DATA()->cpu_active_thread;
390 }
391
392 static inline __attribute__((const)) thread_t
get_active_thread(void)393 get_active_thread(void)
394 {
395 return CPU_DATA()->cpu_active_thread;
396 }
397
398 #define current_thread_fast() get_active_thread()
399 #define current_thread_volatile() get_active_thread_volatile()
400
401 #define cpu_mode_is64bit() TRUE
402
403 static inline int
get_preemption_level(void)404 get_preemption_level(void)
405 {
406 return CPU_DATA()->cpu_preemption_level;
407 }
408 static inline int
get_interrupt_level(void)409 get_interrupt_level(void)
410 {
411 return CPU_DATA()->cpu_interrupt_level;
412 }
413 static inline int
get_cpu_number(void)414 get_cpu_number(void)
415 {
416 return CPU_DATA()->cpu_number;
417 }
418 static inline vm_offset_t
get_current_percpu_base(void)419 get_current_percpu_base(void)
420 {
421 return CPU_DATA()->cpu_pcpu_base;
422 }
423 static inline int
get_cpu_phys_number(void)424 get_cpu_phys_number(void)
425 {
426 return CPU_DATA()->cpu_phys_number;
427 }
428
429 static inline cpu_data_t *
current_cpu_datap(void)430 current_cpu_datap(void)
431 {
432 return CPU_DATA()->cpu_this;
433 }
434
435 /*
436 * Facility to diagnose preemption-level imbalances, which are otherwise
437 * challenging to debug. On each operation that enables or disables preemption,
438 * we record a backtrace into a per-CPU ring buffer, along with the current
439 * preemption level and operation type. Thus, if an imbalance is observed,
440 * one can examine these per-CPU records to determine which codepath failed
441 * to re-enable preemption, enabled premption without a corresponding
442 * disablement etc. The backtracer determines which stack is currently active,
443 * and uses that to perform bounds checks on unterminated stacks.
444 * To enable, sysctl -w machdep.pltrace=1 on DEVELOPMENT or DEBUG kernels (DRK '15)
445 * The bounds check currently doesn't account for non-default thread stack sizes.
446 */
447 #if DEVELOPMENT || DEBUG
448 static inline void
rbtrace_bt(uint64_t * __counted_by (maxframes)rets,int maxframes,cpu_data_t * cdata,uint64_t frameptr,bool use_cursp)449 rbtrace_bt(uint64_t *__counted_by(maxframes)rets, int maxframes,
450 cpu_data_t *cdata, uint64_t frameptr, bool use_cursp)
451 {
452 extern uint32_t low_intstack[]; /* bottom */
453 extern uint32_t low_eintstack[]; /* top */
454 extern char mp_slave_stack[];
455 int btidx = 0;
456
457 uint64_t kstackb, kstackt;
458
459 /* Obtain the 'current' program counter, initial backtrace
460 * element. This will also indicate if we were unable to
461 * trace further up the stack for some reason
462 */
463 if (use_cursp) {
464 __asm__ volatile ("leaq 1f(%%rip), %%rax; mov %%rax, %0\n1:"
465 : "=m" (rets[btidx++])
466 :
467 : "rax");
468 }
469
470 thread_t __single cplthread = cdata->cpu_active_thread;
471 if (cplthread) {
472 uintptr_t csp;
473 if (use_cursp == true) {
474 __asm__ __volatile__ ("movq %%rsp, %0": "=r" (csp):);
475 } else {
476 csp = frameptr;
477 }
478 /* Determine which stack we're on to populate stack bounds.
479 * We don't need to trace across stack boundaries for this
480 * routine.
481 */
482 kstackb = cdata->cpu_active_stack;
483 kstackt = kstackb + KERNEL_STACK_SIZE;
484 if (csp < kstackb || csp > kstackt) {
485 kstackt = cdata->cpu_kernel_stack;
486 kstackb = kstackt - KERNEL_STACK_SIZE;
487 if (csp < kstackb || csp > kstackt) {
488 kstackt = cdata->cpu_int_stack_top;
489 kstackb = kstackt - INTSTACK_SIZE;
490 if (csp < kstackb || csp > kstackt) {
491 kstackt = (uintptr_t)&low_eintstack;
492 kstackb = kstackt - INTSTACK_SIZE;
493 if (csp < kstackb || csp > kstackt) {
494 kstackb = (uintptr_t)&mp_slave_stack;
495 kstackt = kstackb + PAGE_SIZE;
496 } else {
497 kstackb = 0;
498 kstackt = 0;
499 }
500 }
501 }
502 }
503
504 if (__probable(kstackb && kstackt)) {
505 uint64_t *cfp = __unsafe_forge_single(uint64_t *, frameptr);
506 int rbbtf;
507
508 for (rbbtf = btidx; rbbtf < maxframes; rbbtf++) {
509 uint64_t cur_retp;
510 /*
511 * cfp == 0 is covered by the first comparison, and we're guaranteed
512 * that kstackb is non-zero from the containing if block. The os_add_overflow is
513 * necessary because it's not uncommon for backtraces to terminate with bogus
514 * frame pointers.
515 */
516 if (((uint64_t)cfp < kstackb) || os_add_overflow((uint64_t)cfp, sizeof(uint64_t), &cur_retp) || cur_retp >= kstackt) {
517 rets[rbbtf] = 0;
518 continue;
519 }
520 rets[rbbtf] = *(uint64_t *)cur_retp;
521 cfp = __unsafe_forge_single(uint64_t *, *cfp);
522 }
523 }
524 }
525 }
526
527 __attribute__((noinline))
528 static inline void
pltrace_internal(boolean_t enable)529 pltrace_internal(boolean_t enable)
530 {
531 cpu_data_t *cdata = current_cpu_datap();
532 int cpli = cdata->cpu_preemption_level;
533 int cplrecord = cdata->cpu_plri;
534 uint64_t *plbts;
535
536 assert(cpli >= 0);
537
538 cdata->plrecords[cplrecord].pltype = enable;
539 cdata->plrecords[cplrecord].plevel = cpli;
540
541 plbts = &cdata->plrecords[cplrecord].plbt[0];
542
543 cplrecord++;
544
545 if (cplrecord >= MAX_PREEMPTION_RECORDS) {
546 cplrecord = 0;
547 }
548
549 cdata->cpu_plri = cplrecord;
550
551 rbtrace_bt(plbts, MAX_TRACE_BTFRAMES - 1, cdata, (uint64_t)__builtin_frame_address(0), false);
552 }
553
554 extern int plctrace_enabled;
555
556 static inline uint32_t
traptrace_start(int vecnum,uint64_t ipc,uint64_t sabs,uint64_t frameptr)557 traptrace_start(int vecnum, uint64_t ipc, uint64_t sabs, uint64_t frameptr)
558 {
559 cpu_data_t *cdata;
560 uint32_t nextidx;
561 traptrace_entry_t *cur_traptrace_ring;
562 uint32_t *nextidxp;
563
564 if (__improbable(traptrace_enabled == 0 || traptrace_entries_per_cpu == 0)) {
565 return TRAPTRACE_INVALID_INDEX;
566 }
567
568 assert(ml_get_interrupts_enabled() == FALSE);
569 cdata = current_cpu_datap();
570 nextidxp = PERCPU_GET(traptrace_next);
571 nextidx = *nextidxp;
572 /* prevent nested interrupts from clobbering this record */
573 *nextidxp = (((nextidx + 1) >= (unsigned int)traptrace_entries_per_cpu) ? 0 : (nextidx + 1));
574
575 cur_traptrace_ring = __unsafe_forge_bidi_indexable(traptrace_entry_t *,
576 *PERCPU_GET(traptrace_ring), sizeof(traptrace_entry_t) * traptrace_entries_per_cpu);
577 cur_traptrace_ring[nextidx].vector = vecnum;
578 cur_traptrace_ring[nextidx].curthread = current_thread_fast();
579 cur_traptrace_ring[nextidx].interrupted_pc = ipc;
580 cur_traptrace_ring[nextidx].curpl = cdata->cpu_preemption_level;
581 cur_traptrace_ring[nextidx].curil = cdata->cpu_interrupt_level;
582 cur_traptrace_ring[nextidx].start_time_abs = sabs;
583 cur_traptrace_ring[nextidx].duration = ~0ULL;
584
585 rbtrace_bt(&cur_traptrace_ring[nextidx].backtrace[0],
586 MAX_TRACE_BTFRAMES - 1, cdata, frameptr, false);
587
588 assert(nextidx <= 0xFFFF);
589
590 /*
591 * encode the cpu number we're on because traptrace_end()
592 * might be called from a different CPU.
593 */
594 return ((uint32_t)cdata->cpu_number << 16) | nextidx;
595 }
596
597 static inline void
traptrace_end(uint32_t index,uint64_t eabs)598 traptrace_end(uint32_t index, uint64_t eabs)
599 {
600 traptrace_entry_t *__unsafe_indexable ring;
601
602 if (index != TRAPTRACE_INVALID_INDEX) {
603 ring = *PERCPU_GET_WITH_BASE(other_percpu_base(index >> 16),
604 traptrace_ring);
605 index &= 0XFFFF;
606 ring[index].duration = eabs - ring[index].start_time_abs;
607 }
608 }
609
610 #endif /* DEVELOPMENT || DEBUG */
611
612 __header_always_inline void
pltrace(boolean_t plenable)613 pltrace(boolean_t plenable)
614 {
615 #if DEVELOPMENT || DEBUG
616 if (__improbable(plctrace_enabled != 0)) {
617 pltrace_internal(plenable);
618 }
619 #else
620 (void)plenable;
621 #endif
622 }
623
624 static inline void
disable_preemption_internal(void)625 disable_preemption_internal(void)
626 {
627 assert(get_preemption_level() >= 0);
628
629 os_compiler_barrier();
630 CPU_DATA()->cpu_preemption_level++;
631 os_compiler_barrier();
632 pltrace(FALSE);
633 }
634
635 static inline void
enable_preemption_internal(void)636 enable_preemption_internal(void)
637 {
638 assert(get_preemption_level() > 0);
639 pltrace(TRUE);
640 os_compiler_barrier();
641 if (0 == --CPU_DATA()->cpu_preemption_level) {
642 kernel_preempt_check();
643 }
644 os_compiler_barrier();
645 }
646
647 static inline void
enable_preemption_no_check(void)648 enable_preemption_no_check(void)
649 {
650 assert(get_preemption_level() > 0);
651
652 pltrace(TRUE);
653 os_compiler_barrier();
654 CPU_DATA()->cpu_preemption_level--;
655 os_compiler_barrier();
656 }
657
658 static inline void
_enable_preemption_no_check(void)659 _enable_preemption_no_check(void)
660 {
661 enable_preemption_no_check();
662 }
663
664 static inline void
mp_disable_preemption(void)665 mp_disable_preemption(void)
666 {
667 disable_preemption_internal();
668 }
669
670 static inline void
_mp_disable_preemption(void)671 _mp_disable_preemption(void)
672 {
673 disable_preemption_internal();
674 }
675
676 static inline void
mp_enable_preemption(void)677 mp_enable_preemption(void)
678 {
679 enable_preemption_internal();
680 }
681
682 static inline void
_mp_enable_preemption(void)683 _mp_enable_preemption(void)
684 {
685 enable_preemption_internal();
686 }
687
688 static inline void
mp_enable_preemption_no_check(void)689 mp_enable_preemption_no_check(void)
690 {
691 enable_preemption_no_check();
692 }
693
694 static inline void
_mp_enable_preemption_no_check(void)695 _mp_enable_preemption_no_check(void)
696 {
697 enable_preemption_no_check();
698 }
699
700 #ifdef XNU_KERNEL_PRIVATE
701 #define disable_preemption() disable_preemption_internal()
702 #define disable_preemption_without_measurements() disable_preemption_internal()
703 #define enable_preemption() enable_preemption_internal()
704 #define MACHINE_PREEMPTION_MACROS (1)
705 #endif
706
707 static inline cpu_data_t *
cpu_datap(int cpu)708 cpu_datap(int cpu)
709 {
710 return cpu_data_ptr[cpu];
711 }
712
713 static inline int
cpu_is_running(int cpu)714 cpu_is_running(int cpu)
715 {
716 return (cpu_datap(cpu) != NULL) && (cpu_datap(cpu)->cpu_running);
717 }
718
719 #ifdef MACH_KERNEL_PRIVATE
720 static inline cpu_data_t *
cpu_shadowp(int cpu)721 cpu_shadowp(int cpu)
722 {
723 return cpu_data_ptr[cpu]->cd_shadow;
724 }
725
726 #endif
727 extern cpu_data_t *cpu_data_alloc(boolean_t is_boot_cpu);
728 extern void cpu_data_realloc(void);
729
730 #endif /* I386_CPU_DATA */
731