1 /* 2 * Copyright (c) 2007-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 * 31 */ 32 33 #ifndef ARM_CPU_DATA_INTERNAL 34 #define ARM_CPU_DATA_INTERNAL 35 36 #include <mach_assert.h> 37 #include <kern/assert.h> 38 #include <kern/kern_types.h> 39 #include <kern/percpu.h> 40 #include <kern/processor.h> 41 #include <os/base.h> 42 #include <pexpert/pexpert.h> 43 #include <arm/dbgwrap.h> 44 #include <arm/machine_routines.h> 45 #include <arm64/proc_reg.h> 46 #include <arm/thread.h> 47 #include <arm/pmap.h> 48 49 #if MONOTONIC 50 #include <machine/monotonic.h> 51 #endif /* MONOTONIC */ 52 53 #include <san/kcov_data.h> 54 55 #define NSEC_PER_HZ (NSEC_PER_SEC / 100) 56 57 typedef struct reset_handler_data { 58 vm_offset_t assist_reset_handler; /* Assist handler phys address */ 59 vm_offset_t cpu_data_entries; /* CpuDataEntries phys address */ 60 } reset_handler_data_t; 61 62 extern reset_handler_data_t ResetHandlerData; 63 64 /* Put the static check for cpumap_t here as it's defined in <kern/processor.h> */ 65 static_assert(sizeof(cpumap_t) * CHAR_BIT >= MAX_CPUS, "cpumap_t bitvector is too small for current MAX_CPUS value"); 66 67 #define CPUWINDOWS_BASE_MASK 0xFFFFFFFFFFE00000UL 68 #define CPUWINDOWS_BASE (VM_MAX_KERNEL_ADDRESS & CPUWINDOWS_BASE_MASK) 69 #define CPUWINDOWS_TOP (CPUWINDOWS_BASE + (MAX_CPUS * CPUWINDOWS_MAX * ARM_PGBYTES)) 70 71 static_assert((CPUWINDOWS_BASE >= VM_MIN_KERNEL_ADDRESS) && ((CPUWINDOWS_TOP - 1) <= VM_MAX_KERNEL_ADDRESS), 72 "CPU copy windows too large for CPUWINDOWS_BASE_MASK value"); 73 74 typedef struct cpu_data_entry { 75 void *cpu_data_paddr; /* Cpu data physical address */ 76 struct cpu_data *cpu_data_vaddr; /* Cpu data virtual address */ 77 #if !defined(__arm64__) 78 #error Check cpu_data_entry padding for this architecture 79 #endif 80 } cpu_data_entry_t; 81 82 83 typedef struct rtclock_timer { 84 mpqueue_head_t queue; 85 uint64_t deadline; 86 uint32_t is_set:1, 87 has_expired:1, 88 :0; 89 } rtclock_timer_t; 90 91 typedef struct { 92 /* 93 * The wake variants of these counters are reset to 0 when the CPU wakes. 94 */ 95 uint64_t irq_ex_cnt; 96 uint64_t irq_ex_cnt_wake; 97 uint64_t ipi_cnt; 98 uint64_t ipi_cnt_wake; 99 uint64_t timer_cnt; 100 #if MONOTONIC 101 uint64_t pmi_cnt_wake; 102 #endif /* MONOTONIC */ 103 uint64_t undef_ex_cnt; 104 uint64_t unaligned_cnt; 105 uint64_t vfp_cnt; 106 uint64_t data_ex_cnt; 107 uint64_t instr_ex_cnt; 108 } cpu_stat_t; 109 110 typedef struct cpu_data { 111 short cpu_number; 112 unsigned short cpu_flags; 113 int cpu_type; 114 int cpu_subtype; 115 int cpu_threadtype; 116 117 void * XNU_PTRAUTH_SIGNED_PTR("cpu_data.istackptr") istackptr; 118 vm_offset_t intstack_top; 119 #if __arm64__ 120 vm_offset_t excepstack_top; 121 #endif 122 thread_t cpu_active_thread; 123 vm_offset_t cpu_active_stack; 124 cpu_id_t cpu_id; 125 unsigned volatile int cpu_signal; 126 ast_t cpu_pending_ast; 127 cache_dispatch_t cpu_cache_dispatch; 128 129 #if __arm64__ 130 uint64_t cpu_base_timebase; 131 uint64_t cpu_timebase; 132 #endif 133 bool cpu_hibernate; /* This cpu is currently hibernating the system */ 134 bool cpu_running; 135 bool cluster_master; 136 #if __ARM_ARCH_8_5__ 137 bool sync_on_cswitch; 138 #endif /* __ARM_ARCH_8_5__ */ 139 /* true if processor_start() or processor_exit() is operating on this CPU */ 140 bool in_state_transition; 141 142 uint32_t cpu_decrementer; 143 get_decrementer_t cpu_get_decrementer_func; 144 set_decrementer_t cpu_set_decrementer_func; 145 fiq_handler_t cpu_get_fiq_handler; 146 147 void *cpu_tbd_hardware_addr; 148 void *cpu_tbd_hardware_val; 149 150 processor_idle_t cpu_idle_notify; 151 uint64_t cpu_idle_latency; 152 uint64_t cpu_idle_pop; 153 154 #if __ARM_KERNEL_PROTECT__ 155 vm_offset_t cpu_exc_vectors; 156 #endif /* __ARM_KERNEL_PROTECT__ */ 157 vm_offset_t cpu_reset_handler; 158 uintptr_t cpu_reset_assist; 159 uint32_t cpu_reset_type; 160 161 unsigned int interrupt_source; 162 void *cpu_int_state; 163 IOInterruptHandler interrupt_handler; 164 void *interrupt_nub; 165 void *interrupt_target; 166 void *interrupt_refCon; 167 168 idle_timer_t idle_timer_notify; 169 void *idle_timer_refcon; 170 uint64_t idle_timer_deadline; 171 172 uint64_t rtcPop; 173 rtclock_timer_t rtclock_timer; 174 struct _rtclock_data_ *rtclock_datap; 175 176 arm_debug_state_t *cpu_user_debug; /* Current debug state */ 177 vm_offset_t cpu_debug_interface_map; 178 179 volatile int debugger_active; 180 volatile int PAB_active; /* Tells the console if we are dumping backtraces */ 181 182 void *cpu_xcall_p0; 183 void *cpu_xcall_p1; 184 void *cpu_imm_xcall_p0; 185 void *cpu_imm_xcall_p1; 186 187 188 #if __arm64__ 189 vm_offset_t coresight_base[CORESIGHT_REGIONS]; 190 #endif 191 192 /* CCC ARMv8 registers */ 193 uint64_t cpu_regmap_paddr; 194 195 uint32_t cpu_phys_id; 196 uint32_t cpu_l2_access_penalty; 197 platform_error_handler_t platform_error_handler; 198 199 int cpu_mcount_off; 200 201 #define ARM_CPU_ON_SLEEP_PATH 0x50535553UL 202 volatile unsigned int cpu_sleep_token; 203 unsigned int cpu_sleep_token_last; 204 205 cluster_type_t cpu_cluster_type; 206 uint32_t cpu_cluster_id; 207 uint32_t cpu_l2_id; 208 uint32_t cpu_l2_size; 209 uint32_t cpu_l3_id; 210 uint32_t cpu_l3_size; 211 212 enum { 213 CPU_NOT_HALTED = 0, 214 CPU_HALTED, 215 CPU_HALTED_WITH_STATE 216 } halt_status; 217 #if defined(HAS_APPLE_PAC) 218 uint64_t rop_key; 219 uint64_t jop_key; 220 #endif /* defined(HAS_APPLE_PAC) */ 221 222 /* large structs with large alignment requirements */ 223 #if KPC 224 /* double-buffered performance counter data */ 225 uint64_t *cpu_kpc_buf[2]; 226 /* PMC shadow and reload value buffers */ 227 uint64_t *cpu_kpc_shadow; 228 uint64_t *cpu_kpc_reload; 229 #endif 230 #if MONOTONIC 231 struct mt_cpu cpu_monotonic; 232 #endif /* MONOTONIC */ 233 cpu_stat_t cpu_stat; 234 #if !XNU_MONITOR 235 struct pmap_cpu_data cpu_pmap_cpu_data; 236 #endif 237 dbgwrap_thread_state_t halt_state; 238 #if DEVELOPMENT || DEBUG 239 uint64_t wfe_count; 240 uint64_t wfe_deadline_checks; 241 uint64_t wfe_terminations; 242 #endif 243 #if CONFIG_KCOV 244 kcov_cpu_data_t cpu_kcov_data; 245 #endif 246 #if __arm64__ 247 /** 248 * Stash the state of the system when an IPI is received. This will be 249 * dumped in the case a panic is getting triggered. 250 */ 251 uint64_t ipi_pc; 252 uint64_t ipi_lr; 253 uint64_t ipi_fp; 254 255 /* Encoded data to store in TPIDR_EL0 on context switch */ 256 uint64_t cpu_tpidr_el0; 257 #endif 258 } cpu_data_t; 259 260 /* 261 * cpu_flags 262 */ 263 #define SleepState 0x0800 264 #define StartedState 0x1000 265 266 extern cpu_data_entry_t CpuDataEntries[MAX_CPUS]; 267 PERCPU_DECL(cpu_data_t, cpu_data); 268 #define BootCpuData __PERCPU_NAME(cpu_data) 269 extern boot_args *BootArgs; 270 271 #if __arm64__ 272 extern unsigned int LowResetVectorBase; 273 extern unsigned int LowResetVectorEnd; 274 #if WITH_CLASSIC_S2R 275 extern uint8_t SleepToken[8]; 276 #endif 277 extern unsigned int LowExceptionVectorBase; 278 #else 279 #error Unknown arch 280 #endif 281 282 extern cpu_data_t *cpu_datap(int cpu); 283 extern cpu_data_t *cpu_data_alloc(boolean_t is_boot); 284 extern void cpu_stack_alloc(cpu_data_t*); 285 extern void cpu_data_init(cpu_data_t *cpu_data_ptr); 286 extern void cpu_data_free(cpu_data_t *cpu_data_ptr); 287 extern kern_return_t cpu_data_register(cpu_data_t *cpu_data_ptr); 288 extern cpu_data_t *processor_to_cpu_datap( processor_t processor); 289 290 #if __arm64__ 291 typedef struct sysreg_restore { 292 uint64_t tcr_el1; 293 } sysreg_restore_t; 294 295 extern sysreg_restore_t sysreg_restore; 296 #endif /* __arm64__ */ 297 298 #endif /* ARM_CPU_DATA_INTERNAL */ 299