1 /* 2 * Copyright (c) 2007-2013 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * Processor registers for ARM64 30 */ 31 #ifndef _ARM64_PROC_REG_H_ 32 #define _ARM64_PROC_REG_H_ 33 34 #include <arm/proc_reg.h> 35 36 #if __ARM_KERNEL_PROTECT__ 37 /* 38 * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential 39 * architectural or microarchitectural vulnerabilities that could allow cores to 40 * read/access EL1-only mappings while in EL0 mode. This is achieved by 41 * removing as many mappings as possible when the core transitions to EL0 mode 42 * from EL1 mode, and restoring those mappings when the core transitions to EL1 43 * mode from EL0 mode. 44 * 45 * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is 46 * used to map and unmap the ordinary kernel mappings, by contracting and 47 * expanding translation zone size for TTBR1 when exiting and entering EL1, 48 * respectively: 49 * 50 * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is 51 * is in EL0. 52 * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in 53 * EL1. 54 * 55 * T1SZ_USER: T1SZ_BOOT + 1 56 * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT) 57 * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER) 58 * TTBR1_EL1_MAX: (2^64) - 1 59 * 60 * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the 61 * the following TTBR1 layout: 62 * 63 * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX 64 * +---------------------------------------------------------+ 65 * | Kernel EL0 Mappings | Kernel EL1 Mappings | 66 * +---------------------------------------------------------+ 67 * 68 * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout: 69 * 70 * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX 71 * +---------------------------------------------------------+ 72 * | Kernel EL0 Mappings | 73 * +---------------------------------------------------------+ 74 * 75 * With the current implementation, both the EL0 and EL1 mappings for the kernel 76 * use otherwise empty translation tables for mapping the exception vectors (so 77 * that we do not need to TLB flush the exception vector address when switching 78 * between EL0 and EL1). The rationale here is that the TLBI would require a 79 * DSB, and DSBs can be extremely expensive. 80 * 81 * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1 82 * ASID. The core switches between ASIDs on EL transitions, so that the TLB 83 * does not need to be fully invalidated on an EL transition. 84 * 85 * Most kernel mappings will be marked non-global in this configuration, as 86 * global mappings would be visible to userspace unless we invalidate them on 87 * eret. 88 */ 89 #if XNU_MONITOR 90 /* 91 * Please note that because we indirect through the thread register in order to 92 * locate the kernel, and because we unmap most of the kernel, the security 93 * model of the PPL is undermined by __ARM_KERNEL_PROTECT__, as we rely on 94 * kernel controlled data to direct codeflow in the exception vectors. 95 * 96 * If we want to ship XNU_MONITOR paired with __ARM_KERNEL_PROTECT__, we will 97 * need to find a performant solution to this problem. 98 */ 99 #endif 100 #endif /* __ARM_KERNEL_PROTECT */ 101 102 #if ARM_PARAMETERIZED_PMAP 103 /* 104 * ARM_PARAMETERIZED_PMAP configures the kernel to get the characteristics of 105 * the page tables (number of levels, size of the root allocation) from the 106 * pmap data structure, rather than treating them as compile-time constants. 107 * This allows the pmap code to dynamically adjust how it deals with page 108 * tables. 109 */ 110 #endif /* ARM_PARAMETERIZED_PMAP */ 111 112 #if __ARM_MIXED_PAGE_SIZE__ 113 /* 114 * __ARM_MIXED_PAGE_SIZE__ configures the kernel to support page tables that do 115 * not use the kernel page size. This is primarily meant to support running 116 * 4KB page processes on a 16KB page kernel. 117 * 118 * This only covers support in the pmap/machine dependent layers. Any support 119 * elsewhere in the kernel must be managed separately. 120 */ 121 #if !ARM_PARAMETERIZED_PMAP 122 /* 123 * Page tables that use non-kernel page sizes require us to reprogram TCR based 124 * on the page tables we are switching to. This means that the parameterized 125 * pmap support is required. 126 */ 127 #error __ARM_MIXED_PAGE_SIZE__ requires ARM_PARAMETERIZED_PMAP 128 #endif /* !ARM_PARAMETERIZED_PMAP */ 129 #if __ARM_KERNEL_PROTECT__ 130 /* 131 * Because switching the page size requires updating TCR based on the pmap, and 132 * __ARM_KERNEL_PROTECT__ relies on TCR being programmed with constants, XNU 133 * does not currently support support configurations that use both 134 * __ARM_KERNEL_PROTECT__ and __ARM_MIXED_PAGE_SIZE__. 135 */ 136 #error __ARM_MIXED_PAGE_SIZE__ and __ARM_KERNEL_PROTECT__ are mutually exclusive 137 #endif /* __ARM_KERNEL_PROTECT__ */ 138 #endif /* __ARM_MIXED_PAGE_SIZE__ */ 139 140 /* 141 * 64-bit Program Status Register (PSR64) 142 * 143 * 31 27 23 22 21 20 19 10 9 5 4 0 144 * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+ 145 * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M | 146 * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+ 147 * 148 * where: 149 * NZCV: Comparison flags 150 * PAN: Privileged Access Never 151 * SS: Single step 152 * IL: Illegal state 153 * DAIF: Interrupt masks 154 * M: Mode field 155 */ 156 157 #define PSR64_NZCV_SHIFT 28 158 #define PSR64_NZCV_MASK (0xF << PSR64_NZCV_SHIFT) 159 160 #define PSR64_N_SHIFT 31 161 #define PSR64_N (1 << PSR64_N_SHIFT) 162 163 #define PSR64_Z_SHIFT 30 164 #define PSR64_Z (1 << PSR64_Z_SHIFT) 165 166 #define PSR64_C_SHIFT 29 167 #define PSR64_C (1 << PSR64_C_SHIFT) 168 169 #define PSR64_V_SHIFT 28 170 #define PSR64_V (1 << PSR64_V_SHIFT) 171 172 #define PSR64_PAN_SHIFT 22 173 #define PSR64_PAN (1 << PSR64_PAN_SHIFT) 174 175 #define PSR64_SS_SHIFT 21 176 #define PSR64_SS (1 << PSR64_SS_SHIFT) 177 178 #define PSR64_IL_SHIFT 20 179 #define PSR64_IL (1 << PSR64_IL_SHIFT) 180 181 /* 182 * SSBS is bit 12 for A64 SPSR and bit 23 for A32 SPSR 183 * I do not want to talk about it! 184 */ 185 #define PSR64_SSBS_SHIFT_32 23 186 #define PSR64_SSBS_SHIFT_64 12 187 #define PSR64_SSBS_32 (1 << PSR64_SSBS_SHIFT_32) 188 #define PSR64_SSBS_64 (1 << PSR64_SSBS_SHIFT_64) 189 190 /* 191 * msr DAIF, Xn and mrs Xn, DAIF transfer into 192 * and out of bits 9:6 193 */ 194 #define DAIF_DEBUG_SHIFT 9 195 #define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT) 196 197 #define DAIF_ASYNC_SHIFT 8 198 #define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT) 199 200 #define DAIF_IRQF_SHIFT 7 201 #define DAIF_IRQF (1 << DAIF_IRQF_SHIFT) 202 203 #define DAIF_FIQF_SHIFT 6 204 #define DAIF_FIQF (1 << DAIF_FIQF_SHIFT) 205 206 #define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) 207 #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) 208 209 #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF)) 210 211 #if __ARM_ARCH_8_5__ 212 #define PSR64_SSBS_U32_DEFAULT PSR64_SSBS_32 213 #define PSR64_SSBS_U64_DEFAULT PSR64_SSBS_64 214 #define PSR64_SSBS_KRN_DEFAULT PSR64_SSBS_64 215 #else 216 #define PSR64_SSBS_U32_DEFAULT (0) 217 #define PSR64_SSBS_U64_DEFAULT (0) 218 #define PSR64_SSBS_KRN_DEFAULT (0) 219 #endif 220 221 /* 222 * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer 223 * from bits 3:0. 224 */ 225 #define DAIFSC_DEBUGF (1 << 3) 226 #define DAIFSC_ASYNCF (1 << 2) 227 #define DAIFSC_IRQF (1 << 1) 228 #define DAIFSC_FIQF (1 << 0) 229 #define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) 230 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) 231 232 /* 233 * ARM64_TODO: unify with ARM? 234 */ 235 #define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */ 236 237 #define PSR64_MODE_MASK 0x1F 238 239 #define PSR64_USER_MASK PSR64_NZCV_MASK 240 241 #define PSR64_MODE_USER32_THUMB 0x20 242 243 #define PSR64_MODE_RW_SHIFT 4 244 #define PSR64_MODE_RW_64 0 245 #define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT) 246 247 #define PSR64_MODE_EL_SHIFT 2 248 #define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT) 249 #define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT) 250 #define PSR64_MODE_EL2 (0x2 << PSR64_MODE_EL_SHIFT) 251 #define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT) 252 #define PSR64_MODE_EL0 0 253 254 #define PSR64_MODE_SPX 0x1 255 #define PSR64_MODE_SP0 0 256 257 #define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U32_DEFAULT) 258 #define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U64_DEFAULT) 259 #define PSR64_KERNEL_STANDARD (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0 | PSR64_SSBS_KRN_DEFAULT) 260 #if __ARM_PAN_AVAILABLE__ 261 #define PSR64_KERNEL_DEFAULT (PSR64_KERNEL_STANDARD | PSR64_PAN) 262 #else 263 #define PSR64_KERNEL_DEFAULT PSR64_KERNEL_STANDARD 264 #endif 265 266 #define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0) 267 #define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0) 268 269 #define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32)) 270 #define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32)) 271 272 273 274 /* 275 * System Control Register (SCTLR) 276 */ 277 278 #define SCTLR_DSSBS (1ULL << 44) 279 280 #define SCTLR_RESERVED ((3ULL << 28) | (1ULL << 20)) 281 #if defined(HAS_APPLE_PAC) 282 283 // 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled 284 #define SCTLR_PACIA_ENABLED_SHIFT 31 285 #define SCTLR_PACIA_ENABLED (1ULL << SCTLR_PACIA_ENABLED_SHIFT) 286 // 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled 287 #define SCTLR_PACIB_ENABLED (1ULL << 30) 288 // 29:28 RES1 11 289 // 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled 290 #define SCTLR_PACDA_ENABLED (1ULL << 27) 291 // 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled 292 #define SCTLR_PACDB_ENABLED (1ULL << 13) 293 294 #define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED) 295 #endif /* defined(HAS_APPLE_PAC) */ 296 297 // 26 UCI User Cache Instructions 298 #define SCTLR_UCI_ENABLED (1ULL << 26) 299 300 // 25 EE Exception Endianness 301 #define SCTLR_EE_BIG_ENDIAN (1ULL << 25) 302 303 // 24 E0E EL0 Endianness 304 #define SCTLR_E0E_BIG_ENDIAN (1ULL << 24) 305 306 // 23 SPAN Set PAN 307 #define SCTLR_PAN_UNCHANGED (1ULL << 23) 308 309 // 22 EIS Taking an exception is a context synchronization event 310 #define SCTLR_EIS (1ULL << 22) 311 312 // 21 RES0 0 313 // 20 RES1 1 314 315 // 19 WXN Writeable implies eXecute Never 316 #define SCTLR_WXN_ENABLED (1ULL << 19) 317 318 // 18 nTWE Not trap WFE from EL0 319 #define SCTLR_nTWE_WFE_ENABLED (1ULL << 18) 320 321 // 17 RES0 0 322 323 // 16 nTWI Not trap WFI from EL0 324 #define SCTRL_nTWI_WFI_ENABLED (1ULL << 16) 325 326 // 15 UCT User Cache Type register (CTR_EL0) 327 #define SCTLR_UCT_ENABLED (1ULL << 15) 328 329 // 14 DZE User Data Cache Zero (DC ZVA) 330 #define SCTLR_DZE_ENABLED (1ULL << 14) 331 332 // 12 I Instruction cache enable 333 #define SCTLR_I_ENABLED (1ULL << 12) 334 335 // 11 EOS Exception return is a context synchronization event 336 #define SCTLR_EOS (1ULL << 11) 337 338 // 10 EnRCTX EL0 Access to FEAT_SPECRES speculation restriction instructions 339 #define SCTLR_EnRCTX (1ULL << 10) 340 341 // 9 UMA User Mask Access 342 #define SCTLR_UMA_ENABLED (1ULL << 9) 343 344 // 8 SED SETEND Disable 345 #define SCTLR_SED_DISABLED (1ULL << 8) 346 347 // 7 ITD IT Disable 348 #define SCTLR_ITD_DISABLED (1ULL << 7) 349 350 // 6 RES0 0 351 352 // 5 CP15BEN CP15 Barrier ENable 353 #define SCTLR_CP15BEN_ENABLED (1ULL << 5) 354 355 // 4 SA0 Stack Alignment check for EL0 356 #define SCTLR_SA0_ENABLED (1ULL << 4) 357 358 // 3 SA Stack Alignment check 359 #define SCTLR_SA_ENABLED (1ULL << 3) 360 361 // 2 C Cache enable 362 #define SCTLR_C_ENABLED (1ULL << 2) 363 364 // 1 A Alignment check 365 #define SCTLR_A_ENABLED (1ULL << 1) 366 367 // 0 M MMU enable 368 #define SCTLR_M_ENABLED (1ULL << 0) 369 370 #if __ARM_ARCH_8_5__ 371 #define SCTLR_CSEH_DEFAULT (0) 372 #define SCTLR_DSSBS_DEFAULT SCTLR_DSSBS 373 #else 374 #define SCTLR_CSEH_DEFAULT (SCTLR_EIS | SCTLR_EOS) 375 #define SCTLR_DSSBS_DEFAULT (0) 376 #endif 377 378 #if HAS_APPLE_PAC 379 #define SCTLR_ROP_KEYS_DEFAULT SCTLR_PACIB_ENABLED /* IB is ROP */ 380 #else /* !HAS_APPLE_PAC */ 381 #define SCTLR_ROP_KEYS_DEFAULT 0 382 #endif /* HAS_APPLE_PAC */ 383 384 #if HAS_APPLE_PAC 385 #define SCTLR_JOP_KEYS_DEFAULT SCTLR_JOP_KEYS_ENABLED 386 #else /* !HAS_APPLE_PAC */ 387 #define SCTLR_JOP_KEYS_DEFAULT 0 388 #endif 389 390 #define SCTLR_EL1_DEFAULT \ 391 (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \ 392 SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | \ 393 SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED | \ 394 SCTLR_CSEH_DEFAULT | SCTLR_DSSBS_DEFAULT | \ 395 SCTLR_ROP_KEYS_DEFAULT | SCTLR_JOP_KEYS_DEFAULT) 396 397 /* 398 * Coprocessor Access Control Register (CPACR) 399 * 400 * 31 28 27 22 21 20 19 0 401 * +---+---+------+------+--------------------+ 402 * |000|TTA|000000| FPEN |00000000000000000000| 403 * +---+---+------+------+--------------------+ 404 * 405 * where: 406 * TTA: Trace trap 407 * FPEN: Floating point enable 408 */ 409 #define CPACR_TTA_SHIFT 28 410 #define CPACR_TTA (1 << CPACR_TTA_SHIFT) 411 412 #define CPACR_FPEN_SHIFT 20 413 #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT) 414 #define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT) 415 416 /* 417 * FPSR: Floating Point Status Register 418 * 419 * 31 30 29 28 27 26 7 6 4 3 2 1 0 420 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ 421 * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC| 422 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ 423 */ 424 425 #define FPSR_N_SHIFT 31 426 #define FPSR_Z_SHIFT 30 427 #define FPSR_C_SHIFT 29 428 #define FPSR_V_SHIFT 28 429 #define FPSR_QC_SHIFT 27 430 #define FPSR_IDC_SHIFT 7 431 #define FPSR_IXC_SHIFT 4 432 #define FPSR_UFC_SHIFT 3 433 #define FPSR_OFC_SHIFT 2 434 #define FPSR_DZC_SHIFT 1 435 #define FPSR_IOC_SHIFT 0 436 #define FPSR_N (1 << FPSR_N_SHIFT) 437 #define FPSR_Z (1 << FPSR_Z_SHIFT) 438 #define FPSR_C (1 << FPSR_C_SHIFT) 439 #define FPSR_V (1 << FPSR_V_SHIFT) 440 #define FPSR_QC (1 << FPSR_QC_SHIFT) 441 #define FPSR_IDC (1 << FPSR_IDC_SHIFT) 442 #define FPSR_IXC (1 << FPSR_IXC_SHIFT) 443 #define FPSR_UFC (1 << FPSR_UFC_SHIFT) 444 #define FPSR_OFC (1 << FPSR_OFC_SHIFT) 445 #define FPSR_DZC (1 << FPSR_DZC_SHIFT) 446 #define FPSR_IOC (1 << FPSR_IOC_SHIFT) 447 448 /* 449 * A mask for all for all of the bits that are not RAZ for FPSR; this 450 * is primarily for converting between a 32-bit view of NEON state 451 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). 452 */ 453 #define FPSR_MASK \ 454 (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \ 455 FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC) 456 457 /* 458 * FPCR: Floating Point Control Register 459 * 460 * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0 461 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ 462 * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000| 463 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ 464 */ 465 466 #define FPCR_AHP_SHIFT 26 467 #define FPCR_DN_SHIFT 25 468 #define FPCR_FZ_SHIFT 24 469 #define FPCR_RMODE_SHIFT 22 470 #define FPCR_STRIDE_SHIFT 20 471 #define FPCR_LEN_SHIFT 16 472 #define FPCR_IDE_SHIFT 15 473 #define FPCR_IXE_SHIFT 12 474 #define FPCR_UFE_SHIFT 11 475 #define FPCR_OFE_SHIFT 10 476 #define FPCR_DZE_SHIFT 9 477 #define FPCR_IOE_SHIFT 8 478 #define FPCR_AHP (1 << FPCR_AHP_SHIFT) 479 #define FPCR_DN (1 << FPCR_DN_SHIFT) 480 #define FPCR_FZ (1 << FPCR_FZ_SHIFT) 481 #define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT) 482 #define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT) 483 #define FPCR_LEN (0x7 << FPCR_LEN_SHIFT) 484 #define FPCR_IDE (1 << FPCR_IDE_SHIFT) 485 #define FPCR_IXE (1 << FPCR_IXE_SHIFT) 486 #define FPCR_UFE (1 << FPCR_UFE_SHIFT) 487 #define FPCR_OFE (1 << FPCR_OFE_SHIFT) 488 #define FPCR_DZE (1 << FPCR_DZE_SHIFT) 489 #define FPCR_IOE (1 << FPCR_IOE_SHIFT) 490 #define FPCR_DEFAULT (0) 491 #define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ) 492 493 /* 494 * A mask for all for all of the bits that are not RAZ for FPCR; this 495 * is primarily for converting between a 32-bit view of NEON state 496 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). 497 */ 498 #define FPCR_MASK \ 499 (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \ 500 FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE) 501 502 /* 503 * Translation Control Register (TCR) 504 * 505 * Legacy: 506 * 507 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 508 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ 509 * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| 510 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ 511 * 512 * Current (with 16KB granule support): 513 * 514 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 515 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ 516 * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| 517 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ 518 * 519 * TBI1: Top Byte Ignored for TTBR1 region 520 * TBI0: Top Byte Ignored for TTBR0 region 521 * AS: ASID Size 522 * IPS: Physical Address Size limit 523 * TG1: Granule Size for TTBR1 region 524 * SH1: Shareability for TTBR1 region 525 * ORGN1: Outer Cacheability for TTBR1 region 526 * IRGN1: Inner Cacheability for TTBR1 region 527 * EPD1: Translation table walk disable for TTBR1 528 * A1: ASID selection from TTBR1 enable 529 * T1SZ: Virtual address size for TTBR1 530 * TG0: Granule Size for TTBR0 region 531 * SH0: Shareability for TTBR0 region 532 * ORGN0: Outer Cacheability for TTBR0 region 533 * IRGN0: Inner Cacheability for TTBR0 region 534 * T0SZ: Virtual address size for TTBR0 535 */ 536 537 #define TCR_T0SZ_SHIFT 0ULL 538 #define TCR_TSZ_BITS 6ULL 539 #define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL) 540 541 #define TCR_IRGN0_SHIFT 8ULL 542 #define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT) 543 #define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT) 544 #define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT) 545 #define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT) 546 547 #define TCR_ORGN0_SHIFT 10ULL 548 #define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT) 549 #define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT) 550 #define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT) 551 #define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT) 552 553 #define TCR_SH0_SHIFT 12ULL 554 #define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT) 555 #define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT) 556 #define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT) 557 558 #define TCR_TG0_GRANULE_SHIFT (14ULL) 559 #define TCR_TG0_GRANULE_BITS (2ULL) 560 #define TCR_TG0_GRANULE_MASK ((1ULL << TCR_TG0_GRANULE_BITS) - 1ULL) 561 562 #define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT) 563 #define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT) 564 #define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT) 565 566 #if __ARM_16K_PG__ 567 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB) 568 #else 569 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB) 570 #endif 571 572 #define TCR_T1SZ_SHIFT 16ULL 573 574 #define TCR_A1_ASID1 (1ULL << 22ULL) 575 #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL) 576 577 #define TCR_IRGN1_SHIFT 24ULL 578 #define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT) 579 #define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT) 580 #define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT) 581 #define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT) 582 583 #define TCR_ORGN1_SHIFT 26ULL 584 #define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT) 585 #define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT) 586 #define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT) 587 #define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT) 588 589 #define TCR_SH1_SHIFT 28ULL 590 #define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT) 591 #define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT) 592 #define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT) 593 594 #define TCR_TG1_GRANULE_SHIFT 30ULL 595 596 #define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT) 597 #define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT) 598 #define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT) 599 600 #if __ARM_16K_PG__ 601 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB) 602 #else 603 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB) 604 #endif 605 606 #define TCR_IPS_SHIFT 32ULL 607 #define TCR_IPS_BITS 3ULL 608 #define TCR_IPS_MASK ((1ULL << TCR_IPS_BITS) - 1ULL) 609 #define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT) 610 #define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT) 611 #define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT) 612 #define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT) 613 #define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT) 614 #define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT) 615 616 #define TCR_AS_16BIT_ASID (1ULL << 36) 617 #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37) 618 #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38) 619 #define TCR_TBID0_TBI_DATA_ONLY (1ULL << 51) 620 #define TCR_TBID1_TBI_DATA_ONLY (1ULL << 52) 621 622 #if defined(HAS_APPLE_PAC) 623 #define TCR_TBID0_ENABLE TCR_TBID0_TBI_DATA_ONLY 624 #define TCR_TBID1_ENABLE TCR_TBID1_TBI_DATA_ONLY 625 #else 626 #define TCR_TBID0_ENABLE 0 627 #define TCR_TBID1_ENABLE 0 628 #endif 629 630 #define TCR_E0PD0_BIT (1ULL << 55) 631 #define TCR_E0PD1_BIT (1ULL << 56) 632 633 #if defined(HAS_E0PD) 634 #define TCR_E0PD_VALUE (TCR_E0PD1_BIT) 635 #else 636 #define TCR_E0PD_VALUE 0 637 #endif 638 639 640 /* 641 * Multiprocessor Affinity Register (MPIDR_EL1) 642 * 643 * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+ 644 * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0| 645 * +---------------------------------+--+-----+--+-----+----+----+ 646 * 647 * where 648 * U: Uniprocessor 649 * MT: Multi-threading at lowest affinity level 650 * Aff2: "1" - PCORE, "0" - ECORE 651 * Aff1: Cluster ID 652 * Aff0: CPU ID 653 */ 654 #define MPIDR_AFF0_SHIFT 0 655 #define MPIDR_AFF0_WIDTH 8 656 #define MPIDR_AFF0_MASK (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT) 657 #define MPIDR_AFF1_SHIFT 8 658 #define MPIDR_AFF1_WIDTH 8 659 #define MPIDR_AFF1_MASK (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT) 660 #define MPIDR_AFF2_SHIFT 16 661 #define MPIDR_AFF2_WIDTH 8 662 #define MPIDR_AFF2_MASK (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT) 663 664 /* 665 * TXSZ indicates the size of the range a TTBR covers. Currently, 666 * we support the following: 667 * 668 * 4KB pages, full page L1: 39 bit range. 669 * 4KB pages, sub-page L1: 38 bit range. 670 * 16KB pages, full page L1: 47 bit range. 671 * 16KB pages, sub-page L1: 39 bit range. 672 * 16KB pages, two level page tables: 36 bit range. 673 */ 674 #if __ARM_KERNEL_PROTECT__ 675 /* 676 * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the 677 * address space is used for the mappings that will remain in place when in EL0. 678 * As a result, 1 bit less of address space is available to the rest of the 679 * the kernel. 680 */ 681 #endif /* __ARM_KERNEL_PROTECT__ */ 682 #ifdef __ARM_16K_PG__ 683 #if __ARM64_PMAP_SUBPAGE_L1__ 684 #define T0SZ_BOOT 25ULL 685 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ 686 #define T0SZ_BOOT 17ULL 687 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ 688 #else /* __ARM_16K_PG__ */ 689 #if __ARM64_PMAP_SUBPAGE_L1__ 690 #define T0SZ_BOOT 26ULL 691 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 692 #define T0SZ_BOOT 25ULL 693 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 694 #endif /* __ARM_16K_PG__ */ 695 696 #if defined(APPLE_ARM64_ARCH_FAMILY) 697 /* T0SZ must be the same as T1SZ */ 698 #define T1SZ_BOOT T0SZ_BOOT 699 #else /* defined(APPLE_ARM64_ARCH_FAMILY) */ 700 #ifdef __ARM_16K_PG__ 701 #if __ARM64_PMAP_SUBPAGE_L1__ 702 #define T1SZ_BOOT 25ULL 703 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ 704 #define T1SZ_BOOT 17ULL 705 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ 706 #else /* __ARM_16K_PG__ */ 707 #if __ARM64_PMAP_SUBPAGE_L1__ 708 #define T1SZ_BOOT 26ULL 709 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 710 #define T1SZ_BOOT 25ULL 711 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/ 712 #endif /* __ARM_16K_PG__ */ 713 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ 714 715 #if __ARM_42BIT_PA_SPACE__ 716 #define TCR_IPS_VALUE TCR_IPS_42BITS 717 #else /* !__ARM_42BIT_PA_SPACE__ */ 718 #define TCR_IPS_VALUE TCR_IPS_40BITS 719 #endif /* !__ARM_42BIT_PA_SPACE__ */ 720 721 #if CONFIG_KERNEL_TBI 722 #define TCR_EL1_DTBI (TCR_TBI1_TOPBYTE_IGNORED | TCR_TBID1_ENABLE) 723 #else /* CONFIG_KERNEL_TBI */ 724 #define TCR_EL1_DTBI 0 725 #endif /* CONFIG_KERNEL_TBI */ 726 727 #define TCR_EL1_BASE \ 728 (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | \ 729 TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | \ 730 TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \ 731 TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) | \ 732 TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE) | TCR_E0PD_VALUE | \ 733 TCR_EL1_DTBI) 734 735 #if __ARM_KERNEL_PROTECT__ 736 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 737 #define T1SZ_USER (T1SZ_BOOT + 1) 738 #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 739 #else 740 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 741 #endif /* __ARM_KERNEL_PROTECT__ */ 742 743 #define TCR_EL1_4KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_4KB)) 744 #define TCR_EL1_16KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_16KB)) 745 746 747 748 749 /* 750 * Monitor Debug System Control Register (MDSCR) 751 */ 752 753 #define MDSCR_TFO_SHIFT 31 754 #define MDSCR_TFO (1ULL << MDSCR_TFO_SHIFT) 755 #define MDSCR_RXFULL_SHIFT 30 756 #define MDSCR_RXFULL (1ULL << MDSCR_RXFULL_SHIFT) 757 #define MDSCR_TXFULL_SHIFT 29 758 #define MDSCR_TXFULL (1ULL << MDSCR_TXFULL_SHIFT) 759 #define MDSCR_RXO_SHIFT 27 760 #define MDSCR_RXO (1ULL << MDSCR_RXO_SHIFT) 761 #define MDSCR_TXU_SHIFT 26 762 #define MDSCR_TXU (1ULL << MDSCR_TXU_SHIFT) 763 #define MDSCR_INTDIS_SHIFT 22 764 #define MDSCR_INTDIS_MASK (0x2U << MDSCR_INTDIS_SHIFT) 765 #define MDSCR_TDA_SHIFT 21 766 #define MDSCR_TDA (1ULL << MDSCR_TDA_SHIFT) 767 #define MDSCR_SC2_SHIFT 19 768 #define MDSCR_SC2 (1ULL << MDSCR_SC2_SHIFT) 769 #define MDSCR_MDE_SHIFT 15 770 #define MDSCR_MDE (1ULL << MDSCR_MDE_SHIFT) 771 #define MDSCR_HDE_SHIFT 14 772 #define MDSCR_HDE (1ULL << MDSCR_HDE_SHIFT) 773 #define MDSCR_KDE_SHIFT 13 774 #define MDSCR_KDE (1ULL << MDSCR_KDE_SHIFT) 775 #define MDSCR_TDCC_SHIFT 12 776 #define MDSCR_TDCC (1ULL << MDSCR_TDCC_SHIFT) 777 #define MDSCR_ERR_SHIFT 6 778 #define MDSCR_ERR (1ULL << MDSCR_ERR_SHIFT) 779 #define MDSCR_SS_SHIFT 0 780 #define MDSCR_SS (1ULL << MDSCR_SS_SHIFT) 781 782 /* 783 * Translation Table Base Register (TTBR) 784 * 785 * 63 48 47 x x-1 0 786 * +--------+------------------+------+ 787 * | ASID | Base Address | zero | 788 * +--------+------------------+------+ 789 * 790 */ 791 #define TTBR_ASID_SHIFT 48 792 #define TTBR_ASID_MASK 0xffff000000000000 793 794 #define TTBR_BADDR_MASK 0x0000ffffffffffff 795 796 /* 797 * Memory Attribute Indirection Register 798 * 799 * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0 800 * +-------+-------+-------+-------+-------+-------+-------+-------+ 801 * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 | 802 * +-------+-------+-------+-------+-------+-------+-------+-------+ 803 * 804 */ 805 806 #define MAIR_ATTR_SHIFT(x) (8*(x)) 807 808 /* Strongly ordered or device memory attributes */ 809 #define MAIR_OUTER_STRONGLY_ORDERED 0x0 810 #define MAIR_OUTER_DEVICE 0x0 811 812 #define MAIR_INNER_STRONGLY_ORDERED 0x0 813 #define MAIR_INNER_DEVICE 0x4 814 815 /* Normal memory attributes */ 816 #define MAIR_OUTER_NON_CACHEABLE 0x40 817 #define MAIR_OUTER_WRITE_THROUGH 0x80 818 #define MAIR_OUTER_WRITE_BACK 0xc0 819 820 #define MAIR_INNER_NON_CACHEABLE 0x4 821 #define MAIR_INNER_WRITE_THROUGH 0x8 822 #define MAIR_INNER_WRITE_BACK 0xc 823 824 /* Allocate policy for cacheable memory */ 825 #define MAIR_OUTER_WRITE_ALLOCATE 0x10 826 #define MAIR_OUTER_READ_ALLOCATE 0x20 827 828 #define MAIR_INNER_WRITE_ALLOCATE 0x1 829 #define MAIR_INNER_READ_ALLOCATE 0x2 830 831 /* Memory Atribute Encoding */ 832 833 /* 834 * Device memory types: 835 * G (gathering): multiple reads/writes can be combined 836 * R (reordering): reads or writes may reach device out of program order 837 * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes) 838 */ 839 #define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */ 840 #define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */ 841 #define MAIR_POSTED_REORDERED 0x08 /* Device Memory, nGRE (reorderable, posted writes) */ 842 #define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */ 843 #define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */ 844 #define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */ 845 #define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */ 846 #define MAIR_INNERWRITEBACK 0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */ 847 848 849 /* 850 * ARM 4-level Page Table support - 2*1024TB (2^48) of address space 851 */ 852 853 854 /* 855 * Memory Attribute Index. If these values change, please also update the pmap 856 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). 857 */ 858 #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ 859 #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ 860 #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ 861 #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */ 862 #define CACHE_ATTRINDX_INNERWRITEBACK 0x4 /* inner cache enabled, buffer enabled, write allocate (normal memory) */ 863 #define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes (device memory) */ 864 #define CACHE_ATTRINDX_POSTED_REORDERED 0x6 /* no cache, reorderable access, posted writes (device memory) */ 865 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */ 866 #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK 867 868 869 /* 870 * Access protection bit values (TTEs and PTEs), stage 1 871 * 872 * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access) 873 */ 874 #define AP_RWNA 0x0 /* priv=read-write, user=no-access */ 875 #define AP_RWRW 0x1 /* priv=read-write, user=read-write */ 876 #define AP_RONA 0x2 /* priv=read-only, user=no-access */ 877 #define AP_RORO 0x3 /* priv=read-only, user=read-only */ 878 #define AP_MASK 0x3 /* mask to find ap bits */ 879 880 /* 881 * Shareability attributes 882 */ 883 #define SH_NONE 0x0 /* Non shareable */ 884 #define SH_NONE 0x0 /* Device shareable */ 885 #define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */ 886 #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */ 887 #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */ 888 889 890 /* 891 * ARM Page Granule 892 */ 893 #ifdef __ARM_16K_PG__ 894 #define ARM_PGSHIFT 14 895 #else 896 #define ARM_PGSHIFT 12 897 #endif 898 #define ARM_PGBYTES (1 << ARM_PGSHIFT) 899 #define ARM_PGMASK (ARM_PGBYTES-1) 900 901 /* 902 * L0 Translation table 903 * 904 * 4KB granule size: 905 * Each translation table is 4KB 906 * 512 64-bit entries of 512GB (2^39) of address space. 907 * Covers 256TB (2^48) of address space. 908 * 909 * 16KB granule size: 910 * Each translation table is 16KB 911 * 2 64-bit entries of 128TB (2^47) of address space. 912 * Covers 256TB (2^48) of address space. 913 */ 914 915 /* 16K L0 */ 916 #define ARM_16K_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */ 917 #define ARM_16K_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */ 918 #define ARM_16K_TT_L0_SHIFT 47 /* page descriptor shift */ 919 #define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */ 920 921 /* 4K L0 */ 922 #define ARM_4K_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */ 923 #define ARM_4K_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */ 924 #define ARM_4K_TT_L0_SHIFT 39 /* page descriptor shift */ 925 #define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */ 926 927 /* 928 * L1 Translation table 929 * 930 * 4KB granule size: 931 * Each translation table is 4KB 932 * 512 64-bit entries of 1GB (2^30) of address space. 933 * Covers 512GB (2^39) of address space. 934 * 935 * 16KB granule size: 936 * Each translation table is 16KB 937 * 2048 64-bit entries of 64GB (2^36) of address space. 938 * Covers 128TB (2^47) of address space. 939 */ 940 941 /* 16K L1 */ 942 #define ARM_16K_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */ 943 #define ARM_16K_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */ 944 #define ARM_16K_TT_L1_SHIFT 36 /* page descriptor shift */ 945 #if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ 946 /* This config supports 512GB per TTBR. */ 947 #define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */ 948 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 949 #define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */ 950 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 951 952 /* 4K L1 */ 953 #define ARM_4K_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */ 954 #define ARM_4K_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */ 955 #define ARM_4K_TT_L1_SHIFT 30 /* page descriptor shift */ 956 #if __ARM64_PMAP_SUBPAGE_L1__ && !__ARM_16K_PG__ 957 /* This config supports 256GB per TTBR. */ 958 #define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */ 959 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 960 #define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */ 961 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 962 963 /* some sugar for getting pointers to page tables and entries */ 964 965 #define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT) 966 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) 967 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT) 968 969 #define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK)) 970 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK)) 971 972 /* 973 * L2 Translation table 974 * 975 * 4KB granule size: 976 * Each translation table is 4KB 977 * 512 64-bit entries of 2MB (2^21) of address space. 978 * Covers 1GB (2^30) of address space. 979 * 980 * 16KB granule size: 981 * Each translation table is 16KB 982 * 2048 64-bit entries of 32MB (2^25) of address space. 983 * Covers 64GB (2^36) of address space. 984 */ 985 986 /* 16K L2 */ 987 #define ARM_16K_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */ 988 #define ARM_16K_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */ 989 #define ARM_16K_TT_L2_SHIFT 25 /* page descriptor shift */ 990 #define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */ 991 992 /* 4K L2 */ 993 #define ARM_4K_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */ 994 #define ARM_4K_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */ 995 #define ARM_4K_TT_L2_SHIFT 21 /* page descriptor shift */ 996 #define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */ 997 998 /* 999 * L3 Translation table 1000 * 1001 * 4KB granule size: 1002 * Each translation table is 4KB 1003 * 512 64-bit entries of 4KB (2^12) of address space. 1004 * Covers 2MB (2^21) of address space. 1005 * 1006 * 16KB granule size: 1007 * Each translation table is 16KB 1008 * 2048 64-bit entries of 16KB (2^14) of address space. 1009 * Covers 32MB (2^25) of address space. 1010 */ 1011 1012 /* 16K L3 */ 1013 #define ARM_16K_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ 1014 #define ARM_16K_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */ 1015 #define ARM_16K_TT_L3_SHIFT 14 /* page descriptor shift */ 1016 #define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */ 1017 1018 /* 4K L3 */ 1019 #define ARM_4K_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ 1020 #define ARM_4K_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */ 1021 #define ARM_4K_TT_L3_SHIFT 12 /* page descriptor shift */ 1022 #define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */ 1023 1024 #ifdef __ARM_16K_PG__ 1025 1026 /* Native L0 defines */ 1027 #define ARM_TT_L0_SIZE ARM_16K_TT_L0_SIZE 1028 #define ARM_TT_L0_OFFMASK ARM_16K_TT_L0_OFFMASK 1029 #define ARM_TT_L0_SHIFT ARM_16K_TT_L0_SHIFT 1030 #define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK 1031 1032 /* Native L1 defines */ 1033 #define ARM_TT_L1_SIZE ARM_16K_TT_L1_SIZE 1034 #define ARM_TT_L1_OFFMASK ARM_16K_TT_L1_OFFMASK 1035 #define ARM_TT_L1_SHIFT ARM_16K_TT_L1_SHIFT 1036 #define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK 1037 1038 /* Native L2 defines */ 1039 #define ARM_TT_L2_SIZE ARM_16K_TT_L2_SIZE 1040 #define ARM_TT_L2_OFFMASK ARM_16K_TT_L2_OFFMASK 1041 #define ARM_TT_L2_SHIFT ARM_16K_TT_L2_SHIFT 1042 #define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK 1043 1044 /* Native L3 defines */ 1045 #define ARM_TT_L3_SIZE ARM_16K_TT_L3_SIZE 1046 #define ARM_TT_L3_OFFMASK ARM_16K_TT_L3_OFFMASK 1047 #define ARM_TT_L3_SHIFT ARM_16K_TT_L3_SHIFT 1048 #define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK 1049 1050 #else /* !__ARM_16K_PG__ */ 1051 1052 /* Native L0 defines */ 1053 #define ARM_TT_L0_SIZE ARM_4K_TT_L0_SIZE 1054 #define ARM_TT_L0_OFFMASK ARM_4K_TT_L0_OFFMASK 1055 #define ARM_TT_L0_SHIFT ARM_4K_TT_L0_SHIFT 1056 #define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK 1057 1058 /* Native L1 defines */ 1059 #define ARM_TT_L1_SIZE ARM_4K_TT_L1_SIZE 1060 #define ARM_TT_L1_OFFMASK ARM_4K_TT_L1_OFFMASK 1061 #define ARM_TT_L1_SHIFT ARM_4K_TT_L1_SHIFT 1062 #define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK 1063 1064 /* Native L2 defines */ 1065 #define ARM_TT_L2_SIZE ARM_4K_TT_L2_SIZE 1066 #define ARM_TT_L2_OFFMASK ARM_4K_TT_L2_OFFMASK 1067 #define ARM_TT_L2_SHIFT ARM_4K_TT_L2_SHIFT 1068 #define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK 1069 1070 /* Native L3 defines */ 1071 #define ARM_TT_L3_SIZE ARM_4K_TT_L3_SIZE 1072 #define ARM_TT_L3_OFFMASK ARM_4K_TT_L3_OFFMASK 1073 #define ARM_TT_L3_SHIFT ARM_4K_TT_L3_SHIFT 1074 #define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK 1075 1076 #endif /* !__ARM_16K_PG__ */ 1077 1078 /* 1079 * Convenience definitions for: 1080 * ARM_TT_LEAF: The last level of the configured page table format. 1081 * ARM_TT_TWIG: The second to last level of the configured page table format. 1082 * ARM_TT_ROOT: The first level of the configured page table format. 1083 * 1084 * My apologies to any botanists who may be reading this. 1085 */ 1086 #define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE 1087 #define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK 1088 #define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT 1089 #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK 1090 1091 #define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE 1092 #define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK 1093 #define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT 1094 #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK 1095 1096 #define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE 1097 #define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK 1098 #define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT 1099 #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK 1100 1101 /* 1102 * 4KB granule size: 1103 * 1104 * Level 0 Translation Table Entry 1105 * 1106 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1107 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1108 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| 1109 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1110 * 1111 * Level 1 Translation Table Entry 1112 * 1113 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1114 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1115 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| 1116 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1117 * 1118 * Level 1 Translation Block Entry 1119 * 1120 * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0 1121 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1122 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1123 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1124 * 1125 * Level 2 Translation Table Entry 1126 * 1127 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1128 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1129 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| 1130 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1131 * 1132 * Level 2 Translation Block Entry 1133 * 1134 * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0 1135 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1136 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1137 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1138 * 1139 * 16KB granule size: 1140 * 1141 * Level 0 Translation Table Entry 1142 * 1143 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1144 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1145 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| 1146 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1147 * 1148 * Level 1 Translation Table Entry 1149 * 1150 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1151 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1152 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| 1153 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1154 * 1155 * Level 2 Translation Table Entry 1156 * 1157 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1158 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1159 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| 1160 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1161 * 1162 * Level 2 Translation Block Entry 1163 * 1164 * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0 1165 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1166 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1167 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1168 * 1169 * where: 1170 * nG: notGlobal bit 1171 * SH: Shareability field 1172 * AP: access protection 1173 * XN: eXecute Never bit 1174 * PXN: Privilege eXecute Never bit 1175 * NS: Non-Secure bit 1176 * HINT: 16 entry continuguous output hint 1177 * AttrIdx: Memory Attribute Index 1178 */ 1179 1180 #define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */ 1181 #ifdef __ARM_16K_PG__ 1182 #define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */ 1183 #else 1184 #define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */ 1185 #endif 1186 1187 #define ARM_TTE_MAX (TTE_PGENTRIES) 1188 1189 #define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ 1190 #define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */ 1191 1192 #define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */ 1193 1194 #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ 1195 #define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */ 1196 #define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */ 1197 #define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL 1198 #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ 1199 1200 #ifdef __ARM_16K_PG__ 1201 /* 1202 * Note that L0/L1 block entries are disallowed for the 16KB granule size; what 1203 * are we doing with these? 1204 */ 1205 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */ 1206 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */ 1207 #define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */ 1208 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */ 1209 #define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */ 1210 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */ 1211 #else 1212 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */ 1213 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */ 1214 #define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */ 1215 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */ 1216 #define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */ 1217 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */ 1218 #endif 1219 1220 #define ARM_TTE_BLOCK_APSHIFT 6 1221 #define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */ 1222 #define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT) 1223 1224 #define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */ 1225 #define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ 1226 1227 #define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */ 1228 #define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */ 1229 1230 #define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */ 1231 #define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */ 1232 1233 #define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */ 1234 #define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ 1235 1236 #define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */ 1237 #define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ 1238 1239 #define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ 1240 #define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ 1241 1242 #define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */ 1243 #define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */ 1244 1245 #define ARM_TTE_BLOCK_WIRED 0x0400000000000000ULL /* value for software wired bit */ 1246 #define ARM_TTE_BLOCK_WIREDMASK 0x0400000000000000ULL /* software wired mask */ 1247 1248 #define ARM_TTE_BLOCK_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ 1249 #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL /* software writeable mask */ 1250 1251 #define ARM_TTE_TABLE_MASK 0x0000fffffffff000ULL /* mask for extracting pointer to next table (works at any level) */ 1252 1253 #define ARM_TTE_TABLE_APSHIFT 61 1254 #define ARM_TTE_TABLE_AP_NO_EFFECT 0x0ULL 1255 #define ARM_TTE_TABLE_AP_USER_NA 0x1ULL 1256 #define ARM_TTE_TABLE_AP_RO 0x2ULL 1257 #define ARM_TTE_TABLE_AP_KERN_RO 0x3ULL 1258 #define ARM_TTE_TABLE_AP(x) ((x) << ARM_TTE_TABLE_APSHIFT) /* access protection */ 1259 1260 #define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */ 1261 #define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */ 1262 1263 #define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */ 1264 #define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */ 1265 1266 #define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */ 1267 #define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */ 1268 1269 #if __ARM_KERNEL_PROTECT__ 1270 #define ARM_TTE_BOOT_BLOCK \ 1271 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ 1272 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG) 1273 #else /* __ARM_KERNEL_PROTECT__ */ 1274 #define ARM_TTE_BOOT_BLOCK \ 1275 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ 1276 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF) 1277 #endif /* __ARM_KERNEL_PROTECT__ */ 1278 1279 #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID ) 1280 /* 1281 * L3 Translation table 1282 * 1283 * 4KB granule size: 1284 * Each translation table is 4KB 1285 * 512 64-bit entries of 4KB (2^12) of address space. 1286 * Covers 2MB (2^21) of address space. 1287 * 1288 * 16KB granule size: 1289 * Each translation table is 16KB 1290 * 2048 64-bit entries of 16KB (2^14) of address space. 1291 * Covers 32MB (2^25) of address space. 1292 */ 1293 1294 #ifdef __ARM_16K_PG__ 1295 #define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ 1296 #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */ 1297 #define ARM_PTE_SHIFT 14 /* page descriptor shift */ 1298 #define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */ 1299 #else 1300 #define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ 1301 #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */ 1302 #define ARM_PTE_SHIFT 12 /* page descriptor shift */ 1303 #define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */ 1304 #endif 1305 1306 #define ARM_TTE_PA_MASK 0x0000fffffffff000ULL 1307 1308 /* 1309 * L3 Page table entries 1310 * 1311 * The following page table entry types are possible: 1312 * 1313 * fault page entry 1314 * 63 2 0 1315 * +------------------------------+--+ 1316 * | ignored |00| 1317 * +------------------------------+--+ 1318 * 1319 * 1320 * 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0 1321 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+ 1322 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V| 1323 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+ 1324 * 1325 * where: 1326 * nG: notGlobal bit 1327 * SH: Shareability field 1328 * AP: access protection 1329 * XN: eXecute Never bit 1330 * PXN: Privilege eXecute Never bit 1331 * NS: Non-Secure bit 1332 * HINT: 16 entry continuguous output hint 1333 * AttrIdx: Memory Attribute Index 1334 */ 1335 1336 #define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */ 1337 #ifdef __ARM_16K_PG__ 1338 #define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */ 1339 #else 1340 #define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */ 1341 #endif 1342 1343 #define ARM_PTE_EMPTY 0x0000000000000000ULL /* unassigned - invalid entry */ 1344 1345 /* markers for (invalid) PTE for a page sent to compressor */ 1346 #define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */ 1347 #define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */ 1348 #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL 1349 1350 #define ARM_PTE_IS_COMPRESSED(x, p) \ 1351 ((((x) & 0x3) == 0) && /* PTE is not valid... */ \ 1352 ((x) & ARM_PTE_COMPRESSED) && /* ...has "compressed" marker" */ \ 1353 ((!((x) & ~ARM_PTE_COMPRESSED_MASK)) || /* ...no other bits */ \ 1354 (panic("compressed PTE %p 0x%llx has extra bits 0x%llx: corrupted?", \ 1355 (p), (x), (x) & ~ARM_PTE_COMPRESSED_MASK), FALSE))) 1356 1357 #define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ 1358 #define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ 1359 #define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */ 1360 #define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */ 1361 1362 /* This mask works for both 16K and 4K pages because bits 12-13 will be zero in 16K pages */ 1363 #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* output address mask for page */ 1364 #define ARM_PTE_PAGE_SHIFT 12 /* page shift for the output address in the entry */ 1365 1366 #define ARM_PTE_AP(x) ((x) << 6) /* access protections */ 1367 #define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */ 1368 #define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */ 1369 1370 #define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */ 1371 #define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ 1372 1373 #define ARM_PTE_SH(x) ((x) << 8) /* access shared */ 1374 #define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */ 1375 1376 #define ARM_PTE_AF 0x0000000000000400ULL /* value for access */ 1377 #define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */ 1378 1379 #define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */ 1380 #define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ 1381 1382 #define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */ 1383 #define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ 1384 1385 #define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */ 1386 #define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */ 1387 1388 #if __ARM_16K_PG__ 1389 #define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */ 1390 #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */ 1391 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */ 1392 #define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */ 1393 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */ 1394 #else 1395 #define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */ 1396 #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */ 1397 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */ 1398 #define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */ 1399 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */ 1400 #endif 1401 1402 #define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ 1403 #define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ 1404 1405 #define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */ 1406 #define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */ 1407 1408 #define ARM_PTE_XMASK (ARM_PTE_PNXMASK | ARM_PTE_NXMASK) 1409 1410 #define ARM_PTE_WIRED 0x0400000000000000ULL /* value for software wired bit */ 1411 #define ARM_PTE_WIRED_MASK 0x0400000000000000ULL /* software wired mask */ 1412 1413 #define ARM_PTE_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ 1414 #define ARM_PTE_WRITEABLE_MASK 0x0800000000000000ULL /* software writeable mask */ 1415 1416 #define ARM_PTE_BOOT_PAGE_BASE \ 1417 (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | \ 1418 ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF) 1419 1420 #if __ARM_KERNEL_PROTECT__ 1421 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG) 1422 #else /* __ARM_KERNEL_PROTECT__ */ 1423 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE) 1424 #endif /* __ARM_KERNEL_PROTECT__ */ 1425 1426 /* 1427 * TLBI appers to only deal in 4KB page addresses, so give 1428 * it an explicit shift of 12. 1429 */ 1430 #define TLBI_ADDR_SHIFT (0) 1431 #define TLBI_ADDR_SIZE (44) 1432 #define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) 1433 #define TLBI_ASID_SHIFT (48) 1434 #define TLBI_ASID_SIZE (16) 1435 #define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1)) 1436 1437 #define RTLBI_ADDR_SIZE (37) 1438 #define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1) 1439 #define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT 1440 #define RTLBI_TG(_page_shift_) ((uint64_t)((((_page_shift_) - 12) >> 1) + 1) << 46) 1441 #define RTLBI_SCALE_SHIFT (44) 1442 #define RTLBI_NUM_SHIFT (39) 1443 1444 /* 1445 * Exception Syndrome Register 1446 * 1447 * 31 26 25 24 0 1448 * +------+--+------------------+ 1449 * | EC |IL| ISS | 1450 * +------+--+------------------+ 1451 * 1452 * EC - Exception Class 1453 * IL - Instruction Length 1454 * ISS - Instruction Specific Syndrome 1455 * 1456 * Note: The ISS can have many forms. These are defined separately below. 1457 */ 1458 1459 #define ESR_EC_SHIFT 26 1460 #define ESR_EC_MASK (0x3FULL << ESR_EC_SHIFT) 1461 #define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT) 1462 1463 #define ESR_IL_SHIFT 25 1464 #define ESR_IL (1 << ESR_IL_SHIFT) 1465 1466 #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL)) 1467 1468 #define ESR_ISS_MASK 0x01FFFFFF 1469 #define ESR_ISS(x) (x & ESR_ISS_MASK) 1470 1471 #ifdef __ASSEMBLER__ 1472 /* Define only the classes we need to test in the exception vectors. */ 1473 #define ESR_EC_IABORT_EL1 0x21 1474 #define ESR_EC_DABORT_EL1 0x25 1475 #define ESR_EC_SP_ALIGN 0x26 1476 #else 1477 typedef enum { 1478 ESR_EC_UNCATEGORIZED = 0x00, 1479 ESR_EC_WFI_WFE = 0x01, 1480 ESR_EC_MCR_MRC_CP15_TRAP = 0x03, 1481 ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04, 1482 ESR_EC_MCR_MRC_CP14_TRAP = 0x05, 1483 ESR_EC_LDC_STC_CP14_TRAP = 0x06, 1484 ESR_EC_TRAP_SIMD_FP = 0x07, 1485 ESR_EC_PTRAUTH_INSTR_TRAP = 0x09, 1486 ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c, 1487 ESR_EC_ILLEGAL_INSTR_SET = 0x0e, 1488 ESR_EC_SVC_32 = 0x11, 1489 ESR_EC_SVC_64 = 0x15, 1490 ESR_EC_MSR_TRAP = 0x18, 1491 #ifdef __ARM_ARCH_8_6__ 1492 ESR_EC_PAC_FAIL = 0x1C, 1493 #endif /* __ARM_ARCH_8_6__ */ 1494 ESR_EC_IABORT_EL0 = 0x20, 1495 ESR_EC_IABORT_EL1 = 0x21, 1496 ESR_EC_PC_ALIGN = 0x22, 1497 ESR_EC_DABORT_EL0 = 0x24, 1498 ESR_EC_DABORT_EL1 = 0x25, 1499 ESR_EC_SP_ALIGN = 0x26, 1500 ESR_EC_FLOATING_POINT_32 = 0x28, 1501 ESR_EC_FLOATING_POINT_64 = 0x2C, 1502 ESR_EC_SERROR_INTERRUPT = 0x2F, 1503 ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL. 1504 ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL. 1505 ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL. 1506 ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL. 1507 ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL. 1508 ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL. 1509 ESR_EC_BKPT_AARCH32 = 0x38, 1510 ESR_EC_BRK_AARCH64 = 0x3C, 1511 } esr_exception_class_t; 1512 1513 typedef enum { 1514 FSC_TRANSLATION_FAULT_L0 = 0x04, 1515 FSC_TRANSLATION_FAULT_L1 = 0x05, 1516 FSC_TRANSLATION_FAULT_L2 = 0x06, 1517 FSC_TRANSLATION_FAULT_L3 = 0x07, 1518 FSC_ACCESS_FLAG_FAULT_L1 = 0x09, 1519 FSC_ACCESS_FLAG_FAULT_L2 = 0x0A, 1520 FSC_ACCESS_FLAG_FAULT_L3 = 0x0B, 1521 FSC_PERMISSION_FAULT_L1 = 0x0D, 1522 FSC_PERMISSION_FAULT_L2 = 0x0E, 1523 FSC_PERMISSION_FAULT_L3 = 0x0F, 1524 FSC_SYNC_EXT_ABORT = 0x10, 1525 FSC_ASYNC_EXT_ABORT = 0x11, 1526 FSC_SYNC_EXT_ABORT_TT_L1 = 0x15, 1527 FSC_SYNC_EXT_ABORT_TT_L2 = 0x16, 1528 FSC_SYNC_EXT_ABORT_TT_L3 = 0x17, 1529 FSC_SYNC_PARITY = 0x18, 1530 FSC_ASYNC_PARITY = 0x19, 1531 FSC_SYNC_PARITY_TT_L1 = 0x1D, 1532 FSC_SYNC_PARITY_TT_L2 = 0x1E, 1533 FSC_SYNC_PARITY_TT_L3 = 0x1F, 1534 FSC_ALIGNMENT_FAULT = 0x21, 1535 FSC_DEBUG_FAULT = 0x22, 1536 } fault_status_t; 1537 #endif /* ASSEMBLER */ 1538 1539 /* 1540 * Software step debug event ISS (EL1) 1541 * 24 23 6 5 0 1542 * +---+-----------------+--+------+ 1543 * |ISV|00000000000000000|EX| IFSC | 1544 * +---+-----------------+--+------+ 1545 * 1546 * where: 1547 * ISV: Instruction syndrome valid 1548 * EX: Exclusive access 1549 * IFSC: Instruction Fault Status Code 1550 */ 1551 1552 #define ISS_SSDE_ISV_SHIFT 24 1553 #define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT) 1554 1555 #define ISS_SSDE_EX_SHIFT 6 1556 #define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT) 1557 1558 #define ISS_SSDE_FSC_MASK 0x3F 1559 #define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK) 1560 1561 /* 1562 * Instruction Abort ISS (EL1) 1563 * 24 10 9 5 0 1564 * +---------------+--+---+------+ 1565 * |000000000000000|EA|000| IFSC | 1566 * +---------------+--+---+------+ 1567 * 1568 * where: 1569 * EA: External Abort type 1570 * IFSC: Instruction Fault Status Code 1571 */ 1572 1573 #define ISS_IA_EA_SHIFT 9 1574 #define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT) 1575 1576 #define ISS_IA_FSC_MASK 0x3F 1577 #define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK) 1578 1579 1580 /* 1581 * Data Abort ISS (EL1) 1582 * 1583 * 24 9 8 7 6 5 0 1584 * +---------------+--+--+-+---+----+ 1585 * |000000000000000|EA|CM|S1PTW|WnR|DFSC| 1586 * +---------------+--+--+-+---+----+ 1587 * 1588 * where: 1589 * EA: External Abort type 1590 * CM: Cache Maintenance operation 1591 * WnR: Write not Read 1592 * S1PTW: Stage 2 exception on Stage 1 page table walk 1593 * DFSC: Data Fault Status Code 1594 */ 1595 #define ISS_DA_EA_SHIFT 9 1596 #define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT) 1597 1598 #define ISS_DA_CM_SHIFT 8 1599 #define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT) 1600 1601 #define ISS_DA_WNR_SHIFT 6 1602 #define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT) 1603 1604 #define ISS_DA_S1PTW_SHIFT 7 1605 #define ISS_DA_S1PTW (0x1 << ISS_DA_S1PTW_SHIFT) 1606 1607 #define ISS_DA_FSC_MASK 0x3F 1608 #define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK) 1609 1610 /* 1611 * Floating Point Exception ISS (EL1) 1612 * 1613 * 24 23 22 8 7 4 3 2 1 0 1614 * +-+---+---------------+---+--+---+---+---+---+---+ 1615 * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF| 1616 * +-+---+---------------+---+--+---+---+---+---+---+ 1617 * 1618 * where: 1619 * TFV: Trapped Fault Valid 1620 * IDF: Input Denormal Exception 1621 * IXF: Input Inexact Exception 1622 * UFF: Underflow Exception 1623 * OFF: Overflow Exception 1624 * DZF: Divide by Zero Exception 1625 * IOF: Invalid Operation Exception 1626 */ 1627 #define ISS_FP_TFV_SHIFT 23 1628 #define ISS_FP_TFV (0x1 << ISS_FP_TFV_SHIFT) 1629 1630 #define ISS_FP_IDF_SHIFT 7 1631 #define ISS_FP_IDF (0x1 << ISS_FP_IDF_SHIFT) 1632 1633 #define ISS_FP_IXF_SHIFT 4 1634 #define ISS_FP_IXF (0x1 << ISS_FP_IXF_SHIFT) 1635 1636 #define ISS_FP_UFF_SHIFT 3 1637 #define ISS_FP_UFF (0x1 << ISS_FP_UFF_SHIFT) 1638 1639 #define ISS_FP_OFF_SHIFT 2 1640 #define ISS_FP_OFF (0x1 << ISS_FP_OFF_SHIFT) 1641 1642 #define ISS_FP_DZF_SHIFT 1 1643 #define ISS_FP_DZF (0x1 << ISS_FP_DZF_SHIFT) 1644 1645 #define ISS_FP_IOF_SHIFT 0 1646 #define ISS_FP_IOF (0x1 << ISS_FP_IOF_SHIFT) 1647 1648 /* 1649 * Breakpoint Exception ISS (EL1) 1650 * 24 16 0 1651 * +---------+---------+ 1652 * |000000000| Comment | 1653 * +---------+---------+ 1654 * 1655 * where: 1656 * Comment: Instruction Comment Field Value 1657 */ 1658 #define ISS_BRK_COMMENT_MASK 0xFFFF 1659 #define ISS_BRK_COMMENT(x) (x & ISS_BRK_COMMENT_MASK) 1660 1661 1662 #if HAS_UCNORMAL_MEM 1663 #define ISS_UC 0x11 1664 #endif /* HAS_UCNORMAL_MEM */ 1665 1666 1667 1668 /* 1669 * Physical Address Register (EL1) 1670 */ 1671 #define PAR_F_SHIFT 0 1672 #define PAR_F (0x1 << PAR_F_SHIFT) 1673 1674 #define PLATFORM_SYSCALL_TRAP_NO 0x80000000 1675 1676 #define ARM64_SYSCALL_CODE_REG_NUM (16) 1677 1678 #define ARM64_CLINE_SHIFT 6 1679 1680 #if defined(APPLE_ARM64_ARCH_FAMILY) 1681 #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */ 1682 #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */ 1683 #endif 1684 1685 /* 1686 * Timer definitions. 1687 */ 1688 #define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */ 1689 #define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */ 1690 #define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */ 1691 #define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */ 1692 #define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */ 1693 #define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */ 1694 #define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to virtual timebase + frequency reg enabled */ 1695 #define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to physical timebase + frequency reg enabled */ 1696 1697 #define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */ 1698 #define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */ 1699 #define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */ 1700 1701 #define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS 1702 #define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED 1703 #define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE 1704 1705 /* 1706 * At present all other uses of ARM_DBG_* are shared bit compatibly with the 32bit definitons. 1707 * (cf. osfmk/arm/proc_reg.h) 1708 */ 1709 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */ 1710 1711 #define MIDR_EL1_REV_SHIFT 0 1712 #define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT) 1713 #define MIDR_EL1_PNUM_SHIFT 4 1714 #define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT) 1715 #define MIDR_EL1_ARCH_SHIFT 16 1716 #define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT) 1717 #define MIDR_EL1_VAR_SHIFT 20 1718 #define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT) 1719 #define MIDR_EL1_IMP_SHIFT 24 1720 #define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT) 1721 1722 #define MIDR_FIJI (0x002 << MIDR_EL1_PNUM_SHIFT) 1723 #define MIDR_CAPRI (0x003 << MIDR_EL1_PNUM_SHIFT) 1724 #define MIDR_MAUI (0x004 << MIDR_EL1_PNUM_SHIFT) 1725 #define MIDR_ELBA (0x005 << MIDR_EL1_PNUM_SHIFT) 1726 #define MIDR_CAYMAN (0x006 << MIDR_EL1_PNUM_SHIFT) 1727 #define MIDR_MYST (0x007 << MIDR_EL1_PNUM_SHIFT) 1728 #define MIDR_SKYE_MONSOON (0x008 << MIDR_EL1_PNUM_SHIFT) 1729 #define MIDR_SKYE_MISTRAL (0x009 << MIDR_EL1_PNUM_SHIFT) 1730 #define MIDR_CYPRUS_VORTEX (0x00B << MIDR_EL1_PNUM_SHIFT) 1731 #define MIDR_CYPRUS_TEMPEST (0x00C << MIDR_EL1_PNUM_SHIFT) 1732 #define MIDR_M9 (0x00F << MIDR_EL1_PNUM_SHIFT) 1733 #define MIDR_ARUBA_VORTEX (0x010 << MIDR_EL1_PNUM_SHIFT) 1734 #define MIDR_ARUBA_TEMPEST (0x011 << MIDR_EL1_PNUM_SHIFT) 1735 1736 #ifdef APPLELIGHTNING 1737 #define MIDR_CEBU_LIGHTNING (0x012 << MIDR_EL1_PNUM_SHIFT) 1738 #define MIDR_CEBU_THUNDER (0x013 << MIDR_EL1_PNUM_SHIFT) 1739 #define MIDR_TURKS (0x026 << MIDR_EL1_PNUM_SHIFT) 1740 #endif 1741 1742 #ifdef APPLEFIRESTORM 1743 #define MIDR_SICILY_ICESTORM (0x020 << MIDR_EL1_PNUM_SHIFT) 1744 #define MIDR_SICILY_FIRESTORM (0x021 << MIDR_EL1_PNUM_SHIFT) 1745 #define MIDR_TONGA_ICESTORM (0x022 << MIDR_EL1_PNUM_SHIFT) 1746 #define MIDR_TONGA_FIRESTORM (0x023 << MIDR_EL1_PNUM_SHIFT) 1747 #define MIDR_JADE_CHOP_ICESTORM (0x024 << MIDR_EL1_PNUM_SHIFT) 1748 #define MIDR_JADE_CHOP_FIRESTORM (0x025 << MIDR_EL1_PNUM_SHIFT) 1749 #define MIDR_JADE_DIE_ICESTORM (0x028 << MIDR_EL1_PNUM_SHIFT) 1750 #define MIDR_JADE_DIE_FIRESTORM (0x029 << MIDR_EL1_PNUM_SHIFT) 1751 #endif 1752 1753 1754 1755 /* 1756 * Apple-ISA-Extensions ID Register. 1757 */ 1758 #define AIDR_MUL53 (1 << 0) 1759 #define AIDR_WKDM (1 << 1) 1760 #define AIDR_ARCHRETENTION (1 << 2) 1761 1762 1763 /* 1764 * CoreSight debug registers 1765 */ 1766 #define CORESIGHT_ED 0 1767 #define CORESIGHT_CTI 1 1768 #define CORESIGHT_PMU 2 1769 #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */ 1770 1771 #define CORESIGHT_OFFSET(x) ((x) * 0x10000) 1772 #define CORESIGHT_REGIONS 4 1773 #define CORESIGHT_SIZE 0x1000 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 /* 1785 * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0 1786 * 1787 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 1788 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ 1789 * | rndr | tlb | ts | fhm | dp | sm4 | sm3 | sha3 | rdm | res0 | atomic | crc32 | sha2 | sha1 | aes | res0 | 1790 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ 1791 */ 1792 1793 #define ID_AA64ISAR0_EL1_TS_OFFSET 52 1794 #define ID_AA64ISAR0_EL1_TS_MASK (0xfull << ID_AA64ISAR0_EL1_TS_OFFSET) 1795 #define ID_AA64ISAR0_EL1_TS_FLAGM_EN (1ull << ID_AA64ISAR0_EL1_TS_OFFSET) 1796 #define ID_AA64ISAR0_EL1_TS_FLAGM2_EN (2ull << ID_AA64ISAR0_EL1_TS_OFFSET) 1797 1798 #define ID_AA64ISAR0_EL1_FHM_OFFSET 48 1799 #define ID_AA64ISAR0_EL1_FHM_MASK (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET) 1800 #define ID_AA64ISAR0_EL1_FHM_8_2 (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET) 1801 1802 #define ID_AA64ISAR0_EL1_DP_OFFSET 44 1803 #define ID_AA64ISAR0_EL1_DP_MASK (0xfull << ID_AA64ISAR0_EL1_DP_OFFSET) 1804 #define ID_AA64ISAR0_EL1_DP_EN (1ull << ID_AA64ISAR0_EL1_DP_OFFSET) 1805 1806 #define ID_AA64ISAR0_EL1_SHA3_OFFSET 32 1807 #define ID_AA64ISAR0_EL1_SHA3_MASK (0xfull << ID_AA64ISAR0_EL1_SHA3_OFFSET) 1808 #define ID_AA64ISAR0_EL1_SHA3_EN (1ull << ID_AA64ISAR0_EL1_SHA3_OFFSET) 1809 1810 #define ID_AA64ISAR0_EL1_RDM_OFFSET 28 1811 #define ID_AA64ISAR0_EL1_RDM_MASK (0xfull << ID_AA64ISAR0_EL1_RDM_OFFSET) 1812 #define ID_AA64ISAR0_EL1_RDM_EN (1ull << ID_AA64ISAR0_EL1_RDM_OFFSET) 1813 1814 #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20 1815 #define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) 1816 #define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) 1817 1818 #define ID_AA64ISAR0_EL1_CRC32_OFFSET 16 1819 #define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET) 1820 #define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET) 1821 1822 #define ID_AA64ISAR0_EL1_SHA2_OFFSET 12 1823 #define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 1824 #define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 1825 #define ID_AA64ISAR0_EL1_SHA2_512_EN (2ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 1826 1827 #define ID_AA64ISAR0_EL1_SHA1_OFFSET 8 1828 #define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET) 1829 #define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET) 1830 1831 #define ID_AA64ISAR0_EL1_AES_OFFSET 4 1832 #define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET) 1833 #define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET) 1834 #define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET) 1835 1836 /* 1837 * ID_AA64ISAR1_EL1 - AArch64 Instruction Set Attribute Register 1 1838 * 1839 * 63 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 1840 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ 1841 * | res0 | i8mm | dgh | bf16 | specres | sb | frintts | gpi | gpa | lrcpc | fcma | jscvt | api | apa | dpb | 1842 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ 1843 */ 1844 1845 #define ID_AA64ISAR1_EL1_I8MM_OFFSET 52 1846 #define ID_AA64ISAR1_EL1_I8MM_MASK (0xfull << ID_AA64ISAR1_EL1_I8MM_OFFSET) 1847 #define ID_AA64ISAR1_EL1_I8MM_EN (1ull << ID_AA64ISAR1_EL1_I8MM_OFFSET) 1848 1849 #define ID_AA64ISAR1_EL1_DGH_OFFSET 48 1850 #define ID_AA64ISAR1_EL1_DGH_MASK (0xfull << ID_AA64ISAR1_EL1_DGH_OFFSET) 1851 1852 #define ID_AA64ISAR1_EL1_BF16_OFFSET 44 1853 #define ID_AA64ISAR1_EL1_BF16_MASK (0xfull << ID_AA64ISAR1_EL1_BF16_OFFSET) 1854 #define ID_AA64ISAR1_EL1_BF16_EN (1ull << ID_AA64ISAR1_EL1_BF16_OFFSET) 1855 1856 #define ID_AA64ISAR1_EL1_SPECRES_OFFSET 40 1857 #define ID_AA64ISAR1_EL1_SPECRES_MASK (0xfull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 1858 #define ID_AA64ISAR1_EL1_SPECRES_EN (1ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 1859 1860 #define ID_AA64ISAR1_EL1_SB_OFFSET 36 1861 #define ID_AA64ISAR1_EL1_SB_MASK (0xfull << ID_AA64ISAR1_EL1_SB_OFFSET) 1862 #define ID_AA64ISAR1_EL1_SB_EN (1ull << ID_AA64ISAR1_EL1_SB_OFFSET) 1863 1864 #define ID_AA64ISAR1_EL1_FRINTTS_OFFSET 32 1865 #define ID_AA64ISAR1_EL1_FRINTTS_MASK (0xfull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) 1866 #define ID_AA64ISAR1_EL1_FRINTTS_EN (1ull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) 1867 1868 #define ID_AA64ISAR1_EL1_GPI_OFFSET 28 1869 #define ID_AA64ISAR1_EL1_GPI_MASK (0xfull << ID_AA64ISAR1_EL1_GPI_OFFSET) 1870 #define ID_AA64ISAR1_EL1_GPI_EN (1ull << ID_AA64ISAR1_EL1_GPI_OFFSET) 1871 1872 #define ID_AA64ISAR1_EL1_GPA_OFFSET 24 1873 #define ID_AA64ISAR1_EL1_GPA_MASK (0xfull << ID_AA64ISAR1_EL1_GPA_OFFSET) 1874 1875 #define ID_AA64ISAR1_EL1_LRCPC_OFFSET 20 1876 #define ID_AA64ISAR1_EL1_LRCPC_MASK (0xfull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 1877 #define ID_AA64ISAR1_EL1_LRCPC_EN (1ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 1878 #define ID_AA64ISAR1_EL1_LRCP2C_EN (2ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 1879 1880 #define ID_AA64ISAR1_EL1_FCMA_OFFSET 16 1881 #define ID_AA64ISAR1_EL1_FCMA_MASK (0xfull << ID_AA64ISAR1_EL1_FCMA_OFFSET) 1882 #define ID_AA64ISAR1_EL1_FCMA_EN (1ull << ID_AA64ISAR1_EL1_FCMA_OFFSET) 1883 1884 #define ID_AA64ISAR1_EL1_JSCVT_OFFSET 12 1885 #define ID_AA64ISAR1_EL1_JSCVT_MASK (0xfull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) 1886 #define ID_AA64ISAR1_EL1_JSCVT_EN (1ull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) 1887 1888 #define ID_AA64ISAR1_EL1_API_OFFSET 8 1889 #define ID_AA64ISAR1_EL1_API_MASK (0xfull << ID_AA64ISAR1_EL1_API_OFFSET) 1890 #define ID_AA64ISAR1_EL1_API_PAuth_EN (1ull << ID_AA64ISAR1_EL1_API_OFFSET) 1891 #define ID_AA64ISAR1_EL1_API_PAuth2_EN (3ull << ID_AA64ISAR1_EL1_API_OFFSET) 1892 #define ID_AA64ISAR1_EL1_API_FPAC_EN (4ull << ID_AA64ISAR1_EL1_API_OFFSET) 1893 1894 #define ID_AA64ISAR1_EL1_APA_OFFSET 4 1895 #define ID_AA64ISAR1_EL1_APA_MASK (0xfull << ID_AA64ISAR1_EL1_APA_OFFSET) 1896 1897 #define ID_AA64ISAR1_EL1_DPB_OFFSET 0 1898 #define ID_AA64ISAR1_EL1_DPB_MASK (0xfull << ID_AA64ISAR1_EL1_DPB_OFFSET) 1899 #define ID_AA64ISAR1_EL1_DPB_EN (1ull << ID_AA64ISAR1_EL1_DPB_OFFSET) 1900 #define ID_AA64ISAR1_EL1_DPB2_EN (2ull << ID_AA64ISAR1_EL1_DPB_OFFSET) 1901 1902 /* 1903 * ID_AA64MMFR0_EL1 - AArch64 Memory Model Feature Register 0 1904 * 63 60 59 56 55 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 1905 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ 1906 * | ECV | FGT | RES0 | ExS | TGran4_2 | TGran64_2 | TGran16_2 | TGran4 | TGran64 | TGran16 | BigEndEL0 | SNSMem | BigEnd | ASIDBits | PARange | 1907 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ 1908 */ 1909 1910 #define ID_AA64MMFR0_EL1_ECV_OFFSET 60 1911 #define ID_AA64MMFR0_EL1_ECV_MASK (0xfull << ID_AA64MMFR2_EL1_AT_OFFSET) 1912 #define ID_AA64MMFR0_EL1_ECV_EN (1ull << ID_AA64MMFR2_EL1_AT_OFFSET) 1913 1914 /* 1915 * ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2 1916 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 14 8 7 4 3 0 1917 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ 1918 * | E0PD | EVT | BBM | TTL | RES0 | FWB | IDS | AT | ST | NV | CCIDX | VARANGE | IESB | LSM | UAO | CnP | 1919 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ 1920 */ 1921 1922 #define ID_AA64MMFR2_EL1_AT_OFFSET 32 1923 #define ID_AA64MMFR2_EL1_AT_MASK (0xfull << ID_AA64MMFR2_EL1_AT_OFFSET) 1924 #define ID_AA64MMFR2_EL1_AT_LSE2_EN (1ull << ID_AA64MMFR2_EL1_AT_OFFSET) 1925 1926 /* 1927 * ID_AA64PFR0_EL1 - AArch64 Processor Feature Register 0 1928 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 1929 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ 1930 * | CSV3 | CSV2 | RES0 | DIT | AMU | MPAM | SEL2 | SVE | RAS | GIC | AdvSIMD | FP | EL3 | EL2 | EL1 | EL0 | 1931 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ 1932 */ 1933 1934 #define ID_AA64PFR0_EL1_CSV3_OFFSET 60 1935 #define ID_AA64PFR0_EL1_CSV3_MASK (0xfull << ID_AA64PFR0_EL1_CSV3_OFFSET) 1936 #define ID_AA64PFR0_EL1_CSV3_EN (1ull << ID_AA64PFR0_EL1_CSV3_OFFSET) 1937 1938 #define ID_AA64PFR0_EL1_CSV2_OFFSET 56 1939 #define ID_AA64PFR0_EL1_CSV2_MASK (0xfull << ID_AA64PFR0_EL1_CSV2_OFFSET) 1940 #define ID_AA64PFR0_EL1_CSV2_EN (1ull << ID_AA64PFR0_EL1_CSV2_OFFSET) 1941 1942 #define ID_AA64PFR0_EL1_AdvSIMD_OFFSET 20 1943 #define ID_AA64PFR0_EL1_AdvSIMD_MASK (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 1944 #define ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT (0x0ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 1945 #define ID_AA64PFR0_EL1_AdvSIMD_FP16 (0x1ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 1946 #define ID_AA64PFR0_EL1_AdvSIMD_DIS (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 1947 1948 /* 1949 * ID_AA64PFR1_EL1 - AArch64 Processor Feature Register 1 1950 * 63 20 19 16 15 12 11 8 7 4 3 0 1951 * +----------------------------------+-----------+----------+-------+------+------+ 1952 * | RES0 | MPAM_frac | RAS_frac | MTE | SSBS | BT | 1953 * +----------------------------------+-----------+----------+-------+------+------+ 1954 */ 1955 1956 #define ID_AA64PFR1_EL1_SSBS_OFFSET 4 1957 #define ID_AA64PFR1_EL1_SSBS_MASK (0xfull << ID_AA64PFR1_EL1_SSBS_OFFSET) 1958 #define ID_AA64PFR1_EL1_SSBS_EN (1ull << ID_AA64PFR1_EL1_SSBS_OFFSET) 1959 1960 #define ID_AA64PFR1_EL1_BT_OFFSET 0 1961 #define ID_AA64PFR1_EL1_BT_MASK (0xfull << ID_AA64PFR1_EL1_BT_OFFSET) 1962 #define ID_AA64PFR1_EL1_BT_EN (1ull << ID_AA64PFR1_EL1_BT_OFFSET) 1963 1964 /* 1965 * ID_AA64MMFR1_EL1 - AArch64 Memory Model Feature Register 1 1966 * 1967 * 63 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 1968 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ 1969 * | res0 | nTLBPA | AFP | HCX | ETS | TWED | XNX | SpecSEI | PAN | LO | HPDS | VH | VMIDBits | HAFDBS | 1970 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ 1971 */ 1972 1973 #define ID_AA64MMFR1_EL1_AFP_OFFSET 44 1974 #define ID_AA64MMFR1_EL1_AFP_MASK (0xfull << ID_AA64MMFR1_EL1_AFP_OFFSET) 1975 #define ID_AA64MMFR1_EL1_AFP_EN (1ull << ID_AA64MMFR1_EL1_AFP_OFFSET) 1976 1977 1978 1979 #define APSTATE_G_SHIFT (0) 1980 #define APSTATE_P_SHIFT (1) 1981 #define APSTATE_A_SHIFT (2) 1982 #define APSTATE_AP_MASK ((1ULL << APSTATE_A_SHIFT) | (1ULL << APSTATE_P_SHIFT)) 1983 1984 1985 #define ACTLR_EL1_EnTSO (1ULL << 1) 1986 #define ACTLR_EL1_EnAPFLG (1ULL << 4) 1987 #define ACTLR_EL1_EnAFP (1ULL << 5) 1988 #define ACTLR_EL1_EnPRSV (1ULL << 6) 1989 1990 1991 #define ACTLR_EL1_DisHWP_OFFSET 3 1992 #define ACTLR_EL1_DisHWP_MASK (1ULL << ACTLR_EL1_DisHWP_OFFSET) 1993 #define ACTLR_EL1_DisHWP ACTLR_EL1_DisHWP_MASK 1994 1995 1996 1997 1998 #if defined(HAS_APPLE_PAC) 1999 // The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code 2000 #define PAC_DISCRIMINATOR_RECOVER 0x1e02 2001 #endif 2002 2003 2004 #define CTR_EL0_L1Ip_OFFSET 14 2005 #define CTR_EL0_L1Ip_VIPT (2ULL << CTR_EL0_L1Ip_OFFSET) 2006 #define CTR_EL0_L1Ip_PIPT (3ULL << CTR_EL0_L1Ip_OFFSET) 2007 #define CTR_EL0_L1Ip_MASK (3ULL << CTR_EL0_L1Ip_OFFSET) 2008 2009 2010 #ifdef __ASSEMBLER__ 2011 2012 /* 2013 * Conditionally write to system/special-purpose register. 2014 * The register is written to only when the first two arguments 2015 * do not match. If they do match, the macro jumps to a 2016 * caller-provided label. 2017 * The _ISB variant also conditionally issues an ISB after the MSR. 2018 * 2019 * $0 - System/special-purpose register to modify 2020 * $1 - Register containing current FPCR value 2021 * $2 - Register containing expected value 2022 * $3 - Label to jump to when register is already set to expected value 2023 */ 2024 .macro CMSR 2025 cmp $1, $2 2026 2027 /* Skip expensive MSR if not required */ 2028 b.eq $3f 2029 msr $0, $2 2030 .endmacro 2031 2032 .macro CMSR_ISB 2033 CMSR $0, $1, $2, $3 2034 isb sy 2035 .endmacro 2036 2037 /* 2038 * Modify FPCR only if it does not contain the XNU default value. 2039 * $0 - Register containing current FPCR value 2040 * $1 - Scratch register 2041 * $2 - Label to jump to when FPCR is already set to default value 2042 */ 2043 .macro SANITIZE_FPCR 2044 mov $1, #FPCR_DEFAULT 2045 CMSR FPCR, $0, $1, $2 2046 .endmacro 2047 2048 /* 2049 * Family of macros that can be used to protect code sections such that they 2050 * are only executed on a particular SoC/Revision/CPU, and skipped otherwise. 2051 * All macros will forward-jump to 1f when the condition is not matched. 2052 * This label may be defined manually, or implicitly through the use of 2053 * the EXEC_END macro. 2054 * For cores, XX can be: EQ (equal), ALL (don't care). 2055 * For revisions, XX can be: EQ (equal), LO (lower than), HS (higher or same), ALL (don't care). 2056 */ 2057 2058 /* 2059 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX 2060 * $1 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2061 * $2 - GPR containing MIDR_EL1 value 2062 * $3 - Scratch register 2063 */ 2064 .macro EXEC_COREEQ_REVEQ 2065 and $3, $2, #MIDR_EL1_PNUM_MASK 2066 cmp $3, $0 2067 b.ne 1f 2068 2069 mov $3, $2 2070 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2071 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2072 cmp $3, $1 2073 b.ne 1f 2074 .endmacro 2075 2076 .macro EXEC_COREEQ_REVLO 2077 and $3, $2, #MIDR_EL1_PNUM_MASK 2078 cmp $3, $0 2079 b.ne 1f 2080 2081 mov $3, $2 2082 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2083 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2084 cmp $3, $1 2085 b.pl 1f 2086 .endmacro 2087 2088 .macro EXEC_COREEQ_REVHS 2089 and $3, $2, #MIDR_EL1_PNUM_MASK 2090 cmp $3, $0 2091 b.ne 1f 2092 2093 mov $3, $2 2094 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2095 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2096 cmp $3, $1 2097 b.mi 1f 2098 .endmacro 2099 2100 /* 2101 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2102 * $1 - GPR containing MIDR_EL1 value 2103 * $2 - Scratch register 2104 */ 2105 .macro EXEC_COREALL_REVEQ 2106 mov $2, $1 2107 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2108 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2109 cmp $2, $0 2110 b.ne 1f 2111 .endmacro 2112 2113 .macro EXEC_COREALL_REVLO 2114 mov $2, $1 2115 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2116 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2117 cmp $2, $0 2118 b.pl 1f 2119 .endmacro 2120 2121 .macro EXEC_COREALL_REVHS 2122 mov $2, $1 2123 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2124 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2125 cmp $2, $0 2126 b.mi 1f 2127 .endmacro 2128 2129 .macro CMP_FOREACH reg, cc, label, car, cdr:vararg 2130 cmp \reg, \car 2131 b.\cc \label 2132 .ifnb \cdr 2133 CMP_FOREACH \reg, \cc, \label, \cdr 2134 .endif 2135 .endm 2136 2137 .macro EXEC_COREIN_REVALL midr_el1, scratch, midr_list:vararg 2138 and \scratch, \midr_el1, #MIDR_EL1_PNUM_MASK 2139 CMP_FOREACH \scratch, eq, Lmatch\@, \midr_list 2140 b 1f 2141 Lmatch\@: 2142 .endm 2143 2144 /* 2145 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX 2146 * $1 - GPR containing MIDR_EL1 value 2147 * $2 - Scratch register 2148 */ 2149 .macro EXEC_COREEQ_REVALL 2150 and $2, $1, #MIDR_EL1_PNUM_MASK 2151 cmp $2, $0 2152 b.ne 1f 2153 .endmacro 2154 2155 /* 2156 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2157 * $1 - GPR containing MIDR_EL1 value 2158 * $2 - Scratch register 2159 */ 2160 .macro EXEC_PCORE_REVEQ 2161 mrs $2, MPIDR_EL1 2162 and $2, $2, #(MPIDR_PNE) 2163 cmp $2, xzr 2164 b.eq 1f 2165 2166 mov $2, $1 2167 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2168 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2169 cmp $2, $0 2170 b.ne 1f 2171 .endmacro 2172 2173 .macro EXEC_PCORE_REVLO 2174 mrs $2, MPIDR_EL1 2175 and $2, $2, #(MPIDR_PNE) 2176 cmp $2, xzr 2177 b.eq 1f 2178 2179 mov $2, $1 2180 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2181 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2182 cmp $2, $0 2183 b.pl 1f 2184 .endmacro 2185 2186 .macro EXEC_PCORE_REVHS 2187 mrs $2, MPIDR_EL1 2188 and $2, $2, #(MPIDR_PNE) 2189 cmp $2, xzr 2190 b.eq 1f 2191 2192 mov $2, $1 2193 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2194 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2195 cmp $2, $0 2196 b.mi 1f 2197 .endmacro 2198 2199 .macro EXEC_ECORE_REVEQ 2200 mrs $2, MPIDR_EL1 2201 and $2, $2, #(MPIDR_PNE) 2202 cmp $2, xzr 2203 b.ne 1f 2204 2205 mov $2, $1 2206 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2207 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2208 cmp $2, $0 2209 b.ne 1f 2210 .endmacro 2211 2212 .macro EXEC_ECORE_REVLO 2213 mrs $2, MPIDR_EL1 2214 and $2, $2, #(MPIDR_PNE) 2215 cmp $2, xzr 2216 b.ne 1f 2217 2218 mov $2, $1 2219 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2220 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2221 cmp $2, $0 2222 b.pl 1f 2223 .endmacro 2224 2225 .macro EXEC_ECORE_REVHS 2226 mrs $2, MPIDR_EL1 2227 and $2, $2, #(MPIDR_PNE) 2228 cmp $2, xzr 2229 b.ne 1f 2230 2231 mov $2, $1 2232 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2233 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2234 cmp $2, $0 2235 b.mi 1f 2236 .endmacro 2237 2238 /* 2239 * $0 - GPR containing MIDR_EL1 value 2240 * $1 - Scratch register 2241 */ 2242 .macro EXEC_PCORE_REVALL 2243 mrs $1, MPIDR_EL1 2244 and $1, $1, #(MPIDR_PNE) 2245 cmp $1, xzr 2246 b.eq 1f 2247 .endmacro 2248 2249 .macro EXEC_ECORE_REVALL 2250 mrs $1, MPIDR_EL1 2251 and $1, $1, #(MPIDR_PNE) 2252 cmp $1, xzr 2253 b.ne 1f 2254 .endmacro 2255 2256 2257 2258 /* 2259 * Macro that defines the label that all EXEC_COREXX_REVXX macros jump to. 2260 */ 2261 .macro EXEC_END 2262 1: 2263 .endmacro 2264 2265 /* 2266 * Wedges CPUs with a specified core that are below a specified revision. This 2267 * macro is intended for CPUs that have been deprecated in iBoot and may have 2268 * incorrect behavior if they continue running xnu. 2269 */ 2270 .macro DEPRECATE_COREEQ_REVLO core, rev, midr_el1, scratch 2271 EXEC_COREEQ_REVLO \core, \rev, \midr_el1, \scratch 2272 /* BEGIN IGNORE CODESTYLE */ 2273 b . 2274 /* END IGNORE CODESTYLE */ 2275 EXEC_END 2276 .endmacro 2277 2278 /* 2279 * Sets bits in an SPR register. 2280 * arg0: Name of the register to be accessed. 2281 * arg1: Mask of bits to be set. 2282 * arg2: Scratch register 2283 */ 2284 .macro HID_SET_BITS 2285 mrs $2, $0 2286 orr $2, $2, $1 2287 msr $0, $2 2288 .endmacro 2289 2290 /* 2291 * Clears bits in an SPR register. 2292 * arg0: Name of the register to be accessed. 2293 * arg1: Mask of bits to be cleared. 2294 * arg2: Scratch register 2295 */ 2296 .macro HID_CLEAR_BITS 2297 mrs $2, $0 2298 bic $2, $2, $1 2299 msr $0, $2 2300 .endmacro 2301 2302 /* 2303 * Clears bits in an SPR register. 2304 * arg0: Name of the register to be accessed. 2305 * arg1: Mask of bits to be cleared. 2306 * arg2: Value to insert 2307 * arg3: Scratch register 2308 */ 2309 .macro HID_INSERT_BITS 2310 mrs $3, $0 2311 bic $3, $3, $1 2312 orr $3, $3, $2 2313 msr $0, $3 2314 .endmacro 2315 2316 /* 2317 * Replaces the value of a field in an implementation-defined system register. 2318 * sreg: system register name 2319 * field: field name within the sysreg, where the assembler symbols 2320 * ARM64_REG_<field>_{shift,width} specify the bounds of the field 2321 * (note that preprocessor macros will not work here) 2322 * value: the value to insert 2323 * scr{1,2}: scratch regs 2324 */ 2325 .macro HID_WRITE_FIELD sreg, field, val, scr1, scr2 2326 mrs \scr1, \sreg 2327 mov \scr2, \val 2328 bfi \scr1, \scr2, ARM64_REG_\sreg\()_\field\()_shift, ARM64_REG_\sreg\()_\field\()_width 2329 msr \sreg, \scr1 2330 .endmacro 2331 2332 /* 2333 * Macro intended to be used as a replacement for ERET. 2334 * It prevents speculation past ERET instructions by padding 2335 * up to the decoder width. 2336 */ 2337 .macro ERET_CONTEXT_SYNCHRONIZING 2338 eret 2339 #if __ARM_SB_AVAILABLE__ 2340 sb // Technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures 2341 #else /* __ARM_SB_AVAILABLE__ */ 2342 isb // ISB technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures 2343 nop // Sequence of six NOPs to pad out and terminate instruction decode group */ 2344 nop 2345 nop 2346 nop 2347 nop 2348 nop 2349 #endif /* !__ARM_SB_AVAILABLE__ */ 2350 .endmacro 2351 2352 #endif /* __ASSEMBLER__ */ 2353 2354 #define MSR(reg, src) __asm__ volatile ("msr " reg ", %0" :: "r" (src)) 2355 #define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest)) 2356 2357 #if XNU_MONITOR 2358 #define __ARM_PTE_PHYSMAP__ 1 2359 #define PPL_STATE_KERNEL 0 2360 #define PPL_STATE_DISPATCH 1 2361 #define PPL_STATE_PANIC 2 2362 #define PPL_STATE_EXCEPTION 3 2363 #endif 2364 2365 2366 #endif /* _ARM64_PROC_REG_H_ */ 2367