1 /* 2 * Copyright (c) 2007-2023 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 /* CMU_ENDHIST */ 32 /* 33 * Mach Operating System 34 * Copyright (c) 1991,1990 Carnegie Mellon University 35 * All Rights Reserved. 36 * 37 * Permission to use, copy, modify and distribute this software and its 38 * documentation is hereby granted, provided that both the copyright 39 * notice and this permission notice appear in all copies of the 40 * software, derivative works or modified versions, and any portions 41 * thereof, and that both notices appear in supporting documentation. 42 * 43 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 44 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 45 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 46 * 47 * Carnegie Mellon requests users of this software to return to 48 * 49 * Software Distribution Coordinator or [email protected] 50 * School of Computer Science 51 * Carnegie Mellon University 52 * Pittsburgh PA 15213-3890 53 * 54 * any improvements or extensions that they make and grant Carnegie Mellon 55 * the rights to redistribute these changes. 56 */ 57 58 /* 59 * Processor registers for ARM/ARM64 60 */ 61 #ifndef _ARM64_PROC_REG_H_ 62 #define _ARM64_PROC_REG_H_ 63 64 #if defined (__arm64__) 65 #include <pexpert/arm64/board_config.h> 66 #elif defined (__arm__) 67 #include <pexpert/arm/board_config.h> 68 #endif 69 70 /* 71 * Processor registers for ARM 72 */ 73 #if __ARM_42BIT_PA_SPACE__ 74 /* For now, force the issue! */ 75 /* We need more VA space for the identity map to bootstrap the MMU */ 76 #undef __ARM64_PMAP_SUBPAGE_L1__ 77 #endif /* __ARM_42BIT_PA_SPACE__ */ 78 79 /* For arm platforms, create one pset per cluster */ 80 #define MAX_PSETS MAX_CPU_CLUSTERS 81 82 83 /* Thread groups are enabled on all ARM platforms (irrespective of scheduler) */ 84 #define CONFIG_THREAD_GROUPS 1 85 86 #ifdef XNU_KERNEL_PRIVATE 87 88 #if __ARM_VFP__ 89 #define ARM_VFP_DEBUG 0 90 #endif /* __ARM_VFP__ */ 91 92 #endif /* XNU_KERNEL_PRIVATE */ 93 94 /* 95 * FSR registers 96 * 97 * CPSR: Current Program Status Register 98 * SPSR: Saved Program Status Registers 99 * 100 * 31 30 29 28 27 24 19 16 9 8 7 6 5 4 0 101 * +-----------------------------------------------------------+ 102 * | N| Z| C| V| Q|...| J|...|GE[3:0]|...| E| A| I| F| T| MODE | 103 * +-----------------------------------------------------------+ 104 */ 105 106 /* 107 * Flags 108 */ 109 #define PSR_NF 0x80000000 /* Negative/Less than */ 110 #define PSR_ZF 0x40000000 /* Zero */ 111 #define PSR_CF 0x20000000 /* Carry/Borrow/Extend */ 112 #define PSR_VF 0x10000000 /* Overflow */ 113 114 /* 115 * Modified execution mode flags 116 */ 117 #define PSR_TF 0x00000020 /* thumb flag (BX ARMv4T) */ 118 119 /* 120 * CPU mode 121 */ 122 #define PSR_USER_MODE 0x00000010 /* User mode */ 123 124 #define PSR_MODE_MASK 0x0000001F 125 #define PSR_IS_KERNEL(psr) (((psr) & PSR_MODE_MASK) != PSR_USER_MODE) 126 #define PSR_IS_USER(psr) (((psr) & PSR_MODE_MASK) == PSR_USER_MODE) 127 128 #define PSR_USERDFLT PSR_USER_MODE 129 130 /* 131 * Cache configuration 132 */ 133 134 #if defined (APPLETYPHOON) 135 136 /* I-Cache */ 137 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 138 139 /* D-Cache */ 140 #define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */ 141 142 #elif defined (APPLETWISTER) 143 144 /* I-Cache */ 145 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 146 147 /* D-Cache */ 148 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 149 150 #elif defined (APPLEHURRICANE) 151 152 /* I-Cache */ 153 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 154 155 /* D-Cache */ 156 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 157 158 #elif defined (APPLEMONSOON) 159 160 /* I-Cache, 96KB for Monsoon, 48KB for Mistral, 6-way. */ 161 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 162 163 /* D-Cache, 64KB for Monsoon, 32KB for Mistral, 4-way. */ 164 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 165 166 #elif defined (APPLEVORTEX) 167 168 /* I-Cache, 128KB 8-way for Vortex, 48KB 6-way for Tempest. */ 169 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 170 171 /* D-Cache, 128KB 8-way for Vortex, 32KB 4-way for Tempest. */ 172 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 173 174 #elif defined (APPLELIGHTNING) 175 176 /* I-Cache, 192KB for Lightning, 96KB for Thunder, 6-way. */ 177 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 178 179 /* D-Cache, 128KB for Lightning, 8-way. 48KB for Thunder, 6-way. */ 180 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 181 182 #elif defined (APPLEFIRESTORM) 183 184 /* I-Cache, 256KB for Firestorm, 128KB for Icestorm, 6-way. */ 185 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 186 187 /* D-Cache, 160KB for Firestorm, 8-way. 64KB for Icestorm, 6-way. */ 188 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 189 190 #elif defined (APPLEAVALANCHE) 191 192 /* I-Cache, 192KB for Avalanche, 128KB for Blizzard, 6-way. */ 193 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 194 195 /* D-Cache, 128KB for Avalanche, 8-way. 64KB for Blizzard, 8-way. */ 196 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 197 198 #elif defined (BCM2837) /* Raspberry Pi 3 */ 199 200 /* I-Cache. We don't have detailed spec so we just follow the ARM technical reference. */ 201 #define MMU_I_CLINE 6 202 203 /* D-Cache. */ 204 #define MMU_CLINE 6 205 206 #elif defined (VMAPPLE) 207 208 /* I-Cache. */ 209 #define MMU_I_CLINE 6 210 211 /* D-Cache. */ 212 #define MMU_CLINE 6 213 214 #else 215 #error processor not supported 216 #endif 217 218 #define MAX_L2_CLINE_BYTES (1 << MAX_L2_CLINE) 219 220 /* 221 * Format of the Debug & Watchpoint Breakpoint Value and Control Registers 222 */ 223 #define ARM_DBG_VR_ADDRESS_MASK 0xFFFFFFFC /* BVR & WVR */ 224 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */ 225 226 #define ARM_DBG_CR_ADDRESS_MASK_MASK 0x1F000000 /* BCR & WCR */ 227 #define ARM_DBGBCR_MATCH_MASK (1 << 22) /* BCR only */ 228 #define ARM_DBGBCR_TYPE_MASK (1 << 21) /* BCR only */ 229 #define ARM_DBGBCR_TYPE_IVA (0 << 21) 230 #define ARM_DBG_CR_LINKED_MASK (1 << 20) /* BCR & WCR */ 231 #define ARM_DBG_CR_LINKED_UNLINKED (0 << 20) 232 #define ARM_DBG_CR_SECURITY_STATE_BOTH (0 << 14) 233 #define ARM_DBG_CR_HIGHER_MODE_ENABLE (1 << 13) 234 #define ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK 0x00001FE0 /* WCR only */ 235 #define ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK 0x000001E0 /* BCR & WCR */ 236 #define ARM_DBGWCR_ACCESS_CONTROL_MASK (3 << 3) /* WCR only */ 237 #define ARM_DBG_CR_MODE_CONTROL_PRIVILEGED (1 << 1) /* BCR & WCR */ 238 #define ARM_DBG_CR_MODE_CONTROL_USER (2 << 1) /* BCR & WCR */ 239 #define ARM_DBG_CR_ENABLE_MASK (1 << 0) /* BCR & WCR */ 240 #define ARM_DBG_CR_ENABLE_ENABLE (1 << 0) 241 242 /* 243 * Format of the OS Lock Access (DBGOSLAR) and Lock Access Registers (DBGLAR) 244 */ 245 #define ARM_DBG_LOCK_ACCESS_KEY 0xC5ACCE55 246 247 /* ARM Debug registers of interest */ 248 #define ARM_DEBUG_OFFSET_DBGPRCR (0x310) 249 #define ARM_DEBUG_OFFSET_DBGLAR (0xFB0) 250 251 /* 252 * Main ID Register (MIDR) 253 * 254 * 31 24 23 20 19 16 15 4 3 0 255 * +-----+-----+------+------+-----+ 256 * | IMP | VAR | ARCH | PNUM | REV | 257 * +-----+-----+------+------+-----+ 258 * 259 * where: 260 * IMP: Implementor code 261 * VAR: Variant number 262 * ARCH: Architecture code 263 * PNUM: Primary part number 264 * REV: Minor revision number 265 */ 266 #define MIDR_REV_SHIFT 0 267 #define MIDR_REV_MASK (0xf << MIDR_REV_SHIFT) 268 #define MIDR_VAR_SHIFT 20 269 #define MIDR_VAR_MASK (0xf << MIDR_VAR_SHIFT) 270 271 272 #if __ARM_KERNEL_PROTECT__ 273 /* 274 * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential 275 * architectural or microarchitectural vulnerabilities that could allow cores to 276 * read/access EL1-only mappings while in EL0 mode. This is achieved by 277 * removing as many mappings as possible when the core transitions to EL0 mode 278 * from EL1 mode, and restoring those mappings when the core transitions to EL1 279 * mode from EL0 mode. 280 * 281 * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is 282 * used to map and unmap the ordinary kernel mappings, by contracting and 283 * expanding translation zone size for TTBR1 when exiting and entering EL1, 284 * respectively: 285 * 286 * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is 287 * is in EL0. 288 * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in 289 * EL1. 290 * 291 * T1SZ_USER: T1SZ_BOOT + 1 292 * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT) 293 * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER) 294 * TTBR1_EL1_MAX: (2^64) - 1 295 * 296 * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the 297 * the following TTBR1 layout: 298 * 299 * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX 300 * +---------------------------------------------------------+ 301 * | Kernel EL0 Mappings | Kernel EL1 Mappings | 302 * +---------------------------------------------------------+ 303 * 304 * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout: 305 * 306 * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX 307 * +---------------------------------------------------------+ 308 * | Kernel EL0 Mappings | 309 * +---------------------------------------------------------+ 310 * 311 * With the current implementation, both the EL0 and EL1 mappings for the kernel 312 * use otherwise empty translation tables for mapping the exception vectors (so 313 * that we do not need to TLB flush the exception vector address when switching 314 * between EL0 and EL1). The rationale here is that the TLBI would require a 315 * DSB, and DSBs can be extremely expensive. 316 * 317 * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1 318 * ASID. The core switches between ASIDs on EL transitions, so that the TLB 319 * does not need to be fully invalidated on an EL transition. 320 * 321 * Most kernel mappings will be marked non-global in this configuration, as 322 * global mappings would be visible to userspace unless we invalidate them on 323 * eret. 324 */ 325 #if XNU_MONITOR 326 /* 327 * Please note that because we indirect through the thread register in order to 328 * locate the kernel, and because we unmap most of the kernel, the security 329 * model of the PPL is undermined by __ARM_KERNEL_PROTECT__, as we rely on 330 * kernel controlled data to direct codeflow in the exception vectors. 331 * 332 * If we want to ship XNU_MONITOR paired with __ARM_KERNEL_PROTECT__, we will 333 * need to find a performant solution to this problem. 334 */ 335 #endif 336 #endif /* __ARM_KERNEL_PROTECT */ 337 338 #if ARM_PARAMETERIZED_PMAP 339 /* 340 * ARM_PARAMETERIZED_PMAP configures the kernel to get the characteristics of 341 * the page tables (number of levels, size of the root allocation) from the 342 * pmap data structure, rather than treating them as compile-time constants. 343 * This allows the pmap code to dynamically adjust how it deals with page 344 * tables. 345 */ 346 #endif /* ARM_PARAMETERIZED_PMAP */ 347 348 #if __ARM_MIXED_PAGE_SIZE__ 349 /* 350 * __ARM_MIXED_PAGE_SIZE__ configures the kernel to support page tables that do 351 * not use the kernel page size. This is primarily meant to support running 352 * 4KB page processes on a 16KB page kernel. 353 * 354 * This only covers support in the pmap/machine dependent layers. Any support 355 * elsewhere in the kernel must be managed separately. 356 */ 357 #if !ARM_PARAMETERIZED_PMAP 358 /* 359 * Page tables that use non-kernel page sizes require us to reprogram TCR based 360 * on the page tables we are switching to. This means that the parameterized 361 * pmap support is required. 362 */ 363 #error __ARM_MIXED_PAGE_SIZE__ requires ARM_PARAMETERIZED_PMAP 364 #endif /* !ARM_PARAMETERIZED_PMAP */ 365 #if __ARM_KERNEL_PROTECT__ 366 /* 367 * Because switching the page size requires updating TCR based on the pmap, and 368 * __ARM_KERNEL_PROTECT__ relies on TCR being programmed with constants, XNU 369 * does not currently support support configurations that use both 370 * __ARM_KERNEL_PROTECT__ and __ARM_MIXED_PAGE_SIZE__. 371 */ 372 #error __ARM_MIXED_PAGE_SIZE__ and __ARM_KERNEL_PROTECT__ are mutually exclusive 373 #endif /* __ARM_KERNEL_PROTECT__ */ 374 #endif /* __ARM_MIXED_PAGE_SIZE__ */ 375 376 /* 377 * 64-bit Program Status Register (PSR64) 378 * 379 * 31 27 23 22 21 20 19 10 9 5 4 0 380 * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+ 381 * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M | 382 * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+ 383 * 384 * where: 385 * NZCV: Comparison flags 386 * PAN: Privileged Access Never 387 * SS: Single step 388 * IL: Illegal state 389 * DAIF: Interrupt masks 390 * M: Mode field 391 */ 392 393 #define PSR64_NZCV_SHIFT 28 394 #define PSR64_NZCV_MASK (0xF << PSR64_NZCV_SHIFT) 395 396 #define PSR64_N_SHIFT 31 397 #define PSR64_N (1 << PSR64_N_SHIFT) 398 399 #define PSR64_Z_SHIFT 30 400 #define PSR64_Z (1 << PSR64_Z_SHIFT) 401 402 #define PSR64_C_SHIFT 29 403 #define PSR64_C (1 << PSR64_C_SHIFT) 404 405 #define PSR64_V_SHIFT 28 406 #define PSR64_V (1 << PSR64_V_SHIFT) 407 408 #define PSR64_TCO_SHIFT 25 409 #define PSR64_TCO (1 << PSR64_TCO_SHIFT) 410 411 #define PSR64_DIT_SHIFT 24 412 #define PSR64_DIT (1 << PSR64_DIT_SHIFT) 413 414 #define PSR64_UAO_SHIFT 23 415 #define PSR64_UAO (1 << PSR64_UAO_SHIFT) 416 417 #define PSR64_PAN_SHIFT 22 418 #define PSR64_PAN (1 << PSR64_PAN_SHIFT) 419 420 #define PSR64_SS_SHIFT 21 421 #define PSR64_SS (1 << PSR64_SS_SHIFT) 422 423 #define PSR64_IL_SHIFT 20 424 #define PSR64_IL (1 << PSR64_IL_SHIFT) 425 426 /* 427 * SSBS is bit 12 for A64 SPSR and bit 23 for A32 SPSR 428 * I do not want to talk about it! 429 */ 430 #define PSR64_SSBS_SHIFT_32 23 431 #define PSR64_SSBS_SHIFT_64 12 432 #define PSR64_SSBS_32 (1 << PSR64_SSBS_SHIFT_32) 433 #define PSR64_SSBS_64 (1 << PSR64_SSBS_SHIFT_64) 434 435 /* 436 * msr DAIF, Xn and mrs Xn, DAIF transfer into 437 * and out of bits 9:6 438 */ 439 #define DAIF_DEBUG_SHIFT 9 440 #define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT) 441 442 #define DAIF_ASYNC_SHIFT 8 443 #define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT) 444 445 #define DAIF_IRQF_SHIFT 7 446 #define DAIF_IRQF (1 << DAIF_IRQF_SHIFT) 447 448 #define DAIF_FIQF_SHIFT 6 449 #define DAIF_FIQF (1 << DAIF_FIQF_SHIFT) 450 451 #define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) 452 #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) 453 454 #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF)) 455 456 #if HAS_ARM_FEAT_SSBS2 457 #define PSR64_SSBS_U32_DEFAULT PSR64_SSBS_32 458 #define PSR64_SSBS_U64_DEFAULT PSR64_SSBS_64 459 #define PSR64_SSBS_KRN_DEFAULT PSR64_SSBS_64 460 #else 461 #define PSR64_SSBS_U32_DEFAULT (0) 462 #define PSR64_SSBS_U64_DEFAULT (0) 463 #define PSR64_SSBS_KRN_DEFAULT (0) 464 #endif 465 466 /* 467 * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer 468 * from bits 3:0. 469 */ 470 #define DAIFSC_DEBUGF (1 << 3) 471 #define DAIFSC_ASYNCF (1 << 2) 472 #define DAIFSC_IRQF (1 << 1) 473 #define DAIFSC_FIQF (1 << 0) 474 #define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) 475 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) 476 #define DAIFSC_NOASYNC (DAIFSC_DEBUGF | DAIFSC_IRQF | DAIFSC_FIQF) 477 478 /* 479 * ARM64_TODO: unify with ARM? 480 */ 481 #define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */ 482 483 #define PSR64_MODE_MASK 0x1F 484 485 #define PSR64_USER_MASK PSR64_NZCV_MASK 486 487 #define PSR64_MODE_USER32_THUMB 0x20 488 489 #define PSR64_MODE_RW_SHIFT 4 490 #define PSR64_MODE_RW_64 0 491 #define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT) 492 493 #define PSR64_MODE_EL_SHIFT 2 494 #define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT) 495 #define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT) 496 #define PSR64_MODE_EL2 (0x2 << PSR64_MODE_EL_SHIFT) 497 #define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT) 498 #define PSR64_MODE_EL0 0 499 500 #define PSR64_MODE_EL_KERNEL (PSR64_MODE_EL1) 501 502 #define PSR64_MODE_SPX 0x1 503 #define PSR64_MODE_SP0 0 504 505 #define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U32_DEFAULT) 506 #define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U64_DEFAULT) 507 #define PSR64_KERNEL_STANDARD (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0 | PSR64_SSBS_KRN_DEFAULT) 508 #if __ARM_PAN_AVAILABLE__ 509 #define PSR64_KERNEL_DEFAULT (PSR64_KERNEL_STANDARD | PSR64_PAN) 510 #else 511 #define PSR64_KERNEL_DEFAULT PSR64_KERNEL_STANDARD 512 #endif 513 514 #define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0) 515 #define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0) 516 517 #define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32)) 518 #define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32)) 519 520 521 522 /* 523 * System Control Register (SCTLR) 524 */ 525 526 #define SCTLR_OTHER (0) 527 528 #define SCTLR_DSSBS (1ULL << 44) 529 530 531 #define SCTLR_EXTRA (0) 532 533 534 #define SCTLR_RESERVED ((3ULL << 28) | (1ULL << 20)) 535 #if defined(HAS_APPLE_PAC) 536 537 // 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled 538 #define SCTLR_PACIA_ENABLED_SHIFT 31 539 #define SCTLR_PACIA_ENABLED (1ULL << SCTLR_PACIA_ENABLED_SHIFT) 540 // 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled 541 #define SCTLR_PACIB_ENABLED (1ULL << 30) 542 // 29:28 RES1 11 543 // 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled 544 #define SCTLR_PACDA_ENABLED (1ULL << 27) 545 // 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled 546 #define SCTLR_PACDB_ENABLED (1ULL << 13) 547 548 #define SCTLR_JOP_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED) 549 #endif /* defined(HAS_APPLE_PAC) */ 550 551 // 26 UCI User Cache Instructions 552 #define SCTLR_UCI_ENABLED (1ULL << 26) 553 554 // 25 EE Exception Endianness 555 #define SCTLR_EE_BIG_ENDIAN (1ULL << 25) 556 557 // 24 E0E EL0 Endianness 558 #define SCTLR_E0E_BIG_ENDIAN (1ULL << 24) 559 560 // 23 SPAN Set PAN 561 #define SCTLR_PAN_UNCHANGED (1ULL << 23) 562 563 // 22 EIS Taking an exception is a context synchronization event 564 #define SCTLR_EIS (1ULL << 22) 565 566 // 21 RES0 0 567 // 20 RES1 1 568 569 // 19 WXN Writeable implies eXecute Never 570 #define SCTLR_WXN_ENABLED (1ULL << 19) 571 572 // 18 nTWE Not trap WFE from EL0 573 #define SCTLR_nTWE_WFE_ENABLED (1ULL << 18) 574 575 // 17 RES0 0 576 577 // 16 nTWI Not trap WFI from EL0 578 #define SCTRL_nTWI_WFI_ENABLED (1ULL << 16) 579 580 // 15 UCT User Cache Type register (CTR_EL0) 581 #define SCTLR_UCT_ENABLED (1ULL << 15) 582 583 // 14 DZE User Data Cache Zero (DC ZVA) 584 #define SCTLR_DZE_ENABLED (1ULL << 14) 585 586 // 12 I Instruction cache enable 587 #define SCTLR_I_ENABLED (1ULL << 12) 588 589 // 11 EOS Exception return is a context synchronization event 590 #define SCTLR_EOS (1ULL << 11) 591 592 // 10 EnRCTX EL0 Access to FEAT_SPECRES speculation restriction instructions 593 #define SCTLR_EnRCTX (1ULL << 10) 594 595 // 9 UMA User Mask Access 596 #define SCTLR_UMA_ENABLED (1ULL << 9) 597 598 // 8 SED SETEND Disable 599 #define SCTLR_SED_DISABLED (1ULL << 8) 600 601 // 7 ITD IT Disable 602 #define SCTLR_ITD_DISABLED (1ULL << 7) 603 604 // 6 RES0 0 605 606 // 5 CP15BEN CP15 Barrier ENable 607 #define SCTLR_CP15BEN_ENABLED (1ULL << 5) 608 609 // 4 SA0 Stack Alignment check for EL0 610 #define SCTLR_SA0_ENABLED (1ULL << 4) 611 612 // 3 SA Stack Alignment check 613 #define SCTLR_SA_ENABLED (1ULL << 3) 614 615 // 2 C Cache enable 616 #define SCTLR_C_ENABLED (1ULL << 2) 617 618 // 1 A Alignment check 619 #define SCTLR_A_ENABLED (1ULL << 1) 620 621 // 0 M MMU enable 622 #define SCTLR_M_ENABLED (1ULL << 0) 623 624 #if __ARM_ARCH_8_5__ 625 #define SCTLR_CSEH_DEFAULT (0) 626 #define SCTLR_DSSBS_DEFAULT SCTLR_DSSBS 627 #else 628 #define SCTLR_CSEH_DEFAULT (SCTLR_EIS | SCTLR_EOS) 629 #define SCTLR_DSSBS_DEFAULT (0) 630 #endif 631 632 #if HAS_APPLE_PAC 633 #define SCTLR_ROP_KEYS_DEFAULT SCTLR_PACIB_ENABLED /* IB is ROP */ 634 #else /* !HAS_APPLE_PAC */ 635 #define SCTLR_ROP_KEYS_DEFAULT 0 636 #endif /* HAS_APPLE_PAC */ 637 638 #if HAS_APPLE_PAC 639 #define SCTLR_JOP_KEYS_DEFAULT SCTLR_JOP_KEYS_ENABLED 640 #else /* !HAS_APPLE_PAC */ 641 #define SCTLR_JOP_KEYS_DEFAULT 0 642 #endif 643 644 #define SCTLR_EL1_DEFAULT \ 645 (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \ 646 SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | \ 647 SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED | \ 648 SCTLR_CSEH_DEFAULT | SCTLR_DSSBS_DEFAULT | SCTLR_ROP_KEYS_DEFAULT | \ 649 SCTLR_JOP_KEYS_DEFAULT | SCTLR_OTHER | SCTLR_EXTRA ) 650 651 /* 652 * Coprocessor Access Control Register (CPACR) 653 * 654 * 31 28 27 22 21 20 19 0 655 * +---+---+------+------+--------------------+ 656 * |000|TTA|000000| FPEN |00000000000000000000| 657 * +---+---+------+------+--------------------+ 658 * 659 * where: 660 * TTA: Trace trap 661 * FPEN: Floating point enable 662 */ 663 #define CPACR_TTA_SHIFT 28 664 #define CPACR_TTA (1 << CPACR_TTA_SHIFT) 665 666 667 #define CPACR_FPEN_SHIFT 20 668 #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT) 669 #define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT) 670 671 672 /* 673 * FPSR: Floating Point Status Register 674 * 675 * 31 30 29 28 27 26 7 6 4 3 2 1 0 676 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ 677 * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC| 678 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ 679 */ 680 681 #define FPSR_N_SHIFT 31 682 #define FPSR_Z_SHIFT 30 683 #define FPSR_C_SHIFT 29 684 #define FPSR_V_SHIFT 28 685 #define FPSR_QC_SHIFT 27 686 #define FPSR_IDC_SHIFT 7 687 #define FPSR_IXC_SHIFT 4 688 #define FPSR_UFC_SHIFT 3 689 #define FPSR_OFC_SHIFT 2 690 #define FPSR_DZC_SHIFT 1 691 #define FPSR_IOC_SHIFT 0 692 #define FPSR_N (1 << FPSR_N_SHIFT) 693 #define FPSR_Z (1 << FPSR_Z_SHIFT) 694 #define FPSR_C (1 << FPSR_C_SHIFT) 695 #define FPSR_V (1 << FPSR_V_SHIFT) 696 #define FPSR_QC (1 << FPSR_QC_SHIFT) 697 #define FPSR_IDC (1 << FPSR_IDC_SHIFT) 698 #define FPSR_IXC (1 << FPSR_IXC_SHIFT) 699 #define FPSR_UFC (1 << FPSR_UFC_SHIFT) 700 #define FPSR_OFC (1 << FPSR_OFC_SHIFT) 701 #define FPSR_DZC (1 << FPSR_DZC_SHIFT) 702 #define FPSR_IOC (1 << FPSR_IOC_SHIFT) 703 704 /* 705 * A mask for all for all of the bits that are not RAZ for FPSR; this 706 * is primarily for converting between a 32-bit view of NEON state 707 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). 708 */ 709 #define FPSR_MASK \ 710 (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \ 711 FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC) 712 713 /* 714 * FPCR: Floating Point Control Register 715 * 716 * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0 717 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ 718 * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000| 719 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ 720 */ 721 722 #define FPCR_AHP_SHIFT 26 723 #define FPCR_DN_SHIFT 25 724 #define FPCR_FZ_SHIFT 24 725 #define FPCR_RMODE_SHIFT 22 726 #define FPCR_STRIDE_SHIFT 20 727 #define FPCR_LEN_SHIFT 16 728 #define FPCR_IDE_SHIFT 15 729 #define FPCR_IXE_SHIFT 12 730 #define FPCR_UFE_SHIFT 11 731 #define FPCR_OFE_SHIFT 10 732 #define FPCR_DZE_SHIFT 9 733 #define FPCR_IOE_SHIFT 8 734 #define FPCR_AHP (1 << FPCR_AHP_SHIFT) 735 #define FPCR_DN (1 << FPCR_DN_SHIFT) 736 #define FPCR_FZ (1 << FPCR_FZ_SHIFT) 737 #define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT) 738 #define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT) 739 #define FPCR_LEN (0x7 << FPCR_LEN_SHIFT) 740 #define FPCR_IDE (1 << FPCR_IDE_SHIFT) 741 #define FPCR_IXE (1 << FPCR_IXE_SHIFT) 742 #define FPCR_UFE (1 << FPCR_UFE_SHIFT) 743 #define FPCR_OFE (1 << FPCR_OFE_SHIFT) 744 #define FPCR_DZE (1 << FPCR_DZE_SHIFT) 745 #define FPCR_IOE (1 << FPCR_IOE_SHIFT) 746 #define FPCR_DEFAULT (0) 747 #define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ) 748 749 /* 750 * A mask for all for all of the bits that are not RAZ for FPCR; this 751 * is primarily for converting between a 32-bit view of NEON state 752 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). 753 */ 754 #define FPCR_MASK \ 755 (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \ 756 FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE) 757 758 /* 759 * Translation Control Register (TCR) 760 * 761 * Legacy: 762 * 763 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 764 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ 765 * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| 766 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ 767 * 768 * Current (with 16KB granule support): 769 * 770 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 771 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ 772 * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| 773 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ 774 * 775 * TBI1: Top Byte Ignored for TTBR1 region 776 * TBI0: Top Byte Ignored for TTBR0 region 777 * AS: ASID Size 778 * IPS: Physical Address Size limit 779 * TG1: Granule Size for TTBR1 region 780 * SH1: Shareability for TTBR1 region 781 * ORGN1: Outer Cacheability for TTBR1 region 782 * IRGN1: Inner Cacheability for TTBR1 region 783 * EPD1: Translation table walk disable for TTBR1 784 * A1: ASID selection from TTBR1 enable 785 * T1SZ: Virtual address size for TTBR1 786 * TG0: Granule Size for TTBR0 region 787 * SH0: Shareability for TTBR0 region 788 * ORGN0: Outer Cacheability for TTBR0 region 789 * IRGN0: Inner Cacheability for TTBR0 region 790 * T0SZ: Virtual address size for TTBR0 791 */ 792 793 #define TCR_T0SZ_SHIFT 0ULL 794 #define TCR_TSZ_BITS 6ULL 795 #define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL) 796 797 #define TCR_IRGN0_SHIFT 8ULL 798 #define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT) 799 #define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT) 800 #define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT) 801 #define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT) 802 803 #define TCR_ORGN0_SHIFT 10ULL 804 #define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT) 805 #define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT) 806 #define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT) 807 #define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT) 808 809 #define TCR_SH0_SHIFT 12ULL 810 #define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT) 811 #define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT) 812 #define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT) 813 814 #define TCR_TG0_GRANULE_SHIFT (14ULL) 815 #define TCR_TG0_GRANULE_BITS (2ULL) 816 #define TCR_TG0_GRANULE_MASK ((1ULL << TCR_TG0_GRANULE_BITS) - 1ULL) 817 818 #define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT) 819 #define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT) 820 #define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT) 821 822 #if __ARM_16K_PG__ 823 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB) 824 #else 825 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB) 826 #endif 827 828 #define TCR_T1SZ_SHIFT 16ULL 829 830 #define TCR_A1_ASID1 (1ULL << 22ULL) 831 #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL) 832 833 #define TCR_IRGN1_SHIFT 24ULL 834 #define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT) 835 #define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT) 836 #define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT) 837 #define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT) 838 839 #define TCR_ORGN1_SHIFT 26ULL 840 #define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT) 841 #define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT) 842 #define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT) 843 #define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT) 844 845 #define TCR_SH1_SHIFT 28ULL 846 #define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT) 847 #define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT) 848 #define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT) 849 850 #define TCR_TG1_GRANULE_SHIFT 30ULL 851 #define TCR_TG1_GRANULE_BITS (2ULL) 852 #define TCR_TG1_GRANULE_MASK ((1ULL << TCR_TG1_GRANULE_BITS) - 1ULL) 853 854 #define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT) 855 #define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT) 856 #define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT) 857 858 #if __ARM_16K_PG__ 859 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB) 860 #else 861 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB) 862 #endif 863 864 #define TCR_IPS_SHIFT 32ULL 865 #define TCR_IPS_BITS 3ULL 866 #define TCR_IPS_MASK ((1ULL << TCR_IPS_BITS) - 1ULL) 867 #define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT) 868 #define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT) 869 #define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT) 870 #define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT) 871 #define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT) 872 #define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT) 873 874 #define TCR_AS_16BIT_ASID (1ULL << 36) 875 #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37) 876 #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38) 877 #define TCR_TBID0_TBI_DATA_ONLY (1ULL << 51) 878 #define TCR_TBID1_TBI_DATA_ONLY (1ULL << 52) 879 880 #if defined(HAS_APPLE_PAC) 881 #define TCR_TBID0_ENABLE TCR_TBID0_TBI_DATA_ONLY 882 #define TCR_TBID1_ENABLE TCR_TBID1_TBI_DATA_ONLY 883 #else 884 #define TCR_TBID0_ENABLE 0 885 #define TCR_TBID1_ENABLE 0 886 #endif 887 888 #define TCR_E0PD0_BIT (1ULL << 55) 889 #define TCR_E0PD1_BIT (1ULL << 56) 890 891 #if defined(HAS_E0PD) 892 #define TCR_E0PD_VALUE (TCR_E0PD1_BIT) 893 #else 894 #define TCR_E0PD_VALUE 0 895 #endif 896 897 898 #define TCR_EL1_EXTRA 0 899 900 901 /* 902 * Multiprocessor Affinity Register (MPIDR_EL1) 903 * 904 * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+ 905 * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0| 906 * +---------------------------------+--+-----+--+-----+----+----+ 907 * 908 * where 909 * U: Uniprocessor 910 * MT: Multi-threading at lowest affinity level 911 * Aff2: "1" - PCORE, "0" - ECORE 912 * Aff1: Cluster ID 913 * Aff0: CPU ID 914 */ 915 #define MPIDR_AFF0_SHIFT 0 916 #define MPIDR_AFF0_WIDTH 8 917 #define MPIDR_AFF0_MASK (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT) 918 #define MPIDR_AFF1_SHIFT 8 919 #define MPIDR_AFF1_WIDTH 8 920 #define MPIDR_AFF1_MASK (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT) 921 #define MPIDR_AFF2_SHIFT 16 922 #define MPIDR_AFF2_WIDTH 8 923 #define MPIDR_AFF2_MASK (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT) 924 925 /* 926 * TXSZ indicates the size of the range a TTBR covers. Currently, 927 * we support the following: 928 * 929 * 4KB pages, full page L1: 39 bit range. 930 * 4KB pages, sub-page L1: 38 bit range. 931 * 16KB pages, full page L1: 47 bit range. 932 * 16KB pages, sub-page L1: 39 bit range. 933 * 16KB pages, two level page tables: 36 bit range. 934 */ 935 #if __ARM_KERNEL_PROTECT__ 936 /* 937 * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the 938 * address space is used for the mappings that will remain in place when in EL0. 939 * As a result, 1 bit less of address space is available to the rest of the 940 * the kernel. 941 */ 942 #endif /* __ARM_KERNEL_PROTECT__ */ 943 #ifdef __ARM_16K_PG__ 944 #if __ARM64_PMAP_SUBPAGE_L1__ 945 #define T0SZ_BOOT 25ULL 946 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ 947 #define T0SZ_BOOT 17ULL 948 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ 949 #else /* __ARM_16K_PG__ */ 950 #if __ARM64_PMAP_SUBPAGE_L1__ 951 #define T0SZ_BOOT 26ULL 952 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 953 #define T0SZ_BOOT 25ULL 954 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 955 #endif /* __ARM_16K_PG__ */ 956 957 #if defined(APPLE_ARM64_ARCH_FAMILY) 958 /* T0SZ must be the same as T1SZ */ 959 #define T1SZ_BOOT T0SZ_BOOT 960 #else /* defined(APPLE_ARM64_ARCH_FAMILY) */ 961 #ifdef __ARM_16K_PG__ 962 #if __ARM64_PMAP_SUBPAGE_L1__ 963 #define T1SZ_BOOT 25ULL 964 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ 965 #define T1SZ_BOOT 17ULL 966 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ 967 #else /* __ARM_16K_PG__ */ 968 #if __ARM64_PMAP_SUBPAGE_L1__ 969 #define T1SZ_BOOT 26ULL 970 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 971 #define T1SZ_BOOT 25ULL 972 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/ 973 #endif /* __ARM_16K_PG__ */ 974 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ 975 976 #if __ARM_42BIT_PA_SPACE__ 977 #define TCR_IPS_VALUE TCR_IPS_42BITS 978 #else /* !__ARM_42BIT_PA_SPACE__ */ 979 #define TCR_IPS_VALUE TCR_IPS_40BITS 980 #endif /* !__ARM_42BIT_PA_SPACE__ */ 981 982 #if CONFIG_KERNEL_TBI 983 #define TCR_EL1_DTBI (TCR_TBI1_TOPBYTE_IGNORED | TCR_TBID1_ENABLE) 984 #else /* CONFIG_KERNEL_TBI */ 985 #define TCR_EL1_DTBI 0 986 #endif /* CONFIG_KERNEL_TBI */ 987 988 #if HAS_16BIT_ASID 989 #define TCR_EL1_ASID TCR_AS_16BIT_ASID 990 #else /* HAS_16BIT_ASID */ 991 #define TCR_EL1_ASID 0 992 #endif /* HAS_16BIT_ASID */ 993 994 #define TCR_EL1_BASE \ 995 (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | \ 996 TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | \ 997 TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \ 998 TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) | \ 999 TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE) | TCR_E0PD_VALUE | \ 1000 TCR_EL1_DTBI | TCR_EL1_ASID | TCR_EL1_EXTRA) 1001 1002 #if __ARM_KERNEL_PROTECT__ 1003 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1004 #define T1SZ_USER (T1SZ_BOOT + 1) 1005 #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1006 #else 1007 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1008 #endif /* __ARM_KERNEL_PROTECT__ */ 1009 1010 #define TCR_EL1_4KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_4KB)) 1011 #define TCR_EL1_16KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_16KB)) 1012 1013 1014 1015 1016 /* 1017 * Monitor Debug System Control Register (MDSCR) 1018 */ 1019 1020 #define MDSCR_TFO_SHIFT 31 1021 #define MDSCR_TFO (1ULL << MDSCR_TFO_SHIFT) 1022 #define MDSCR_RXFULL_SHIFT 30 1023 #define MDSCR_RXFULL (1ULL << MDSCR_RXFULL_SHIFT) 1024 #define MDSCR_TXFULL_SHIFT 29 1025 #define MDSCR_TXFULL (1ULL << MDSCR_TXFULL_SHIFT) 1026 #define MDSCR_RXO_SHIFT 27 1027 #define MDSCR_RXO (1ULL << MDSCR_RXO_SHIFT) 1028 #define MDSCR_TXU_SHIFT 26 1029 #define MDSCR_TXU (1ULL << MDSCR_TXU_SHIFT) 1030 #define MDSCR_INTDIS_SHIFT 22 1031 #define MDSCR_INTDIS_MASK (0x2U << MDSCR_INTDIS_SHIFT) 1032 #define MDSCR_TDA_SHIFT 21 1033 #define MDSCR_TDA (1ULL << MDSCR_TDA_SHIFT) 1034 #define MDSCR_SC2_SHIFT 19 1035 #define MDSCR_SC2 (1ULL << MDSCR_SC2_SHIFT) 1036 #define MDSCR_MDE_SHIFT 15 1037 #define MDSCR_MDE (1ULL << MDSCR_MDE_SHIFT) 1038 #define MDSCR_HDE_SHIFT 14 1039 #define MDSCR_HDE (1ULL << MDSCR_HDE_SHIFT) 1040 #define MDSCR_KDE_SHIFT 13 1041 #define MDSCR_KDE (1ULL << MDSCR_KDE_SHIFT) 1042 #define MDSCR_TDCC_SHIFT 12 1043 #define MDSCR_TDCC (1ULL << MDSCR_TDCC_SHIFT) 1044 #define MDSCR_ERR_SHIFT 6 1045 #define MDSCR_ERR (1ULL << MDSCR_ERR_SHIFT) 1046 #define MDSCR_SS_SHIFT 0 1047 #define MDSCR_SS (1ULL << MDSCR_SS_SHIFT) 1048 1049 /* 1050 * Translation Table Base Register (TTBR) 1051 * 1052 * 63 48 47 x x-1 0 1053 * +--------+------------------+------+ 1054 * | ASID | Base Address | zero | 1055 * +--------+------------------+------+ 1056 * 1057 */ 1058 #define TTBR_ASID_SHIFT 48 1059 #define TTBR_ASID_MASK 0xffff000000000000 1060 1061 #define TTBR_BADDR_MASK 0x0000ffffffffffff 1062 1063 /* 1064 * Memory Attribute Indirection Register 1065 * 1066 * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0 1067 * +-------+-------+-------+-------+-------+-------+-------+-------+ 1068 * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 | 1069 * +-------+-------+-------+-------+-------+-------+-------+-------+ 1070 * 1071 */ 1072 1073 #define MAIR_ATTR_SHIFT(x) (8*(x)) 1074 1075 /* Strongly ordered or device memory attributes */ 1076 #define MAIR_OUTER_STRONGLY_ORDERED 0x0 1077 #define MAIR_OUTER_DEVICE 0x0 1078 1079 #define MAIR_INNER_STRONGLY_ORDERED 0x0 1080 #define MAIR_INNER_DEVICE 0x4 1081 1082 /* Normal memory attributes */ 1083 #define MAIR_OUTER_NON_CACHEABLE 0x40 1084 #define MAIR_OUTER_WRITE_THROUGH 0x80 1085 #define MAIR_OUTER_WRITE_BACK 0xc0 1086 1087 #define MAIR_INNER_NON_CACHEABLE 0x4 1088 #define MAIR_INNER_WRITE_THROUGH 0x8 1089 #define MAIR_INNER_WRITE_BACK 0xc 1090 1091 /* Allocate policy for cacheable memory */ 1092 #define MAIR_OUTER_WRITE_ALLOCATE 0x10 1093 #define MAIR_OUTER_READ_ALLOCATE 0x20 1094 1095 #define MAIR_INNER_WRITE_ALLOCATE 0x1 1096 #define MAIR_INNER_READ_ALLOCATE 0x2 1097 1098 /* Memory Atribute Encoding */ 1099 1100 /* 1101 * Device memory types: 1102 * G (gathering): multiple reads/writes can be combined 1103 * R (reordering): reads or writes may reach device out of program order 1104 * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes) 1105 */ 1106 #if HAS_FEAT_XS 1107 1108 #define MAIR_DISABLE 0x01 /* Device Memory, nGnRnE (strongly ordered), XS=0 */ 1109 #define MAIR_POSTED_XS 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes), XS=1 */ 1110 #define MAIR_POSTED_COMBINED_REORDERED_XS 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=1 */ 1111 #define MAIR_POSTED_COMBINED_REORDERED 0x0D /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=0 */ 1112 #define MAIR_WRITECOMB 0x40 /* Normal Memory, Non-Cacheable, XS=0 */ 1113 #define MAIR_WRITETHRU 0xA0 /* Normal Memory, Write-through, XS=0 */ 1114 #define MAIR_WRITEBACK 0xFF /* Normal Memory, Write-back, XS=0 */ 1115 1116 /* 1117 * Memory Attribute Index. If these values change, please also update the pmap 1118 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). 1119 */ 1120 #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ 1121 #define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */ 1122 #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ 1123 #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ 1124 #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */ 1125 #define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */ 1126 #define CACHE_ATTRINDX_POSTED_XS 0x5 /* no cache, no buffer, posted writes (device memory), XS=1 */ 1127 #define CACHE_ATTRINDX_POSTED CACHE_ATTRINDX_POSTED_XS /* posted mappings use XS by default */ 1128 #define CACHE_ATTRINDX_POSTED_REORDERED CACHE_ATTRINDX_DISABLE /* no need for device-nGRE on newer SoCs, fallback to nGnRnE */ 1129 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x6 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=0 */ 1130 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED_XS 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=1 */ 1131 #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK 1132 #define CACHE_ATTRINDX_N_INDICES (8ULL) 1133 1134 #else 1135 1136 #define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */ 1137 #define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */ 1138 #define MAIR_POSTED_REORDERED 0x08 /* Device Memory, nGRE (reorderable, posted writes) */ 1139 #define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */ 1140 #define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */ 1141 #define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */ 1142 #define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */ 1143 1144 /* 1145 * Memory Attribute Index. If these values change, please also update the pmap 1146 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). 1147 */ 1148 #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ 1149 #define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */ 1150 #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ 1151 #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ 1152 #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */ 1153 #define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */ 1154 #define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes (device memory) */ 1155 #define CACHE_ATTRINDX_POSTED_REORDERED 0x6 /* no cache, reorderable access, posted writes (device memory) */ 1156 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */ 1157 #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK 1158 #define CACHE_ATTRINDX_N_INDICES (8ULL) 1159 1160 #endif /* HAS_FEAT_XS */ 1161 1162 #if HAS_UCNORMAL_MEM 1163 #define CACHE_ATTRINDX_RT CACHE_ATTRINDX_WRITECOMB 1164 #else 1165 #define CACHE_ATTRINDX_RT CACHE_ATTRINDX_DISABLE 1166 #endif /* HAS_UCNORMAL_MEM */ 1167 1168 1169 1170 /* 1171 * Access protection bit values (TTEs and PTEs), stage 1 1172 * 1173 * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access) 1174 */ 1175 #define AP_RWNA 0x0 /* priv=read-write, user=no-access */ 1176 #define AP_RWRW 0x1 /* priv=read-write, user=read-write */ 1177 #define AP_RONA 0x2 /* priv=read-only, user=no-access */ 1178 #define AP_RORO 0x3 /* priv=read-only, user=read-only */ 1179 #define AP_MASK 0x3 /* mask to find ap bits */ 1180 1181 /* 1182 * Shareability attributes 1183 */ 1184 #define SH_NONE 0x0 /* Non shareable */ 1185 #define SH_NONE 0x0 /* Device shareable */ 1186 #define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */ 1187 #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */ 1188 #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */ 1189 1190 1191 /* 1192 * ARM Page Granule 1193 */ 1194 #ifdef __ARM_16K_PG__ 1195 #define ARM_PGSHIFT 14 1196 #else 1197 #define ARM_PGSHIFT 12 1198 #endif 1199 #define ARM_PGBYTES (1 << ARM_PGSHIFT) 1200 #define ARM_PGMASK (ARM_PGBYTES-1) 1201 1202 /* 1203 * L0 Translation table 1204 * 1205 * 4KB granule size: 1206 * Each translation table is 4KB 1207 * 512 64-bit entries of 512GB (2^39) of address space. 1208 * Covers 256TB (2^48) of address space. 1209 * 1210 * 16KB granule size: 1211 * Each translation table is 16KB 1212 * 2 64-bit entries of 128TB (2^47) of address space. 1213 * Covers 256TB (2^48) of address space. 1214 */ 1215 1216 /* 16K L0 */ 1217 #define ARM_16K_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */ 1218 #define ARM_16K_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */ 1219 #define ARM_16K_TT_L0_SHIFT 47 /* page descriptor shift */ 1220 #define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */ 1221 1222 /* 4K L0 */ 1223 #define ARM_4K_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */ 1224 #define ARM_4K_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */ 1225 #define ARM_4K_TT_L0_SHIFT 39 /* page descriptor shift */ 1226 #define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */ 1227 1228 /* 1229 * L1 Translation table 1230 * 1231 * 4KB granule size: 1232 * Each translation table is 4KB 1233 * 512 64-bit entries of 1GB (2^30) of address space. 1234 * Covers 512GB (2^39) of address space. 1235 * 1236 * 16KB granule size: 1237 * Each translation table is 16KB 1238 * 2048 64-bit entries of 64GB (2^36) of address space. 1239 * Covers 128TB (2^47) of address space. 1240 */ 1241 1242 /* 16K L1 */ 1243 #define ARM_16K_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */ 1244 #define ARM_16K_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */ 1245 #define ARM_16K_TT_L1_SHIFT 36 /* page descriptor shift */ 1246 #if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ 1247 /* This config supports 512GB per TTBR. */ 1248 #define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */ 1249 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 1250 #define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */ 1251 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 1252 1253 /* 4K L1 */ 1254 #define ARM_4K_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */ 1255 #define ARM_4K_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */ 1256 #define ARM_4K_TT_L1_SHIFT 30 /* page descriptor shift */ 1257 #if __ARM64_PMAP_SUBPAGE_L1__ && !__ARM_16K_PG__ 1258 /* This config supports 256GB per TTBR. */ 1259 #define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */ 1260 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 1261 #define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL /* mask for getting index into L1 table from virtual address */ 1262 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 1263 1264 /* some sugar for getting pointers to page tables and entries */ 1265 1266 #define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT) 1267 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) 1268 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT) 1269 1270 #define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK)) 1271 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK)) 1272 1273 /* 1274 * L2 Translation table 1275 * 1276 * 4KB granule size: 1277 * Each translation table is 4KB 1278 * 512 64-bit entries of 2MB (2^21) of address space. 1279 * Covers 1GB (2^30) of address space. 1280 * 1281 * 16KB granule size: 1282 * Each translation table is 16KB 1283 * 2048 64-bit entries of 32MB (2^25) of address space. 1284 * Covers 64GB (2^36) of address space. 1285 */ 1286 1287 /* 16K L2 */ 1288 #define ARM_16K_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */ 1289 #define ARM_16K_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */ 1290 #define ARM_16K_TT_L2_SHIFT 25 /* page descriptor shift */ 1291 #define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */ 1292 1293 /* 4K L2 */ 1294 #define ARM_4K_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */ 1295 #define ARM_4K_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */ 1296 #define ARM_4K_TT_L2_SHIFT 21 /* page descriptor shift */ 1297 #define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */ 1298 1299 /* 1300 * L3 Translation table 1301 * 1302 * 4KB granule size: 1303 * Each translation table is 4KB 1304 * 512 64-bit entries of 4KB (2^12) of address space. 1305 * Covers 2MB (2^21) of address space. 1306 * 1307 * 16KB granule size: 1308 * Each translation table is 16KB 1309 * 2048 64-bit entries of 16KB (2^14) of address space. 1310 * Covers 32MB (2^25) of address space. 1311 */ 1312 1313 /* 16K L3 */ 1314 #define ARM_16K_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ 1315 #define ARM_16K_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */ 1316 #define ARM_16K_TT_L3_SHIFT 14 /* page descriptor shift */ 1317 #define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */ 1318 1319 /* 4K L3 */ 1320 #define ARM_4K_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ 1321 #define ARM_4K_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */ 1322 #define ARM_4K_TT_L3_SHIFT 12 /* page descriptor shift */ 1323 #define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */ 1324 1325 #ifdef __ARM_16K_PG__ 1326 1327 /* Native L0 defines */ 1328 #define ARM_TT_L0_SIZE ARM_16K_TT_L0_SIZE 1329 #define ARM_TT_L0_OFFMASK ARM_16K_TT_L0_OFFMASK 1330 #define ARM_TT_L0_SHIFT ARM_16K_TT_L0_SHIFT 1331 #define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK 1332 1333 /* Native L1 defines */ 1334 #define ARM_TT_L1_SIZE ARM_16K_TT_L1_SIZE 1335 #define ARM_TT_L1_OFFMASK ARM_16K_TT_L1_OFFMASK 1336 #define ARM_TT_L1_SHIFT ARM_16K_TT_L1_SHIFT 1337 #define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK 1338 1339 /* Native L2 defines */ 1340 #define ARM_TT_L2_SIZE ARM_16K_TT_L2_SIZE 1341 #define ARM_TT_L2_OFFMASK ARM_16K_TT_L2_OFFMASK 1342 #define ARM_TT_L2_SHIFT ARM_16K_TT_L2_SHIFT 1343 #define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK 1344 1345 /* Native L3 defines */ 1346 #define ARM_TT_L3_SIZE ARM_16K_TT_L3_SIZE 1347 #define ARM_TT_L3_OFFMASK ARM_16K_TT_L3_OFFMASK 1348 #define ARM_TT_L3_SHIFT ARM_16K_TT_L3_SHIFT 1349 #define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK 1350 1351 #else /* !__ARM_16K_PG__ */ 1352 1353 /* Native L0 defines */ 1354 #define ARM_TT_L0_SIZE ARM_4K_TT_L0_SIZE 1355 #define ARM_TT_L0_OFFMASK ARM_4K_TT_L0_OFFMASK 1356 #define ARM_TT_L0_SHIFT ARM_4K_TT_L0_SHIFT 1357 #define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK 1358 1359 /* Native L1 defines */ 1360 #define ARM_TT_L1_SIZE ARM_4K_TT_L1_SIZE 1361 #define ARM_TT_L1_OFFMASK ARM_4K_TT_L1_OFFMASK 1362 #define ARM_TT_L1_SHIFT ARM_4K_TT_L1_SHIFT 1363 #define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK 1364 1365 /* Native L2 defines */ 1366 #define ARM_TT_L2_SIZE ARM_4K_TT_L2_SIZE 1367 #define ARM_TT_L2_OFFMASK ARM_4K_TT_L2_OFFMASK 1368 #define ARM_TT_L2_SHIFT ARM_4K_TT_L2_SHIFT 1369 #define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK 1370 1371 /* Native L3 defines */ 1372 #define ARM_TT_L3_SIZE ARM_4K_TT_L3_SIZE 1373 #define ARM_TT_L3_OFFMASK ARM_4K_TT_L3_OFFMASK 1374 #define ARM_TT_L3_SHIFT ARM_4K_TT_L3_SHIFT 1375 #define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK 1376 1377 #endif /* !__ARM_16K_PG__ */ 1378 1379 /* 1380 * Convenience definitions for: 1381 * ARM_TT_LEAF: The last level of the configured page table format. 1382 * ARM_TT_TWIG: The second to last level of the configured page table format. 1383 * ARM_TT_ROOT: The first level of the configured page table format. 1384 * 1385 * My apologies to any botanists who may be reading this. 1386 */ 1387 #define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE 1388 #define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK 1389 #define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT 1390 #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK 1391 1392 #define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE 1393 #define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK 1394 #define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT 1395 #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK 1396 1397 #define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE 1398 #define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK 1399 #define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT 1400 #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK 1401 1402 /* 1403 * 4KB granule size: 1404 * 1405 * Level 0 Translation Table Entry 1406 * 1407 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1408 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1409 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| 1410 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1411 * 1412 * Level 1 Translation Table Entry 1413 * 1414 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1415 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1416 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| 1417 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1418 * 1419 * Level 1 Translation Block Entry 1420 * 1421 * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0 1422 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1423 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1424 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1425 * 1426 * Level 2 Translation Table Entry 1427 * 1428 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1429 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1430 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| 1431 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1432 * 1433 * Level 2 Translation Block Entry 1434 * 1435 * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0 1436 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1437 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1438 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1439 * 1440 * 16KB granule size: 1441 * 1442 * Level 0 Translation Table Entry 1443 * 1444 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1445 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1446 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| 1447 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1448 * 1449 * Level 1 Translation Table Entry 1450 * 1451 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1452 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1453 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| 1454 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1455 * 1456 * Level 2 Translation Table Entry 1457 * 1458 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1459 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1460 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| 1461 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1462 * 1463 * Level 2 Translation Block Entry 1464 * 1465 * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0 1466 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1467 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1468 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1469 * 1470 * where: 1471 * nG: notGlobal bit 1472 * SH: Shareability field 1473 * AP: access protection 1474 * XN: eXecute Never bit 1475 * PXN: Privilege eXecute Never bit 1476 * NS: Non-Secure bit 1477 * HINT: 16 entry continuguous output hint 1478 * AttrIdx: Memory Attribute Index 1479 */ 1480 1481 #define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */ 1482 #ifdef __ARM_16K_PG__ 1483 #define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */ 1484 #else 1485 #define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */ 1486 #endif 1487 1488 #define ARM_TTE_MAX (TTE_PGENTRIES) 1489 1490 #define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ 1491 #define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */ 1492 1493 #define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */ 1494 1495 #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ 1496 #define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */ 1497 #define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */ 1498 #define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL 1499 #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ 1500 1501 #ifdef __ARM_16K_PG__ 1502 /* 1503 * Note that L0/L1 block entries are disallowed for the 16KB granule size; what 1504 * are we doing with these? 1505 */ 1506 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */ 1507 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */ 1508 #define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */ 1509 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */ 1510 #define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */ 1511 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */ 1512 #else 1513 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */ 1514 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */ 1515 #define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */ 1516 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */ 1517 #define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */ 1518 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */ 1519 #endif 1520 1521 #define ARM_TTE_BLOCK_APSHIFT 6 1522 #define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */ 1523 #define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT) 1524 1525 #define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */ 1526 #define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ 1527 1528 #define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */ 1529 #define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */ 1530 1531 #define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */ 1532 #define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */ 1533 1534 #define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */ 1535 #define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ 1536 1537 #define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */ 1538 #define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ 1539 1540 #define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ 1541 #define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ 1542 1543 #define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */ 1544 #define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */ 1545 1546 #define ARM_TTE_BLOCK_WIRED 0x0400000000000000ULL /* value for software wired bit */ 1547 #define ARM_TTE_BLOCK_WIREDMASK 0x0400000000000000ULL /* software wired mask */ 1548 1549 #define ARM_TTE_BLOCK_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ 1550 #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL /* software writeable mask */ 1551 1552 #define ARM_TTE_TABLE_MASK 0x0000fffffffff000ULL /* mask for extracting pointer to next table (works at any level) */ 1553 1554 #define ARM_TTE_TABLE_APSHIFT 61 1555 #define ARM_TTE_TABLE_AP_NO_EFFECT 0x0ULL 1556 #define ARM_TTE_TABLE_AP_USER_NA 0x1ULL 1557 #define ARM_TTE_TABLE_AP_RO 0x2ULL 1558 #define ARM_TTE_TABLE_AP_KERN_RO 0x3ULL 1559 #define ARM_TTE_TABLE_AP(x) ((x) << ARM_TTE_TABLE_APSHIFT) /* access protection */ 1560 1561 #define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */ 1562 #define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */ 1563 1564 #define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */ 1565 #define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */ 1566 1567 #define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */ 1568 #define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */ 1569 1570 #if __ARM_KERNEL_PROTECT__ 1571 #define ARM_TTE_BOOT_BLOCK \ 1572 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ 1573 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG) 1574 #else /* __ARM_KERNEL_PROTECT__ */ 1575 #define ARM_TTE_BOOT_BLOCK \ 1576 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ 1577 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF) 1578 #endif /* __ARM_KERNEL_PROTECT__ */ 1579 1580 #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID ) 1581 /* 1582 * L3 Translation table 1583 * 1584 * 4KB granule size: 1585 * Each translation table is 4KB 1586 * 512 64-bit entries of 4KB (2^12) of address space. 1587 * Covers 2MB (2^21) of address space. 1588 * 1589 * 16KB granule size: 1590 * Each translation table is 16KB 1591 * 2048 64-bit entries of 16KB (2^14) of address space. 1592 * Covers 32MB (2^25) of address space. 1593 */ 1594 1595 #ifdef __ARM_16K_PG__ 1596 #define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ 1597 #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */ 1598 #define ARM_PTE_SHIFT 14 /* page descriptor shift */ 1599 #define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */ 1600 #else 1601 #define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ 1602 #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */ 1603 #define ARM_PTE_SHIFT 12 /* page descriptor shift */ 1604 #define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */ 1605 #endif 1606 1607 #define ARM_TTE_PA_MASK 0x0000fffffffff000ULL 1608 1609 /* 1610 * L3 Page table entries 1611 * 1612 * The following page table entry types are possible: 1613 * 1614 * fault page entry 1615 * 63 2 0 1616 * +------------------------------+--+ 1617 * | ignored |00| 1618 * +------------------------------+--+ 1619 * 1620 * 1621 * 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0 1622 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+ 1623 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V| 1624 * +-----+------+--+---+----+------+----------------------+--+--+----+----+--+-------+-+-+ 1625 * 1626 * where: 1627 * nG: notGlobal bit 1628 * SH: Shareability field 1629 * AP: access protection 1630 * XN: eXecute Never bit 1631 * PXN: Privilege eXecute Never bit 1632 * NS: Non-Secure bit 1633 * HINT: 16 entry continuguous output hint 1634 * AttrIdx: Memory Attribute Index 1635 */ 1636 1637 #define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */ 1638 #ifdef __ARM_16K_PG__ 1639 #define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */ 1640 #else 1641 #define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */ 1642 #endif 1643 1644 #define ARM_PTE_EMPTY 0x0000000000000000ULL /* unassigned - invalid entry */ 1645 1646 /* markers for (invalid) PTE for a page sent to compressor */ 1647 #define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */ 1648 #define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */ 1649 #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL 1650 1651 #define ARM_PTE_TYPE 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ 1652 #define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ 1653 #define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */ 1654 #define ARM_PTE_TYPE_MASK 0x0000000000000002ULL /* mask to get pte type */ 1655 1656 /* This mask works for both 16K and 4K pages because bits 12-13 will be zero in 16K pages */ 1657 #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* output address mask for page */ 1658 #define ARM_PTE_PAGE_SHIFT 12 /* page shift for the output address in the entry */ 1659 1660 #define ARM_PTE_AP(x) ((x) << 6) /* access protections */ 1661 #define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */ 1662 #define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */ 1663 1664 #define ARM_PTE_ATTRINDX(x) ((x) << 2) /* memory attributes index */ 1665 #define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ 1666 #define ARM_PTE_EXTRACT_ATTRINDX(x) (((x) >> 2) & 0x7ULL) /* extract memory attributes index */ 1667 1668 #define ARM_PTE_SH(x) ((x) << 8) /* access shared */ 1669 #define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */ 1670 1671 #define ARM_PTE_AF 0x0000000000000400ULL /* value for access */ 1672 #define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */ 1673 1674 #define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */ 1675 #define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ 1676 1677 #define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */ 1678 #define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ 1679 1680 #define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */ 1681 #define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */ 1682 1683 #if __ARM_16K_PG__ 1684 #define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */ 1685 #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */ 1686 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */ 1687 #define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */ 1688 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */ 1689 #else 1690 #define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */ 1691 #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */ 1692 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */ 1693 #define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */ 1694 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */ 1695 #endif 1696 1697 #define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ 1698 #define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ 1699 1700 #define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */ 1701 #define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */ 1702 1703 #define ARM_PTE_XMASK (ARM_PTE_PNXMASK | ARM_PTE_NXMASK) 1704 1705 #define ARM_PTE_WIRED 0x0400000000000000ULL /* value for software wired bit */ 1706 #define ARM_PTE_WIRED_MASK 0x0400000000000000ULL /* software wired mask */ 1707 1708 #define ARM_PTE_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ 1709 #define ARM_PTE_WRITEABLE_MASK 0x0800000000000000ULL /* software writeable mask */ 1710 1711 #define ARM_PTE_BOOT_PAGE_BASE \ 1712 (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | \ 1713 ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF) 1714 1715 #if __ARM_KERNEL_PROTECT__ 1716 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG) 1717 #else /* __ARM_KERNEL_PROTECT__ */ 1718 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE) 1719 #endif /* __ARM_KERNEL_PROTECT__ */ 1720 1721 /* 1722 * TLBI appers to only deal in 4KB page addresses, so give 1723 * it an explicit shift of 12. 1724 */ 1725 #define TLBI_ADDR_SHIFT (0) 1726 #define TLBI_ADDR_SIZE (44) 1727 #define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) 1728 #define TLBI_ASID_SHIFT (48) 1729 #define TLBI_ASID_SIZE (16) 1730 #define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1)) 1731 1732 #define RTLBI_ADDR_SIZE (37) 1733 #define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1) 1734 #define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT 1735 #define RTLBI_TG(_page_shift_) ((uint64_t)((((_page_shift_) - 12) >> 1) + 1) << 46) 1736 #define RTLBI_SCALE_SHIFT (44) 1737 #define RTLBI_NUM_SHIFT (39) 1738 1739 /* 1740 * Exception Syndrome Register 1741 * 1742 * 31 26 25 24 0 1743 * +------+--+------------------+ 1744 * | EC |IL| ISS | 1745 * +------+--+------------------+ 1746 * 1747 * EC - Exception Class 1748 * IL - Instruction Length 1749 * ISS - Instruction Specific Syndrome 1750 * 1751 * Note: The ISS can have many forms. These are defined separately below. 1752 */ 1753 1754 #define ESR_EC_SHIFT 26 1755 #define ESR_EC_MASK (0x3FULL << ESR_EC_SHIFT) 1756 #define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT) 1757 1758 #define ESR_IL_SHIFT 25 1759 #define ESR_IL (1 << ESR_IL_SHIFT) 1760 1761 #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL)) 1762 1763 #define ESR_ISS_MASK 0x01FFFFFF 1764 #define ESR_ISS(x) (x & ESR_ISS_MASK) 1765 1766 #ifdef __ASSEMBLER__ 1767 /* Define only the classes we need to test in the exception vectors. */ 1768 #define ESR_EC_UNCATEGORIZED 0x00 1769 #define ESR_EC_PAC_FAIL 0x1C 1770 #define ESR_EC_IABORT_EL1 0x21 1771 #define ESR_EC_DABORT_EL1 0x25 1772 #define ESR_EC_SP_ALIGN 0x26 1773 #define ESR_EC_BRK_AARCH64 0x3C 1774 #else 1775 typedef enum { 1776 ESR_EC_UNCATEGORIZED = 0x00, 1777 ESR_EC_WFI_WFE = 0x01, 1778 ESR_EC_MCR_MRC_CP15_TRAP = 0x03, 1779 ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04, 1780 ESR_EC_MCR_MRC_CP14_TRAP = 0x05, 1781 ESR_EC_LDC_STC_CP14_TRAP = 0x06, 1782 ESR_EC_TRAP_SIMD_FP = 0x07, 1783 ESR_EC_PTRAUTH_INSTR_TRAP = 0x09, 1784 ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c, 1785 ESR_EC_ILLEGAL_INSTR_SET = 0x0e, 1786 ESR_EC_SVC_32 = 0x11, 1787 ESR_EC_HVC_32 = 0x12, 1788 ESR_EC_SVC_64 = 0x15, 1789 ESR_EC_HVC_64 = 0x16, 1790 ESR_EC_MSR_TRAP = 0x18, 1791 #if __has_feature(ptrauth_calls) 1792 ESR_EC_PAC_FAIL = 0x1C, 1793 #endif /* __has_feature(ptrauth_calls) */ 1794 ESR_EC_IABORT_EL0 = 0x20, 1795 ESR_EC_IABORT_EL1 = 0x21, 1796 ESR_EC_PC_ALIGN = 0x22, 1797 ESR_EC_DABORT_EL0 = 0x24, 1798 ESR_EC_DABORT_EL1 = 0x25, 1799 ESR_EC_SP_ALIGN = 0x26, 1800 ESR_EC_FLOATING_POINT_32 = 0x28, 1801 ESR_EC_FLOATING_POINT_64 = 0x2C, 1802 ESR_EC_SERROR_INTERRUPT = 0x2F, 1803 ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL. 1804 ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL. 1805 ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL. 1806 ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL. 1807 ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL. 1808 ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL. 1809 ESR_EC_BKPT_AARCH32 = 0x38, 1810 ESR_EC_BRK_AARCH64 = 0x3C, 1811 } esr_exception_class_t; 1812 1813 typedef enum { 1814 FSC_TRANSLATION_FAULT_L0 = 0x04, 1815 FSC_TRANSLATION_FAULT_L1 = 0x05, 1816 FSC_TRANSLATION_FAULT_L2 = 0x06, 1817 FSC_TRANSLATION_FAULT_L3 = 0x07, 1818 FSC_ACCESS_FLAG_FAULT_L1 = 0x09, 1819 FSC_ACCESS_FLAG_FAULT_L2 = 0x0A, 1820 FSC_ACCESS_FLAG_FAULT_L3 = 0x0B, 1821 FSC_PERMISSION_FAULT_L1 = 0x0D, 1822 FSC_PERMISSION_FAULT_L2 = 0x0E, 1823 FSC_PERMISSION_FAULT_L3 = 0x0F, 1824 FSC_SYNC_EXT_ABORT = 0x10, 1825 FSC_SYNC_EXT_ABORT_TT_L1 = 0x15, 1826 FSC_SYNC_EXT_ABORT_TT_L2 = 0x16, 1827 FSC_SYNC_EXT_ABORT_TT_L3 = 0x17, 1828 FSC_SYNC_PARITY = 0x18, 1829 FSC_ASYNC_PARITY = 0x19, 1830 FSC_SYNC_PARITY_TT_L1 = 0x1D, 1831 FSC_SYNC_PARITY_TT_L2 = 0x1E, 1832 FSC_SYNC_PARITY_TT_L3 = 0x1F, 1833 FSC_ALIGNMENT_FAULT = 0x21, 1834 FSC_DEBUG_FAULT = 0x22, 1835 } fault_status_t; 1836 #endif /* ASSEMBLER */ 1837 1838 /* 1839 * HVC event 1840 * 24 16 15 0 1841 * +---------+-----+ 1842 * |000000000| IMM | 1843 * +---------+-----+ 1844 * 1845 * where: 1846 * IMM: Immediate value 1847 */ 1848 1849 #define ISS_HVC_IMM_MASK 0xffff 1850 #define ISS_HVC_IMM(x) ((x) & ISS_HVC_IMM_MASK) 1851 1852 /* 1853 * Software step debug event ISS (EL1) 1854 * 24 23 6 5 0 1855 * +---+-----------------+--+------+ 1856 * |ISV|00000000000000000|EX| IFSC | 1857 * +---+-----------------+--+------+ 1858 * 1859 * where: 1860 * ISV: Instruction syndrome valid 1861 * EX: Exclusive access 1862 * IFSC: Instruction Fault Status Code 1863 */ 1864 1865 #define ISS_SSDE_ISV_SHIFT 24 1866 #define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT) 1867 1868 #define ISS_SSDE_EX_SHIFT 6 1869 #define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT) 1870 1871 #define ISS_SSDE_FSC_MASK 0x3F 1872 #define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK) 1873 1874 /* 1875 * Instruction Abort ISS (EL1) 1876 * 24 10 9 5 0 1877 * +--------------+---+--+---+------+ 1878 * |00000000000000|FnV|EA|000| IFSC | 1879 * +--------------+---+--+---+------+ 1880 * 1881 * where: 1882 * FnV: FAR not Valid 1883 * EA: External Abort type 1884 * IFSC: Instruction Fault Status Code 1885 */ 1886 1887 #define ISS_IA_FNV_SHIFT 10 1888 #define ISS_IA_FNV (0x1 << ISS_IA_FNV_SHIFT) 1889 1890 #define ISS_IA_EA_SHIFT 9 1891 #define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT) 1892 1893 #define ISS_IA_FSC_MASK 0x3F 1894 #define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK) 1895 1896 1897 /* 1898 * Data Abort ISS (EL1) 1899 * 1900 * 24 10 9 8 7 6 5 0 1901 * +--------------+---+--+--+-----+---+----+ 1902 * |00000000000000|FnV|EA|CM|S1PTW|WnR|DFSC| 1903 * +--------------+---+--+--+-----+---+----+ 1904 * 1905 * where: 1906 * FnV: FAR not Valid 1907 * EA: External Abort type 1908 * CM: Cache Maintenance operation 1909 * WnR: Write not Read 1910 * S1PTW: Stage 2 exception on Stage 1 page table walk 1911 * DFSC: Data Fault Status Code 1912 */ 1913 #define ISS_DA_FNV_SHIFT 10 1914 #define ISS_DA_FNV (0x1 << ISS_DA_FNV_SHIFT) 1915 1916 #define ISS_DA_EA_SHIFT 9 1917 #define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT) 1918 1919 #define ISS_DA_CM_SHIFT 8 1920 #define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT) 1921 1922 #define ISS_DA_WNR_SHIFT 6 1923 #define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT) 1924 1925 #define ISS_DA_S1PTW_SHIFT 7 1926 #define ISS_DA_S1PTW (0x1 << ISS_DA_S1PTW_SHIFT) 1927 1928 #define ISS_DA_FSC_MASK 0x3F 1929 #define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK) 1930 1931 /* 1932 * Floating Point Exception ISS (EL1) 1933 * 1934 * 24 23 22 8 7 4 3 2 1 0 1935 * +-+---+---------------+---+--+---+---+---+---+---+ 1936 * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF| 1937 * +-+---+---------------+---+--+---+---+---+---+---+ 1938 * 1939 * where: 1940 * TFV: Trapped Fault Valid 1941 * IDF: Input Denormal Exception 1942 * IXF: Input Inexact Exception 1943 * UFF: Underflow Exception 1944 * OFF: Overflow Exception 1945 * DZF: Divide by Zero Exception 1946 * IOF: Invalid Operation Exception 1947 */ 1948 #define ISS_FP_TFV_SHIFT 23 1949 #define ISS_FP_TFV (0x1 << ISS_FP_TFV_SHIFT) 1950 1951 #define ISS_FP_IDF_SHIFT 7 1952 #define ISS_FP_IDF (0x1 << ISS_FP_IDF_SHIFT) 1953 1954 #define ISS_FP_IXF_SHIFT 4 1955 #define ISS_FP_IXF (0x1 << ISS_FP_IXF_SHIFT) 1956 1957 #define ISS_FP_UFF_SHIFT 3 1958 #define ISS_FP_UFF (0x1 << ISS_FP_UFF_SHIFT) 1959 1960 #define ISS_FP_OFF_SHIFT 2 1961 #define ISS_FP_OFF (0x1 << ISS_FP_OFF_SHIFT) 1962 1963 #define ISS_FP_DZF_SHIFT 1 1964 #define ISS_FP_DZF (0x1 << ISS_FP_DZF_SHIFT) 1965 1966 #define ISS_FP_IOF_SHIFT 0 1967 #define ISS_FP_IOF (0x1 << ISS_FP_IOF_SHIFT) 1968 1969 /* 1970 * Breakpoint Exception ISS (EL1) 1971 * 24 16 0 1972 * +---------+---------+ 1973 * |000000000| Comment | 1974 * +---------+---------+ 1975 * 1976 * where: 1977 * Comment: Instruction Comment Field Value 1978 */ 1979 #define ISS_BRK_COMMENT_MASK 0xFFFF 1980 #define ISS_BRK_COMMENT(x) (x & ISS_BRK_COMMENT_MASK) 1981 1982 1983 /* 1984 * SError Interrupt, IDS=1 1985 * 24 23 0 1986 * +---+------------------------+ 1987 * |IDS| IMPLEMENTATION DEFINED | 1988 * +---+------------------------+ 1989 * 1990 * where: 1991 * IDS: Implementation-defined syndrome (1) 1992 */ 1993 1994 #define ISS_SEI_IDS_SHIFT 24 1995 #define ISS_SEI_IDS (0x1 << ISS_SEI_IDS_SHIFT) 1996 1997 1998 #if HAS_UCNORMAL_MEM 1999 #define ISS_UC 0x11 2000 #endif /* HAS_UCNORMAL_MEM */ 2001 2002 2003 2004 2005 /* 2006 * Physical Address Register (EL1) 2007 */ 2008 #define PAR_F_SHIFT 0 2009 #define PAR_F (0x1 << PAR_F_SHIFT) 2010 2011 #define PLATFORM_SYSCALL_TRAP_NO 0x80000000 2012 2013 #define ARM64_SYSCALL_CODE_REG_NUM (16) 2014 2015 #define ARM64_CLINE_SHIFT 6 2016 2017 #if defined(APPLE_ARM64_ARCH_FAMILY) 2018 #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */ 2019 #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */ 2020 #endif 2021 2022 /* 2023 * Timer definitions. 2024 */ 2025 #define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */ 2026 #define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */ 2027 #define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */ 2028 #define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */ 2029 #define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */ 2030 #define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */ 2031 #define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to virtual timebase + frequency reg enabled */ 2032 #define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to physical timebase + frequency reg enabled */ 2033 2034 #define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */ 2035 #define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */ 2036 #define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */ 2037 2038 #define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS 2039 #define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED 2040 #define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE 2041 2042 #define MIDR_EL1_REV_SHIFT 0 2043 #define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT) 2044 #define MIDR_EL1_PNUM_SHIFT 4 2045 #define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT) 2046 #define MIDR_EL1_ARCH_SHIFT 16 2047 #define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT) 2048 #define MIDR_EL1_VAR_SHIFT 20 2049 #define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT) 2050 #define MIDR_EL1_IMP_SHIFT 24 2051 #define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT) 2052 2053 #define MIDR_FIJI (0x002 << MIDR_EL1_PNUM_SHIFT) 2054 #define MIDR_CAPRI (0x003 << MIDR_EL1_PNUM_SHIFT) 2055 #define MIDR_MAUI (0x004 << MIDR_EL1_PNUM_SHIFT) 2056 #define MIDR_ELBA (0x005 << MIDR_EL1_PNUM_SHIFT) 2057 #define MIDR_CAYMAN (0x006 << MIDR_EL1_PNUM_SHIFT) 2058 #define MIDR_MYST (0x007 << MIDR_EL1_PNUM_SHIFT) 2059 #define MIDR_SKYE_MONSOON (0x008 << MIDR_EL1_PNUM_SHIFT) 2060 #define MIDR_SKYE_MISTRAL (0x009 << MIDR_EL1_PNUM_SHIFT) 2061 #define MIDR_CYPRUS_VORTEX (0x00B << MIDR_EL1_PNUM_SHIFT) 2062 #define MIDR_CYPRUS_TEMPEST (0x00C << MIDR_EL1_PNUM_SHIFT) 2063 #define MIDR_M9 (0x00F << MIDR_EL1_PNUM_SHIFT) 2064 #define MIDR_ARUBA_VORTEX (0x010 << MIDR_EL1_PNUM_SHIFT) 2065 #define MIDR_ARUBA_TEMPEST (0x011 << MIDR_EL1_PNUM_SHIFT) 2066 2067 #ifdef APPLELIGHTNING 2068 #define MIDR_CEBU_LIGHTNING (0x012 << MIDR_EL1_PNUM_SHIFT) 2069 #define MIDR_CEBU_THUNDER (0x013 << MIDR_EL1_PNUM_SHIFT) 2070 #define MIDR_TURKS (0x026 << MIDR_EL1_PNUM_SHIFT) 2071 #endif 2072 2073 #ifdef APPLEFIRESTORM 2074 #define MIDR_SICILY_ICESTORM (0x020 << MIDR_EL1_PNUM_SHIFT) 2075 #define MIDR_SICILY_FIRESTORM (0x021 << MIDR_EL1_PNUM_SHIFT) 2076 #define MIDR_TONGA_ICESTORM (0x022 << MIDR_EL1_PNUM_SHIFT) 2077 #define MIDR_TONGA_FIRESTORM (0x023 << MIDR_EL1_PNUM_SHIFT) 2078 #define MIDR_JADE_CHOP_ICESTORM (0x024 << MIDR_EL1_PNUM_SHIFT) 2079 #define MIDR_JADE_CHOP_FIRESTORM (0x025 << MIDR_EL1_PNUM_SHIFT) 2080 #define MIDR_JADE_DIE_ICESTORM (0x028 << MIDR_EL1_PNUM_SHIFT) 2081 #define MIDR_JADE_DIE_FIRESTORM (0x029 << MIDR_EL1_PNUM_SHIFT) 2082 #endif 2083 2084 #ifdef APPLEAVALANCHE 2085 #define MIDR_ELLIS_BLIZZARD (0x030 << MIDR_EL1_PNUM_SHIFT) 2086 #define MIDR_ELLIS_AVALANCHE (0x031 << MIDR_EL1_PNUM_SHIFT) 2087 #endif 2088 2089 2090 2091 2092 /* 2093 * Apple-ISA-Extensions ID Register. 2094 */ 2095 #define AIDR_MUL53 (1 << 0) 2096 #define AIDR_WKDM (1 << 1) 2097 #define AIDR_ARCHRETENTION (1 << 2) 2098 2099 2100 2101 /* 2102 * CoreSight debug registers 2103 */ 2104 #define CORESIGHT_ED 0 2105 #define CORESIGHT_CTI 1 2106 #define CORESIGHT_PMU 2 2107 #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */ 2108 2109 #define CORESIGHT_OFFSET(x) ((x) * 0x10000) 2110 #define CORESIGHT_REGIONS 4 2111 #define CORESIGHT_SIZE 0x1000 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 /* 2123 * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0 2124 * 2125 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2126 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ 2127 * | rndr | tlb | ts | fhm | dp | sm4 | sm3 | sha3 | rdm | res0 | atomic | crc32 | sha2 | sha1 | aes | res0 | 2128 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ 2129 */ 2130 2131 #define ID_AA64ISAR0_EL1_TS_OFFSET 52 2132 #define ID_AA64ISAR0_EL1_TS_MASK (0xfull << ID_AA64ISAR0_EL1_TS_OFFSET) 2133 #define ID_AA64ISAR0_EL1_TS_FLAGM_EN (1ull << ID_AA64ISAR0_EL1_TS_OFFSET) 2134 #define ID_AA64ISAR0_EL1_TS_FLAGM2_EN (2ull << ID_AA64ISAR0_EL1_TS_OFFSET) 2135 2136 #define ID_AA64ISAR0_EL1_FHM_OFFSET 48 2137 #define ID_AA64ISAR0_EL1_FHM_MASK (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET) 2138 #define ID_AA64ISAR0_EL1_FHM_8_2 (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET) 2139 2140 #define ID_AA64ISAR0_EL1_DP_OFFSET 44 2141 #define ID_AA64ISAR0_EL1_DP_MASK (0xfull << ID_AA64ISAR0_EL1_DP_OFFSET) 2142 #define ID_AA64ISAR0_EL1_DP_EN (1ull << ID_AA64ISAR0_EL1_DP_OFFSET) 2143 2144 #define ID_AA64ISAR0_EL1_SHA3_OFFSET 32 2145 #define ID_AA64ISAR0_EL1_SHA3_MASK (0xfull << ID_AA64ISAR0_EL1_SHA3_OFFSET) 2146 #define ID_AA64ISAR0_EL1_SHA3_EN (1ull << ID_AA64ISAR0_EL1_SHA3_OFFSET) 2147 2148 #define ID_AA64ISAR0_EL1_RDM_OFFSET 28 2149 #define ID_AA64ISAR0_EL1_RDM_MASK (0xfull << ID_AA64ISAR0_EL1_RDM_OFFSET) 2150 #define ID_AA64ISAR0_EL1_RDM_EN (1ull << ID_AA64ISAR0_EL1_RDM_OFFSET) 2151 2152 #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20 2153 #define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) 2154 #define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) 2155 2156 #define ID_AA64ISAR0_EL1_CRC32_OFFSET 16 2157 #define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET) 2158 #define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET) 2159 2160 #define ID_AA64ISAR0_EL1_SHA2_OFFSET 12 2161 #define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 2162 #define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 2163 #define ID_AA64ISAR0_EL1_SHA2_512_EN (2ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 2164 2165 #define ID_AA64ISAR0_EL1_SHA1_OFFSET 8 2166 #define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET) 2167 #define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET) 2168 2169 #define ID_AA64ISAR0_EL1_AES_OFFSET 4 2170 #define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET) 2171 #define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET) 2172 #define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET) 2173 2174 /* 2175 * ID_AA64ISAR1_EL1 - AArch64 Instruction Set Attribute Register 1 2176 * 2177 * 63 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2178 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ 2179 * | res0 | i8mm | dgh | bf16 | specres | sb | frintts | gpi | gpa | lrcpc | fcma | jscvt | api | apa | dpb | 2180 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ 2181 */ 2182 2183 #define ID_AA64ISAR1_EL1_I8MM_OFFSET 52 2184 #define ID_AA64ISAR1_EL1_I8MM_MASK (0xfull << ID_AA64ISAR1_EL1_I8MM_OFFSET) 2185 #define ID_AA64ISAR1_EL1_I8MM_EN (1ull << ID_AA64ISAR1_EL1_I8MM_OFFSET) 2186 2187 #define ID_AA64ISAR1_EL1_DGH_OFFSET 48 2188 #define ID_AA64ISAR1_EL1_DGH_MASK (0xfull << ID_AA64ISAR1_EL1_DGH_OFFSET) 2189 2190 #define ID_AA64ISAR1_EL1_BF16_OFFSET 44 2191 #define ID_AA64ISAR1_EL1_BF16_MASK (0xfull << ID_AA64ISAR1_EL1_BF16_OFFSET) 2192 #define ID_AA64ISAR1_EL1_BF16_EN (1ull << ID_AA64ISAR1_EL1_BF16_OFFSET) 2193 2194 #define ID_AA64ISAR1_EL1_SPECRES_OFFSET 40 2195 #define ID_AA64ISAR1_EL1_SPECRES_MASK (0xfull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 2196 #define ID_AA64ISAR1_EL1_SPECRES_EN (1ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 2197 2198 #define ID_AA64ISAR1_EL1_SB_OFFSET 36 2199 #define ID_AA64ISAR1_EL1_SB_MASK (0xfull << ID_AA64ISAR1_EL1_SB_OFFSET) 2200 #define ID_AA64ISAR1_EL1_SB_EN (1ull << ID_AA64ISAR1_EL1_SB_OFFSET) 2201 2202 #define ID_AA64ISAR1_EL1_FRINTTS_OFFSET 32 2203 #define ID_AA64ISAR1_EL1_FRINTTS_MASK (0xfull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) 2204 #define ID_AA64ISAR1_EL1_FRINTTS_EN (1ull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) 2205 2206 #define ID_AA64ISAR1_EL1_GPI_OFFSET 28 2207 #define ID_AA64ISAR1_EL1_GPI_MASK (0xfull << ID_AA64ISAR1_EL1_GPI_OFFSET) 2208 #define ID_AA64ISAR1_EL1_GPI_EN (1ull << ID_AA64ISAR1_EL1_GPI_OFFSET) 2209 2210 #define ID_AA64ISAR1_EL1_GPA_OFFSET 24 2211 #define ID_AA64ISAR1_EL1_GPA_MASK (0xfull << ID_AA64ISAR1_EL1_GPA_OFFSET) 2212 2213 #define ID_AA64ISAR1_EL1_LRCPC_OFFSET 20 2214 #define ID_AA64ISAR1_EL1_LRCPC_MASK (0xfull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 2215 #define ID_AA64ISAR1_EL1_LRCPC_EN (1ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 2216 #define ID_AA64ISAR1_EL1_LRCP2C_EN (2ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 2217 2218 #define ID_AA64ISAR1_EL1_FCMA_OFFSET 16 2219 #define ID_AA64ISAR1_EL1_FCMA_MASK (0xfull << ID_AA64ISAR1_EL1_FCMA_OFFSET) 2220 #define ID_AA64ISAR1_EL1_FCMA_EN (1ull << ID_AA64ISAR1_EL1_FCMA_OFFSET) 2221 2222 #define ID_AA64ISAR1_EL1_JSCVT_OFFSET 12 2223 #define ID_AA64ISAR1_EL1_JSCVT_MASK (0xfull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) 2224 #define ID_AA64ISAR1_EL1_JSCVT_EN (1ull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) 2225 2226 #define ID_AA64ISAR1_EL1_API_OFFSET 8 2227 #define ID_AA64ISAR1_EL1_API_MASK (0xfull << ID_AA64ISAR1_EL1_API_OFFSET) 2228 #define ID_AA64ISAR1_EL1_API_PAuth_EN (1ull << ID_AA64ISAR1_EL1_API_OFFSET) 2229 #define ID_AA64ISAR1_EL1_API_PAuth2_EN (3ull << ID_AA64ISAR1_EL1_API_OFFSET) 2230 #define ID_AA64ISAR1_EL1_API_FPAC_EN (4ull << ID_AA64ISAR1_EL1_API_OFFSET) 2231 #define ID_AA64ISAR1_EL1_API_FPACCOMBINE (5ull << ID_AA64ISAR1_EL1_API_OFFSET) 2232 2233 #define ID_AA64ISAR1_EL1_APA_OFFSET 4 2234 #define ID_AA64ISAR1_EL1_APA_MASK (0xfull << ID_AA64ISAR1_EL1_APA_OFFSET) 2235 2236 #define ID_AA64ISAR1_EL1_DPB_OFFSET 0 2237 #define ID_AA64ISAR1_EL1_DPB_MASK (0xfull << ID_AA64ISAR1_EL1_DPB_OFFSET) 2238 #define ID_AA64ISAR1_EL1_DPB_EN (1ull << ID_AA64ISAR1_EL1_DPB_OFFSET) 2239 #define ID_AA64ISAR1_EL1_DPB2_EN (2ull << ID_AA64ISAR1_EL1_DPB_OFFSET) 2240 2241 /* 2242 * ID_AA64ISAR2_EL1 - AArch64 Instruction Set Attribute Register 2 2243 * 2244 * 63 8 7 4 3 0 2245 * +------+-------+------+ 2246 * | res0 | RPRES | WFxT | 2247 * +------+-------+------+ 2248 */ 2249 2250 #define ID_AA64ISAR2_EL1_RPRES_OFFSET 4 2251 #define ID_AA64ISAR2_EL1_RPRES_MASK (0xfull << ID_AA64ISAR2_EL1_RPRES_OFFSET) 2252 #define ID_AA64ISAR2_EL1_RPRES_EN (1ull << ID_AA64ISAR2_EL1_RPRES_OFFSET) 2253 2254 #define ID_AA64ISAR2_EL1_WFxT_OFFSET 0 2255 #define ID_AA64ISAR2_EL1_WFxT_MASK (0xfull << ID_AA64ISAR2_EL1_WFxT_OFFSET) 2256 #define ID_AA64ISAR2_EL1_WFxT_EN (1ull << ID_AA64ISAR2_EL1_WFxT_OFFSET) 2257 2258 /* 2259 * ID_AA64MMFR0_EL1 - AArch64 Memory Model Feature Register 0 2260 * 63 60 59 56 55 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2261 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ 2262 * | ECV | FGT | RES0 | ExS | TGran4_2 | TGran64_2 | TGran16_2 | TGran4 | TGran64 | TGran16 | BigEndEL0 | SNSMem | BigEnd | ASIDBits | PARange | 2263 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ 2264 */ 2265 2266 #define ID_AA64MMFR0_EL1_ECV_OFFSET 60 2267 #define ID_AA64MMFR0_EL1_ECV_MASK (0xfull << ID_AA64MMFR0_EL1_ECV_OFFSET) 2268 #define ID_AA64MMFR0_EL1_ECV_EN (1ull << ID_AA64MMFR0_EL1_ECV_OFFSET) 2269 2270 /* 2271 * ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2 2272 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 14 8 7 4 3 0 2273 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ 2274 * | E0PD | EVT | BBM | TTL | RES0 | FWB | IDS | AT | ST | NV | CCIDX | VARANGE | IESB | LSM | UAO | CnP | 2275 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ 2276 */ 2277 2278 #define ID_AA64MMFR2_EL1_AT_OFFSET 32 2279 #define ID_AA64MMFR2_EL1_AT_MASK (0xfull << ID_AA64MMFR2_EL1_AT_OFFSET) 2280 #define ID_AA64MMFR2_EL1_AT_LSE2_EN (1ull << ID_AA64MMFR2_EL1_AT_OFFSET) 2281 2282 /* 2283 * ID_AA64PFR0_EL1 - AArch64 Processor Feature Register 0 2284 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2285 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ 2286 * | CSV3 | CSV2 | RES0 | DIT | AMU | MPAM | SEL2 | SVE | RAS | GIC | AdvSIMD | FP | EL3 | EL2 | EL1 | EL0 | 2287 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ 2288 */ 2289 2290 #define ID_AA64PFR0_EL1_CSV3_OFFSET 60 2291 #define ID_AA64PFR0_EL1_CSV3_MASK (0xfull << ID_AA64PFR0_EL1_CSV3_OFFSET) 2292 #define ID_AA64PFR0_EL1_CSV3_EN (1ull << ID_AA64PFR0_EL1_CSV3_OFFSET) 2293 2294 #define ID_AA64PFR0_EL1_CSV2_OFFSET 56 2295 #define ID_AA64PFR0_EL1_CSV2_MASK (0xfull << ID_AA64PFR0_EL1_CSV2_OFFSET) 2296 #define ID_AA64PFR0_EL1_CSV2_EN (1ull << ID_AA64PFR0_EL1_CSV2_OFFSET) 2297 2298 #define ID_AA64PFR0_EL1_DIT_OFFSET 48 2299 #define ID_AA64PFR0_EL1_DIT_MASK (0xfull << ID_AA64PFR0_EL1_DIT_OFFSET) 2300 #define ID_AA64PFR0_EL1_DIT_EN (1ull << ID_AA64PFR0_EL1_DIT_OFFSET) 2301 2302 #define ID_AA64PFR0_EL1_AdvSIMD_OFFSET 20 2303 #define ID_AA64PFR0_EL1_AdvSIMD_MASK (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 2304 #define ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT (0x0ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 2305 #define ID_AA64PFR0_EL1_AdvSIMD_FP16 (0x1ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 2306 #define ID_AA64PFR0_EL1_AdvSIMD_DIS (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 2307 2308 /* 2309 * ID_AA64PFR1_EL1 - AArch64 Processor Feature Register 1 2310 * 63 20 19 16 15 12 11 8 7 4 3 0 2311 * +----------------------------------+-----------+----------+-------+------+------+ 2312 * | RES0 | MPAM_frac | RAS_frac | MTE | SSBS | BT | 2313 * +----------------------------------+-----------+----------+-------+------+------+ 2314 */ 2315 2316 2317 #define ID_AA64PFR1_EL1_SSBS_OFFSET 4 2318 #define ID_AA64PFR1_EL1_SSBS_MASK (0xfull << ID_AA64PFR1_EL1_SSBS_OFFSET) 2319 #define ID_AA64PFR1_EL1_SSBS_EN (1ull << ID_AA64PFR1_EL1_SSBS_OFFSET) 2320 2321 #define ID_AA64PFR1_EL1_BT_OFFSET 0 2322 #define ID_AA64PFR1_EL1_BT_MASK (0xfull << ID_AA64PFR1_EL1_BT_OFFSET) 2323 #define ID_AA64PFR1_EL1_BT_EN (1ull << ID_AA64PFR1_EL1_BT_OFFSET) 2324 2325 /* 2326 * ID_AA64MMFR1_EL1 - AArch64 Memory Model Feature Register 1 2327 * 2328 * 63 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2329 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ 2330 * | res0 | nTLBPA | AFP | HCX | ETS | TWED | XNX | SpecSEI | PAN | LO | HPDS | VH | VMIDBits | HAFDBS | 2331 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ 2332 */ 2333 2334 #define ID_AA64MMFR1_EL1_AFP_OFFSET 44 2335 #define ID_AA64MMFR1_EL1_AFP_MASK (0xfull << ID_AA64MMFR1_EL1_AFP_OFFSET) 2336 #define ID_AA64MMFR1_EL1_AFP_EN (1ull << ID_AA64MMFR1_EL1_AFP_OFFSET) 2337 2338 2339 2340 2341 2342 #define APSTATE_G_SHIFT (0) 2343 #define APSTATE_P_SHIFT (1) 2344 #define APSTATE_A_SHIFT (2) 2345 #define APSTATE_AP_MASK ((1ULL << APSTATE_A_SHIFT) | (1ULL << APSTATE_P_SHIFT)) 2346 2347 2348 #define ACTLR_EL1_EnTSO (1ULL << 1) 2349 #define ACTLR_EL1_EnAPFLG (1ULL << 4) 2350 #define ACTLR_EL1_EnAFP (1ULL << 5) 2351 #define ACTLR_EL1_EnPRSV (1ULL << 6) 2352 2353 2354 #if HAS_USAT_BIT 2355 #define ACTLR_EL1_USAT_OFFSET 0 2356 #define ACTLR_EL1_USAT_MASK (1ULL << ACTLR_EL1_USAT_OFFSET) 2357 #define ACTLR_EL1_USAT ACTLR_EL1_USAT_MASK 2358 #endif 2359 2360 2361 2362 2363 2364 2365 #if defined(HAS_APPLE_PAC) 2366 // The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code 2367 #define PAC_DISCRIMINATOR_RECOVER 0x1e02 2368 #endif 2369 2370 2371 #define CTR_EL0_L1Ip_OFFSET 14 2372 #define CTR_EL0_L1Ip_VIPT (2ULL << CTR_EL0_L1Ip_OFFSET) 2373 #define CTR_EL0_L1Ip_PIPT (3ULL << CTR_EL0_L1Ip_OFFSET) 2374 #define CTR_EL0_L1Ip_MASK (3ULL << CTR_EL0_L1Ip_OFFSET) 2375 2376 2377 #ifdef __ASSEMBLER__ 2378 2379 /* 2380 * Conditionally write to system/special-purpose register. 2381 * The register is written to only when the first two arguments 2382 * do not match. If they do match, the macro jumps to a 2383 * caller-provided label. 2384 * The _ISB variant also conditionally issues an ISB after the MSR. 2385 * 2386 * $0 - System/special-purpose register to modify 2387 * $1 - Register containing current FPCR value 2388 * $2 - Register containing expected value 2389 * $3 - Label to jump to when register is already set to expected value 2390 */ 2391 .macro CMSR 2392 cmp $1, $2 2393 2394 /* Skip expensive MSR if not required */ 2395 b.eq $3f 2396 msr $0, $2 2397 .endmacro 2398 2399 .macro CMSR_ISB 2400 CMSR $0, $1, $2, $3 2401 isb sy 2402 .endmacro 2403 2404 /* 2405 * Modify FPCR only if it does not contain the XNU default value. 2406 * $0 - Register containing current FPCR value 2407 * $1 - Scratch register 2408 * $2 - Label to jump to when FPCR is already set to default value 2409 */ 2410 .macro SANITIZE_FPCR 2411 mov $1, #FPCR_DEFAULT 2412 CMSR FPCR, $0, $1, $2 2413 .endmacro 2414 2415 /* 2416 * Family of macros that can be used to protect code sections such that they 2417 * are only executed on a particular SoC/Revision/CPU, and skipped otherwise. 2418 * All macros will forward-jump to 1f when the condition is not matched. 2419 * This label may be defined manually, or implicitly through the use of 2420 * the EXEC_END macro. 2421 * For cores, XX can be: EQ (equal), ALL (don't care). 2422 * For revisions, XX can be: EQ (equal), LO (lower than), HS (higher or same), ALL (don't care). 2423 */ 2424 2425 /* 2426 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX 2427 * $1 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2428 * $2 - GPR containing MIDR_EL1 value 2429 * $3 - Scratch register 2430 */ 2431 .macro EXEC_COREEQ_REVEQ 2432 and $3, $2, #MIDR_EL1_PNUM_MASK 2433 cmp $3, $0 2434 b.ne 1f 2435 2436 mov $3, $2 2437 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2438 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2439 cmp $3, $1 2440 b.ne 1f 2441 .endmacro 2442 2443 .macro EXEC_COREEQ_REVLO 2444 and $3, $2, #MIDR_EL1_PNUM_MASK 2445 cmp $3, $0 2446 b.ne 1f 2447 2448 mov $3, $2 2449 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2450 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2451 cmp $3, $1 2452 b.pl 1f 2453 .endmacro 2454 2455 .macro EXEC_COREEQ_REVHS 2456 and $3, $2, #MIDR_EL1_PNUM_MASK 2457 cmp $3, $0 2458 b.ne 1f 2459 2460 mov $3, $2 2461 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2462 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2463 cmp $3, $1 2464 b.mi 1f 2465 .endmacro 2466 2467 /* 2468 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2469 * $1 - GPR containing MIDR_EL1 value 2470 * $2 - Scratch register 2471 */ 2472 .macro EXEC_COREALL_REVEQ 2473 mov $2, $1 2474 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2475 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2476 cmp $2, $0 2477 b.ne 1f 2478 .endmacro 2479 2480 .macro EXEC_COREALL_REVLO 2481 mov $2, $1 2482 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2483 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2484 cmp $2, $0 2485 b.pl 1f 2486 .endmacro 2487 2488 .macro EXEC_COREALL_REVHS 2489 mov $2, $1 2490 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2491 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2492 cmp $2, $0 2493 b.mi 1f 2494 .endmacro 2495 2496 .macro CMP_FOREACH reg, cc, label, car, cdr:vararg 2497 cmp \reg, \car 2498 b.\cc \label 2499 .ifnb \cdr 2500 CMP_FOREACH \reg, \cc, \label, \cdr 2501 .endif 2502 .endm 2503 2504 .macro EXEC_COREIN_REVALL midr_el1, scratch, midr_list:vararg 2505 and \scratch, \midr_el1, #MIDR_EL1_PNUM_MASK 2506 CMP_FOREACH \scratch, eq, Lmatch\@, \midr_list 2507 b 1f 2508 Lmatch\@: 2509 .endm 2510 2511 /* 2512 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX 2513 * $1 - GPR containing MIDR_EL1 value 2514 * $2 - Scratch register 2515 */ 2516 .macro EXEC_COREEQ_REVALL 2517 and $2, $1, #MIDR_EL1_PNUM_MASK 2518 cmp $2, $0 2519 b.ne 1f 2520 .endmacro 2521 2522 /* 2523 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2524 * $1 - GPR containing MIDR_EL1 value 2525 * $2 - Scratch register 2526 */ 2527 .macro EXEC_PCORE_REVEQ 2528 mrs $2, MPIDR_EL1 2529 and $2, $2, #(MPIDR_PNE) 2530 cmp $2, xzr 2531 b.eq 1f 2532 2533 mov $2, $1 2534 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2535 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2536 cmp $2, $0 2537 b.ne 1f 2538 .endmacro 2539 2540 .macro EXEC_PCORE_REVLO 2541 mrs $2, MPIDR_EL1 2542 and $2, $2, #(MPIDR_PNE) 2543 cmp $2, xzr 2544 b.eq 1f 2545 2546 mov $2, $1 2547 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2548 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2549 cmp $2, $0 2550 b.pl 1f 2551 .endmacro 2552 2553 .macro EXEC_PCORE_REVHS 2554 mrs $2, MPIDR_EL1 2555 and $2, $2, #(MPIDR_PNE) 2556 cmp $2, xzr 2557 b.eq 1f 2558 2559 mov $2, $1 2560 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2561 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2562 cmp $2, $0 2563 b.mi 1f 2564 .endmacro 2565 2566 .macro EXEC_ECORE_REVEQ 2567 mrs $2, MPIDR_EL1 2568 and $2, $2, #(MPIDR_PNE) 2569 cmp $2, xzr 2570 b.ne 1f 2571 2572 mov $2, $1 2573 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2574 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2575 cmp $2, $0 2576 b.ne 1f 2577 .endmacro 2578 2579 .macro EXEC_ECORE_REVLO 2580 mrs $2, MPIDR_EL1 2581 and $2, $2, #(MPIDR_PNE) 2582 cmp $2, xzr 2583 b.ne 1f 2584 2585 mov $2, $1 2586 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2587 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2588 cmp $2, $0 2589 b.pl 1f 2590 .endmacro 2591 2592 .macro EXEC_ECORE_REVHS 2593 mrs $2, MPIDR_EL1 2594 and $2, $2, #(MPIDR_PNE) 2595 cmp $2, xzr 2596 b.ne 1f 2597 2598 mov $2, $1 2599 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2600 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2601 cmp $2, $0 2602 b.mi 1f 2603 .endmacro 2604 2605 /* 2606 * $0 - GPR containing MIDR_EL1 value 2607 * $1 - Scratch register 2608 */ 2609 .macro EXEC_PCORE_REVALL 2610 mrs $1, MPIDR_EL1 2611 and $1, $1, #(MPIDR_PNE) 2612 cmp $1, xzr 2613 b.eq 1f 2614 .endmacro 2615 2616 .macro EXEC_ECORE_REVALL 2617 mrs $1, MPIDR_EL1 2618 and $1, $1, #(MPIDR_PNE) 2619 cmp $1, xzr 2620 b.ne 1f 2621 .endmacro 2622 2623 2624 2625 /* 2626 * Macro that defines the label that all EXEC_COREXX_REVXX macros jump to. 2627 */ 2628 .macro EXEC_END 2629 1: 2630 .endmacro 2631 2632 /* 2633 * Wedges CPUs with a specified core that are below a specified revision. This 2634 * macro is intended for CPUs that have been deprecated in iBoot and may have 2635 * incorrect behavior if they continue running xnu. 2636 */ 2637 .macro DEPRECATE_COREEQ_REVLO core, rev, midr_el1, scratch 2638 EXEC_COREEQ_REVLO \core, \rev, \midr_el1, \scratch 2639 /* BEGIN IGNORE CODESTYLE */ 2640 b . 2641 /* END IGNORE CODESTYLE */ 2642 EXEC_END 2643 .endmacro 2644 2645 /* 2646 * Sets bits in an SPR register. 2647 * arg0: Name of the register to be accessed. 2648 * arg1: Mask of bits to be set. 2649 * arg2: Scratch register 2650 */ 2651 .macro HID_SET_BITS 2652 mrs $2, $0 2653 orr $2, $2, $1 2654 msr $0, $2 2655 .endmacro 2656 2657 /* 2658 * Clears bits in an SPR register. 2659 * arg0: Name of the register to be accessed. 2660 * arg1: Mask of bits to be cleared. 2661 * arg2: Scratch register 2662 */ 2663 .macro HID_CLEAR_BITS 2664 mrs $2, $0 2665 bic $2, $2, $1 2666 msr $0, $2 2667 .endmacro 2668 2669 /* 2670 * Combines the functionality of HID_CLEAR_BITS followed by HID_SET_BITS into 2671 * a single read-modify-write sequence. 2672 * arg0: Name of the register to be accessed. 2673 * arg1: Mask of bits to be cleared. 2674 * arg2: Value to insert 2675 * arg3: Scratch register 2676 */ 2677 .macro HID_INSERT_BITS 2678 mrs $3, $0 2679 bic $3, $3, $1 2680 orr $3, $3, $2 2681 msr $0, $3 2682 .endmacro 2683 2684 /* 2685 * Replaces the value of a field in an implementation-defined system register. 2686 * sreg: system register name 2687 * field: field name within the sysreg, where the assembler symbols 2688 * ARM64_REG_<field>_{shift,width} specify the bounds of the field 2689 * (note that preprocessor macros will not work here) 2690 * value: the value to insert 2691 * scr{1,2}: scratch regs 2692 */ 2693 .macro HID_WRITE_FIELD sreg, field, val, scr1, scr2 2694 mrs \scr1, \sreg 2695 mov \scr2, \val 2696 bfi \scr1, \scr2, ARM64_REG_\sreg\()_\field\()_shift, ARM64_REG_\sreg\()_\field\()_width 2697 msr \sreg, \scr1 2698 .endmacro 2699 2700 /* 2701 * Macro intended to be used as a replacement for ERET. 2702 * It prevents speculation past ERET instructions by padding 2703 * up to the decoder width. 2704 */ 2705 .macro ERET_CONTEXT_SYNCHRONIZING 2706 eret 2707 #if __ARM_SB_AVAILABLE__ 2708 sb // Technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures 2709 #else /* __ARM_SB_AVAILABLE__ */ 2710 isb // ISB technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures 2711 nop // Sequence of six NOPs to pad out and terminate instruction decode group */ 2712 nop 2713 nop 2714 nop 2715 nop 2716 nop 2717 #endif /* !__ARM_SB_AVAILABLE__ */ 2718 .endmacro 2719 2720 2721 #endif /* __ASSEMBLER__ */ 2722 2723 #define MSR(reg, src) __asm__ volatile ("msr " reg ", %0" :: "r" (src)) 2724 #define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest)) 2725 2726 #if XNU_MONITOR 2727 #define __ARM_PTE_PHYSMAP__ 1 2728 #define PPL_STATE_KERNEL 0 2729 #define PPL_STATE_DISPATCH 1 2730 #define PPL_STATE_PANIC 2 2731 #define PPL_STATE_EXCEPTION 3 2732 #endif 2733 2734 2735 #endif /* _ARM64_PROC_REG_H_ */ 2736