1 /* 2 * Copyright (c) 2007-2024 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 /* CMU_ENDHIST */ 32 /* 33 * Mach Operating System 34 * Copyright (c) 1991,1990 Carnegie Mellon University 35 * All Rights Reserved. 36 * 37 * Permission to use, copy, modify and distribute this software and its 38 * documentation is hereby granted, provided that both the copyright 39 * notice and this permission notice appear in all copies of the 40 * software, derivative works or modified versions, and any portions 41 * thereof, and that both notices appear in supporting documentation. 42 * 43 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 44 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 45 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 46 * 47 * Carnegie Mellon requests users of this software to return to 48 * 49 * Software Distribution Coordinator or [email protected] 50 * School of Computer Science 51 * Carnegie Mellon University 52 * Pittsburgh PA 15213-3890 53 * 54 * any improvements or extensions that they make and grant Carnegie Mellon 55 * the rights to redistribute these changes. 56 */ 57 58 /* 59 * Processor registers for ARM/ARM64 60 */ 61 #ifndef _ARM64_PROC_REG_H_ 62 #define _ARM64_PROC_REG_H_ 63 64 #if !defined(KERNEL_PRIVATE) && !defined(SPTM_TESTING_PRIVATE) 65 /** 66 * This file is only exported into the internal userspace SDK exclusively for 67 * usage by the SPTM userspace testing system. Let's enforce this by error'ing 68 * the build if an SPTM-specific define is not set. If your userspace project is 69 * not the SPTM testing system, then do not use these files! 70 * 71 * This check does not apply to the kernel itself, or when this file is exported 72 * into Kernel.framework. 73 */ 74 #error This file is only included in the userspace internal SDK for the SPTM project 75 #endif /* !defined(KERNEL_PRIVATE) && !defined(SPTM_TESTING_PRIVATE) */ 76 77 #if defined (__arm64__) 78 #include <pexpert/arm64/board_config.h> 79 #elif defined (__arm__) 80 #include <pexpert/arm/board_config.h> 81 #endif 82 83 /* 84 * Processor registers for ARM 85 */ 86 #if __ARM_42BIT_PA_SPACE__ 87 /* For now, force the issue! */ 88 /* We need more VA space for the identity map to bootstrap the MMU */ 89 #undef __ARM64_PMAP_SUBPAGE_L1__ 90 #endif /* __ARM_42BIT_PA_SPACE__ */ 91 92 /* For arm platforms, create one pset per cluster */ 93 #define MAX_PSETS MAX_CPU_CLUSTERS 94 95 96 /* Thread groups are enabled on all ARM platforms (irrespective of scheduler) */ 97 #define CONFIG_THREAD_GROUPS 1 98 99 #ifdef XNU_KERNEL_PRIVATE 100 101 #if __ARM_VFP__ 102 #define ARM_VFP_DEBUG 0 103 #endif /* __ARM_VFP__ */ 104 105 #endif /* XNU_KERNEL_PRIVATE */ 106 107 /* 108 * FSR registers 109 * 110 * CPSR: Current Program Status Register 111 * SPSR: Saved Program Status Registers 112 * 113 * 31 30 29 28 27 24 19 16 9 8 7 6 5 4 0 114 * +-----------------------------------------------------------+ 115 * | N| Z| C| V| Q|...| J|...|GE[3:0]|...| E| A| I| F| T| MODE | 116 * +-----------------------------------------------------------+ 117 */ 118 119 /* 120 * Flags 121 */ 122 #define PSR_NF 0x80000000 /* Negative/Less than */ 123 #define PSR_ZF 0x40000000 /* Zero */ 124 #define PSR_CF 0x20000000 /* Carry/Borrow/Extend */ 125 #define PSR_VF 0x10000000 /* Overflow */ 126 127 /* 128 * Modified execution mode flags 129 */ 130 #define PSR_TF 0x00000020 /* thumb flag (BX ARMv4T) */ 131 132 /* 133 * CPU mode 134 */ 135 #define PSR_USER_MODE 0x00000010 /* User mode */ 136 137 #define PSR_MODE_MASK 0x0000001F 138 #define PSR_IS_KERNEL(psr) (((psr) & PSR_MODE_MASK) != PSR_USER_MODE) 139 #define PSR_IS_USER(psr) (((psr) & PSR_MODE_MASK) == PSR_USER_MODE) 140 141 #define PSR_USERDFLT PSR_USER_MODE 142 143 #define PSR_BTYPE_SHIFT (10) 144 #define PSR_BTYPE_MASK (0x3 << PSR_BTYPE_SHIFT) 145 146 /* 147 * Cache configuration 148 */ 149 150 #if defined (APPLETYPHOON) 151 152 /* I-Cache */ 153 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 154 155 /* D-Cache */ 156 #define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */ 157 158 #elif defined (APPLETWISTER) 159 160 /* I-Cache */ 161 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 162 163 /* D-Cache */ 164 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 165 166 #elif defined (APPLEHURRICANE) 167 168 /* I-Cache */ 169 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 170 171 /* D-Cache */ 172 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 173 174 #elif defined (APPLEMONSOON) 175 176 /* I-Cache, 96KB for Monsoon, 48KB for Mistral, 6-way. */ 177 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 178 179 /* D-Cache, 64KB for Monsoon, 32KB for Mistral, 4-way. */ 180 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 181 182 #elif defined (APPLEVORTEX) 183 184 /* I-Cache, 128KB 8-way for Vortex, 48KB 6-way for Tempest. */ 185 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 186 187 /* D-Cache, 128KB 8-way for Vortex, 32KB 4-way for Tempest. */ 188 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 189 190 #elif defined (APPLELIGHTNING) 191 192 /* I-Cache, 192KB for Lightning, 96KB for Thunder, 6-way. */ 193 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 194 195 /* D-Cache, 128KB for Lightning, 8-way. 48KB for Thunder, 6-way. */ 196 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 197 198 #elif defined (APPLEFIRESTORM) 199 200 /* I-Cache, 256KB for Firestorm, 128KB for Icestorm, 6-way. */ 201 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 202 203 /* D-Cache, 160KB for Firestorm, 8-way. 64KB for Icestorm, 6-way. */ 204 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 205 206 #elif defined (APPLEAVALANCHE) 207 208 /* I-Cache, 192KB for Avalanche, 128KB for Blizzard, 6-way. */ 209 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 210 211 /* D-Cache, 128KB for Avalanche, 8-way. 64KB for Blizzard, 8-way. */ 212 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 213 214 #elif defined (APPLEEVEREST) 215 216 /* I-Cache, 192KB for Everest, 128KB for SawTooth, 6-way. */ 217 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 218 219 /* D-Cache, 128KB for Everest, 8-way. 64KB for SawTooth, 8-way. */ 220 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 221 222 #elif defined (APPLEH16) 223 224 /* I-Cache, 192KB for AppleH16 PCore, 128KB for ECore, 6-way. */ 225 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 226 227 /* D-Cache, 128KB for AppleH16 PCore, 8-way. 64KB for ECore, 8-way. */ 228 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 229 230 #elif defined (VMAPPLE) 231 232 /* I-Cache. */ 233 #define MMU_I_CLINE 6 234 235 /* D-Cache. */ 236 #define MMU_CLINE 6 237 238 #else 239 #error processor not supported 240 #endif 241 242 #define MAX_L2_CLINE_BYTES (1 << MAX_L2_CLINE) 243 244 /* 245 * Format of the Debug & Watchpoint Breakpoint Value and Control Registers 246 */ 247 #define ARM_DBG_VR_ADDRESS_MASK 0xFFFFFFFC /* BVR & WVR */ 248 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */ 249 250 #define ARM_DBG_CR_ADDRESS_MASK_MASK 0x1F000000 /* BCR & WCR */ 251 #define ARM_DBGBCR_MATCH_MASK (1 << 22) /* BCR only */ 252 #define ARM_DBGBCR_TYPE_MASK (1 << 21) /* BCR only */ 253 #define ARM_DBGBCR_TYPE_IVA (0 << 21) 254 #define ARM_DBG_CR_LINKED_MASK (1 << 20) /* BCR & WCR */ 255 #define ARM_DBG_CR_LINKED_UNLINKED (0 << 20) 256 #define ARM_DBG_CR_SECURITY_STATE_BOTH (0 << 14) 257 #define ARM_DBG_CR_HIGHER_MODE_ENABLE (1 << 13) 258 #define ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK 0x00001FE0 /* WCR only */ 259 #define ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK 0x000001E0 /* BCR & WCR */ 260 #define ARM_DBGWCR_ACCESS_CONTROL_MASK (3 << 3) /* WCR only */ 261 #define ARM_DBG_CR_MODE_CONTROL_PRIVILEGED (1 << 1) /* BCR & WCR */ 262 #define ARM_DBG_CR_MODE_CONTROL_USER (2 << 1) /* BCR & WCR */ 263 #define ARM_DBG_CR_ENABLE_MASK (1 << 0) /* BCR & WCR */ 264 #define ARM_DBG_CR_ENABLE_ENABLE (1 << 0) 265 266 /* 267 * Format of the OS Lock Access (DBGOSLAR) and Lock Access Registers (DBGLAR) 268 */ 269 #define ARM_DBG_LOCK_ACCESS_KEY 0xC5ACCE55 270 271 /* ARM Debug registers of interest */ 272 #define ARM_DEBUG_OFFSET_DBGPRCR (0x310) 273 #define ARM_DEBUG_OFFSET_DBGLAR (0xFB0) 274 275 /* 276 * Main ID Register (MIDR) 277 * 278 * 31 24 23 20 19 16 15 4 3 0 279 * +-----+-----+------+------+-----+ 280 * | IMP | VAR | ARCH | PNUM | REV | 281 * +-----+-----+------+------+-----+ 282 * 283 * where: 284 * IMP: Implementor code 285 * VAR: Variant number 286 * ARCH: Architecture code 287 * PNUM: Primary part number 288 * REV: Minor revision number 289 */ 290 #define MIDR_REV_SHIFT 0 291 #define MIDR_REV_MASK (0xf << MIDR_REV_SHIFT) 292 #define MIDR_VAR_SHIFT 20 293 #define MIDR_VAR_MASK (0xf << MIDR_VAR_SHIFT) 294 295 296 #if __ARM_KERNEL_PROTECT__ 297 /* 298 * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential 299 * architectural or microarchitectural vulnerabilities that could allow cores to 300 * read/access EL1-only mappings while in EL0 mode. This is achieved by 301 * removing as many mappings as possible when the core transitions to EL0 mode 302 * from EL1 mode, and restoring those mappings when the core transitions to EL1 303 * mode from EL0 mode. 304 * 305 * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is 306 * used to map and unmap the ordinary kernel mappings, by contracting and 307 * expanding translation zone size for TTBR1 when exiting and entering EL1, 308 * respectively: 309 * 310 * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is 311 * is in EL0. 312 * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in 313 * EL1. 314 * 315 * T1SZ_USER: T1SZ_BOOT + 1 316 * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT) 317 * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER) 318 * TTBR1_EL1_MAX: (2^64) - 1 319 * 320 * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the 321 * the following TTBR1 layout: 322 * 323 * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX 324 * +---------------------------------------------------------+ 325 * | Kernel EL0 Mappings | Kernel EL1 Mappings | 326 * +---------------------------------------------------------+ 327 * 328 * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout: 329 * 330 * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX 331 * +---------------------------------------------------------+ 332 * | Kernel EL0 Mappings | 333 * +---------------------------------------------------------+ 334 * 335 * With the current implementation, both the EL0 and EL1 mappings for the kernel 336 * use otherwise empty translation tables for mapping the exception vectors (so 337 * that we do not need to TLB flush the exception vector address when switching 338 * between EL0 and EL1). The rationale here is that the TLBI would require a 339 * DSB, and DSBs can be extremely expensive. 340 * 341 * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1 342 * ASID. The core switches between ASIDs on EL transitions, so that the TLB 343 * does not need to be fully invalidated on an EL transition. 344 * 345 * Most kernel mappings will be marked non-global in this configuration, as 346 * global mappings would be visible to userspace unless we invalidate them on 347 * eret. 348 */ 349 #if XNU_MONITOR 350 /* 351 * Please note that because we indirect through the thread register in order to 352 * locate the kernel, and because we unmap most of the kernel, the security 353 * model of the PPL is undermined by __ARM_KERNEL_PROTECT__, as we rely on 354 * kernel controlled data to direct codeflow in the exception vectors. 355 * 356 * If we want to ship XNU_MONITOR paired with __ARM_KERNEL_PROTECT__, we will 357 * need to find a performant solution to this problem. 358 */ 359 #endif 360 #endif /* __ARM_KERNEL_PROTECT */ 361 362 #if ARM_PARAMETERIZED_PMAP 363 /* 364 * ARM_PARAMETERIZED_PMAP configures the kernel to get the characteristics of 365 * the page tables (number of levels, size of the root allocation) from the 366 * pmap data structure, rather than treating them as compile-time constants. 367 * This allows the pmap code to dynamically adjust how it deals with page 368 * tables. 369 */ 370 #endif /* ARM_PARAMETERIZED_PMAP */ 371 372 #if __ARM_MIXED_PAGE_SIZE__ 373 /* 374 * __ARM_MIXED_PAGE_SIZE__ configures the kernel to support page tables that do 375 * not use the kernel page size. This is primarily meant to support running 376 * 4KB page processes on a 16KB page kernel. 377 * 378 * This only covers support in the pmap/machine dependent layers. Any support 379 * elsewhere in the kernel must be managed separately. 380 */ 381 #if !ARM_PARAMETERIZED_PMAP 382 /* 383 * Page tables that use non-kernel page sizes require us to reprogram TCR based 384 * on the page tables we are switching to. This means that the parameterized 385 * pmap support is required. 386 */ 387 #error __ARM_MIXED_PAGE_SIZE__ requires ARM_PARAMETERIZED_PMAP 388 #endif /* !ARM_PARAMETERIZED_PMAP */ 389 #if __ARM_KERNEL_PROTECT__ 390 /* 391 * Because switching the page size requires updating TCR based on the pmap, and 392 * __ARM_KERNEL_PROTECT__ relies on TCR being programmed with constants, XNU 393 * does not currently support support configurations that use both 394 * __ARM_KERNEL_PROTECT__ and __ARM_MIXED_PAGE_SIZE__. 395 */ 396 #error __ARM_MIXED_PAGE_SIZE__ and __ARM_KERNEL_PROTECT__ are mutually exclusive 397 #endif /* __ARM_KERNEL_PROTECT__ */ 398 #endif /* __ARM_MIXED_PAGE_SIZE__ */ 399 400 /* 401 * 64-bit Program Status Register (PSR64) 402 * 403 * 31 27 23 22 21 20 19 10 9 5 4 0 404 * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+ 405 * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M | 406 * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+ 407 * 408 * where: 409 * NZCV: Comparison flags 410 * PAN: Privileged Access Never 411 * SS: Single step 412 * IL: Illegal state 413 * DAIF: Interrupt masks 414 * M: Mode field 415 */ 416 417 #define PSR64_NZCV_SHIFT 28 418 #define PSR64_NZCV_WIDTH 4 419 #define PSR64_NZCV_MASK (0xF << PSR64_NZCV_SHIFT) 420 421 #define PSR64_N_SHIFT 31 422 #define PSR64_N (1 << PSR64_N_SHIFT) 423 424 #define PSR64_Z_SHIFT 30 425 #define PSR64_Z (1 << PSR64_Z_SHIFT) 426 427 #define PSR64_C_SHIFT 29 428 #define PSR64_C (1 << PSR64_C_SHIFT) 429 430 #define PSR64_V_SHIFT 28 431 #define PSR64_V (1 << PSR64_V_SHIFT) 432 433 #define PSR64_TCO_SHIFT 25 434 #define PSR64_TCO (1 << PSR64_TCO_SHIFT) 435 436 #define PSR64_DIT_SHIFT 24 437 #define PSR64_DIT (1 << PSR64_DIT_SHIFT) 438 439 #define PSR64_UAO_SHIFT 23 440 #define PSR64_UAO (1 << PSR64_UAO_SHIFT) 441 442 #define PSR64_PAN_SHIFT 22 443 #define PSR64_PAN (1 << PSR64_PAN_SHIFT) 444 445 #define PSR64_SS_SHIFT 21 446 #define PSR64_SS (1 << PSR64_SS_SHIFT) 447 448 #define PSR64_IL_SHIFT 20 449 #define PSR64_IL (1 << PSR64_IL_SHIFT) 450 451 /* 452 * SSBS is bit 12 for A64 SPSR and bit 23 for A32 SPSR 453 * I do not want to talk about it! 454 */ 455 #define PSR64_SSBS_SHIFT_32 23 456 #define PSR64_SSBS_SHIFT_64 12 457 #define PSR64_SSBS_32 (1 << PSR64_SSBS_SHIFT_32) 458 #define PSR64_SSBS_64 (1 << PSR64_SSBS_SHIFT_64) 459 460 /* 461 * msr DAIF, Xn and mrs Xn, DAIF transfer into 462 * and out of bits 9:6 463 */ 464 #define DAIF_DEBUG_SHIFT 9 465 #define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT) 466 467 #define DAIF_ASYNC_SHIFT 8 468 #define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT) 469 470 #define DAIF_IRQF_SHIFT 7 471 #define DAIF_IRQF (1 << DAIF_IRQF_SHIFT) 472 473 #define DAIF_FIQF_SHIFT 6 474 #define DAIF_FIQF (1 << DAIF_FIQF_SHIFT) 475 476 #define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) 477 #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) 478 479 #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF)) 480 481 #if HAS_ARM_FEAT_SSBS2 482 #define PSR64_SSBS_U32_DEFAULT PSR64_SSBS_32 483 #define PSR64_SSBS_U64_DEFAULT PSR64_SSBS_64 484 #define PSR64_SSBS_KRN_DEFAULT PSR64_SSBS_64 485 #else 486 #define PSR64_SSBS_U32_DEFAULT (0) 487 #define PSR64_SSBS_U64_DEFAULT (0) 488 #define PSR64_SSBS_KRN_DEFAULT (0) 489 #endif 490 491 /* 492 * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer 493 * from bits 3:0. 494 */ 495 #define DAIFSC_DEBUGF (1 << 3) 496 #define DAIFSC_ASYNCF (1 << 2) 497 #define DAIFSC_IRQF (1 << 1) 498 #define DAIFSC_FIQF (1 << 0) 499 #define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) 500 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) 501 #define DAIFSC_NOASYNC (DAIFSC_DEBUGF | DAIFSC_IRQF | DAIFSC_FIQF) 502 503 /* 504 * ARM64_TODO: unify with ARM? 505 */ 506 #define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */ 507 508 #define PSR64_MODE_MASK 0x1F 509 510 #define PSR64_USER_MASK PSR64_NZCV_MASK 511 512 #define PSR64_MODE_USER32_THUMB 0x20 513 514 #define PSR64_MODE_RW_SHIFT 4 515 #define PSR64_MODE_RW_64 0 516 #define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT) 517 518 #define PSR64_MODE_EL_SHIFT 2 519 #define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT) 520 #define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT) 521 #define PSR64_MODE_EL2 (0x2 << PSR64_MODE_EL_SHIFT) 522 #define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT) 523 #define PSR64_MODE_EL0 0 524 525 #define PSR64_MODE_EL_KERNEL (PSR64_MODE_EL1) 526 527 #define PSR64_MODE_SPX 0x1 528 #define PSR64_MODE_SP0 0 529 530 #define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U32_DEFAULT) 531 #define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U64_DEFAULT) 532 #define PSR64_KERNEL_STANDARD (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0 | PSR64_SSBS_KRN_DEFAULT) 533 #if __ARM_PAN_AVAILABLE__ 534 #define PSR64_KERNEL_DEFAULT (PSR64_KERNEL_STANDARD | PSR64_PAN) 535 #else 536 #define PSR64_KERNEL_DEFAULT PSR64_KERNEL_STANDARD 537 #endif 538 539 #define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0) 540 #define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0) 541 542 #define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32)) 543 #define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32)) 544 545 546 547 /* 548 * System Control Register (SCTLR) 549 */ 550 551 552 #if HAS_ARM_FEAT_SME 553 // 60 EnTP2 Enable TPIDR2_EL0 at EL0 554 #define SCTLR_OTHER (1ULL << 60) 555 #else 556 #define SCTLR_OTHER (0) 557 #endif 558 559 #define SCTLR_EPAN_ENABLED (1ULL << 57) 560 561 #define SCTLR_DSSBS (1ULL << 44) 562 563 564 #define SCTLR_EXTRA (0) 565 566 567 #define SCTLR_RESERVED ((3ULL << 28) | (1ULL << 20)) 568 #if defined(HAS_APPLE_PAC) 569 570 // 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled 571 #define SCTLR_PACIA_ENABLED_SHIFT 31 572 #define SCTLR_PACIA_ENABLED (1ULL << SCTLR_PACIA_ENABLED_SHIFT) 573 // 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled 574 #define SCTLR_PACIB_ENABLED (1ULL << 30) 575 // 29:28 RES1 11 576 // 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled 577 #define SCTLR_PACDA_ENABLED (1ULL << 27) 578 // 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled 579 #define SCTLR_PACDB_ENABLED (1ULL << 13) 580 581 #define SCTLR_PAC_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACIB_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED) 582 #endif /* defined(HAS_APPLE_PAC) */ 583 584 // 36 BT1 PACIxSP acts as a BTI C landing pad rather than BTI JC at EL1 585 #define SCTLR_BT1_ENABLED (1ULL << 36) 586 587 // 35 BT0 PACIxSP acts as a BTI C landing pad rather than BTI JC at EL0 588 #define SCTLR_BT0_ENABLED (1ULL << 35) 589 590 // 26 UCI User Cache Instructions 591 #define SCTLR_UCI_ENABLED (1ULL << 26) 592 593 // 25 EE Exception Endianness 594 #define SCTLR_EE_BIG_ENDIAN (1ULL << 25) 595 596 // 24 E0E EL0 Endianness 597 #define SCTLR_E0E_BIG_ENDIAN (1ULL << 24) 598 599 // 23 SPAN Set PAN 600 #define SCTLR_PAN_UNCHANGED (1ULL << 23) 601 602 // 22 EIS Taking an exception is a context synchronization event 603 #define SCTLR_EIS (1ULL << 22) 604 605 // 21 RES0 0 606 // 20 RES1 1 607 608 // 19 WXN Writeable implies eXecute Never 609 #define SCTLR_WXN_ENABLED (1ULL << 19) 610 611 // 18 nTWE Not trap WFE from EL0 612 #define SCTLR_nTWE_WFE_ENABLED (1ULL << 18) 613 614 // 17 RES0 0 615 616 // 16 nTWI Not trap WFI from EL0 617 #define SCTRL_nTWI_WFI_ENABLED (1ULL << 16) 618 619 // 15 UCT User Cache Type register (CTR_EL0) 620 #define SCTLR_UCT_ENABLED (1ULL << 15) 621 622 // 14 DZE User Data Cache Zero (DC ZVA) 623 #define SCTLR_DZE_ENABLED (1ULL << 14) 624 625 // 12 I Instruction cache enable 626 #define SCTLR_I_ENABLED (1ULL << 12) 627 628 // 11 EOS Exception return is a context synchronization event 629 #define SCTLR_EOS (1ULL << 11) 630 631 // 10 EnRCTX EL0 Access to FEAT_SPECRES speculation restriction instructions 632 #define SCTLR_EnRCTX (1ULL << 10) 633 634 // 9 UMA User Mask Access 635 #define SCTLR_UMA_ENABLED (1ULL << 9) 636 637 // 8 SED SETEND Disable 638 #define SCTLR_SED_DISABLED (1ULL << 8) 639 640 // 7 ITD IT Disable 641 #define SCTLR_ITD_DISABLED (1ULL << 7) 642 643 // 6 RES0 0 644 645 // 5 CP15BEN CP15 Barrier ENable 646 #define SCTLR_CP15BEN_ENABLED (1ULL << 5) 647 648 // 4 SA0 Stack Alignment check for EL0 649 #define SCTLR_SA0_ENABLED (1ULL << 4) 650 651 // 3 SA Stack Alignment check 652 #define SCTLR_SA_ENABLED (1ULL << 3) 653 654 // 2 C Cache enable 655 #define SCTLR_C_ENABLED (1ULL << 2) 656 657 // 1 A Alignment check 658 #define SCTLR_A_ENABLED (1ULL << 1) 659 660 // 0 M MMU enable 661 #define SCTLR_M_ENABLED (1ULL << 0) 662 663 #if APPLEVIRTUALPLATFORM 664 #define SCTLR_EPAN_DEFAULT 0 665 /* xnu tries to set SCTLR_EL1.EPAN = 1, but it may be RaZ/WI on some hosts */ 666 #define SCTLR_EPAN_OPTIONAL SCTLR_EPAN_ENABLED 667 #elif HAS_ARM_FEAT_PAN3 668 #define SCTLR_EPAN_DEFAULT SCTLR_EPAN_ENABLED 669 #define SCTLR_EPAN_OPTIONAL 0 670 #else 671 #define SCTLR_EPAN_DEFAULT 0 672 #define SCTLR_EPAN_OPTIONAL 0 673 #endif 674 675 #if __ARM_ARCH_8_5__ 676 #define SCTLR_EIS_DEFAULT (0) 677 #define SCTLR_DSSBS_DEFAULT SCTLR_DSSBS 678 #else 679 #define SCTLR_EIS_DEFAULT (SCTLR_EIS) 680 #define SCTLR_DSSBS_DEFAULT (0) 681 #endif 682 683 #if ERET_IS_NOT_CONTEXT_SYNCHRONIZING 684 #define SCTLR_EOS_DEFAULT (0) 685 #else 686 #define SCTLR_EOS_DEFAULT (SCTLR_EOS) 687 #endif 688 689 #if HAS_APPLE_PAC 690 #define SCTLR_PAC_KEYS_DEFAULT SCTLR_PAC_KEYS_ENABLED 691 #else /* !HAS_APPLE_PAC */ 692 #define SCTLR_PAC_KEYS_DEFAULT 0 693 #endif 694 695 #if BTI_ENFORCED 696 /* In the kernel, we want PACIxSP to behave only as a BTI C */ 697 #define SCTLR_BT_DEFAULT SCTLR_BT1_ENABLED 698 #else 699 #define SCTLR_BT_DEFAULT 0 700 #endif /* BTI_ENFORCED */ 701 702 #define SCTLR_EL1_REQUIRED \ 703 (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \ 704 SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | SCTLR_BT_DEFAULT | \ 705 SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED | \ 706 SCTLR_EPAN_DEFAULT | SCTLR_EIS_DEFAULT | SCTLR_EOS_DEFAULT | SCTLR_DSSBS_DEFAULT | \ 707 SCTLR_PAC_KEYS_DEFAULT | SCTLR_OTHER) 708 709 #define SCTLR_EL1_OPTIONAL \ 710 (SCTLR_EPAN_OPTIONAL) 711 712 #define SCTLR_EL1_DEFAULT \ 713 (SCTLR_EL1_REQUIRED | SCTLR_EL1_OPTIONAL) 714 715 /* 716 * Coprocessor Access Control Register (CPACR) 717 * 718 * 31 28 27 22 21 20 19 0 719 * +---+---+------+------+--------------------+ 720 * |000|TTA|000000| FPEN |00000000000000000000| 721 * +---+---+------+------+--------------------+ 722 * 723 * where: 724 * TTA: Trace trap 725 * FPEN: Floating point enable 726 */ 727 #define CPACR_TTA_SHIFT 28 728 #define CPACR_TTA (1 << CPACR_TTA_SHIFT) 729 730 #if HAS_ARM_FEAT_SME 731 #define CPACR_SMEN_SHIFT 24 732 #define CPACR_SMEN_MASK (0x3 << CPACR_SMEN_SHIFT) 733 #define CPACR_SMEN_EL0_TRAP (0x1 << CPACR_SMEN_SHIFT) 734 #define CPACR_SMEN_ENABLE (0x3 << CPACR_SMEN_SHIFT) 735 #endif /* HAS_ARM_FEAT_SME */ 736 737 #define CPACR_FPEN_SHIFT 20 738 #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT) 739 #define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT) 740 741 #if HAS_ARM_FEAT_SME 742 #define CPACR_ZEN_SHIFT 16 743 #define CPACR_ZEN_MASK (0x3 << CPACR_ZEN_SHIFT) 744 #define CPACR_ZEN_EL0_TRAP (0x1 << CPACR_ZEN_SHIFT) 745 #define CPACR_ZEN_ENABLE (0x3 << CPACR_ZEN_SHIFT) 746 #endif /* HAS_ARM_FEAT_SME */ 747 748 /* 749 * FPSR: Floating Point Status Register 750 * 751 * 31 30 29 28 27 26 7 6 4 3 2 1 0 752 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ 753 * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC| 754 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ 755 */ 756 757 #define FPSR_N_SHIFT 31 758 #define FPSR_Z_SHIFT 30 759 #define FPSR_C_SHIFT 29 760 #define FPSR_V_SHIFT 28 761 #define FPSR_QC_SHIFT 27 762 #define FPSR_IDC_SHIFT 7 763 #define FPSR_IXC_SHIFT 4 764 #define FPSR_UFC_SHIFT 3 765 #define FPSR_OFC_SHIFT 2 766 #define FPSR_DZC_SHIFT 1 767 #define FPSR_IOC_SHIFT 0 768 #define FPSR_N (1 << FPSR_N_SHIFT) 769 #define FPSR_Z (1 << FPSR_Z_SHIFT) 770 #define FPSR_C (1 << FPSR_C_SHIFT) 771 #define FPSR_V (1 << FPSR_V_SHIFT) 772 #define FPSR_QC (1 << FPSR_QC_SHIFT) 773 #define FPSR_IDC (1 << FPSR_IDC_SHIFT) 774 #define FPSR_IXC (1 << FPSR_IXC_SHIFT) 775 #define FPSR_UFC (1 << FPSR_UFC_SHIFT) 776 #define FPSR_OFC (1 << FPSR_OFC_SHIFT) 777 #define FPSR_DZC (1 << FPSR_DZC_SHIFT) 778 #define FPSR_IOC (1 << FPSR_IOC_SHIFT) 779 780 /* 781 * A mask for all for all of the bits that are not RAZ for FPSR; this 782 * is primarily for converting between a 32-bit view of NEON state 783 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). 784 */ 785 #define FPSR_MASK \ 786 (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \ 787 FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC) 788 789 /* 790 * FPCR: Floating Point Control Register 791 * 792 * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0 793 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ 794 * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000| 795 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ 796 */ 797 798 #define FPCR_AHP_SHIFT 26 799 #define FPCR_DN_SHIFT 25 800 #define FPCR_FZ_SHIFT 24 801 #define FPCR_RMODE_SHIFT 22 802 #define FPCR_STRIDE_SHIFT 20 803 #define FPCR_LEN_SHIFT 16 804 #define FPCR_IDE_SHIFT 15 805 #define FPCR_IXE_SHIFT 12 806 #define FPCR_UFE_SHIFT 11 807 #define FPCR_OFE_SHIFT 10 808 #define FPCR_DZE_SHIFT 9 809 #define FPCR_IOE_SHIFT 8 810 #define FPCR_AHP (1 << FPCR_AHP_SHIFT) 811 #define FPCR_DN (1 << FPCR_DN_SHIFT) 812 #define FPCR_FZ (1 << FPCR_FZ_SHIFT) 813 #define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT) 814 #define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT) 815 #define FPCR_LEN (0x7 << FPCR_LEN_SHIFT) 816 #define FPCR_IDE (1 << FPCR_IDE_SHIFT) 817 #define FPCR_IXE (1 << FPCR_IXE_SHIFT) 818 #define FPCR_UFE (1 << FPCR_UFE_SHIFT) 819 #define FPCR_OFE (1 << FPCR_OFE_SHIFT) 820 #define FPCR_DZE (1 << FPCR_DZE_SHIFT) 821 #define FPCR_IOE (1 << FPCR_IOE_SHIFT) 822 #define FPCR_DEFAULT (0) 823 #define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ) 824 825 /* 826 * A mask for all for all of the bits that are not RAZ for FPCR; this 827 * is primarily for converting between a 32-bit view of NEON state 828 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). 829 */ 830 #define FPCR_MASK \ 831 (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \ 832 FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE) 833 834 /* 835 * Translation Control Register (TCR) 836 * 837 * Legacy: 838 * 839 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 840 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ 841 * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| 842 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ 843 * 844 * Current (with 16KB granule support): 845 * 846 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 847 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ 848 * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| 849 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ 850 * 851 * TBI1: Top Byte Ignored for TTBR1 region 852 * TBI0: Top Byte Ignored for TTBR0 region 853 * AS: ASID Size 854 * IPS: Physical Address Size limit 855 * TG1: Granule Size for TTBR1 region 856 * SH1: Shareability for TTBR1 region 857 * ORGN1: Outer Cacheability for TTBR1 region 858 * IRGN1: Inner Cacheability for TTBR1 region 859 * EPD1: Translation table walk disable for TTBR1 860 * A1: ASID selection from TTBR1 enable 861 * T1SZ: Virtual address size for TTBR1 862 * TG0: Granule Size for TTBR0 region 863 * SH0: Shareability for TTBR0 region 864 * ORGN0: Outer Cacheability for TTBR0 region 865 * IRGN0: Inner Cacheability for TTBR0 region 866 * T0SZ: Virtual address size for TTBR0 867 */ 868 869 #define TCR_T0SZ_SHIFT 0ULL 870 #define TCR_TSZ_BITS 6ULL 871 #define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL) 872 873 #define TCR_IRGN0_SHIFT 8ULL 874 #define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT) 875 #define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT) 876 #define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT) 877 #define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT) 878 879 #define TCR_ORGN0_SHIFT 10ULL 880 #define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT) 881 #define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT) 882 #define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT) 883 #define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT) 884 885 #define TCR_SH0_SHIFT 12ULL 886 #define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT) 887 #define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT) 888 #define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT) 889 890 #define TCR_TG0_GRANULE_SHIFT (14ULL) 891 #define TCR_TG0_GRANULE_BITS (2ULL) 892 #define TCR_TG0_GRANULE_MASK ((1ULL << TCR_TG0_GRANULE_BITS) - 1ULL) 893 894 #define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT) 895 #define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT) 896 #define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT) 897 898 #if __ARM_16K_PG__ 899 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB) 900 #else 901 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB) 902 #endif 903 904 #define TCR_T1SZ_SHIFT 16ULL 905 906 #define TCR_A1_ASID1 (1ULL << 22ULL) 907 #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL) 908 909 #define TCR_IRGN1_SHIFT 24ULL 910 #define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT) 911 #define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT) 912 #define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT) 913 #define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT) 914 915 #define TCR_ORGN1_SHIFT 26ULL 916 #define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT) 917 #define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT) 918 #define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT) 919 #define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT) 920 921 #define TCR_SH1_SHIFT 28ULL 922 #define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT) 923 #define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT) 924 #define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT) 925 926 #define TCR_TG1_GRANULE_SHIFT 30ULL 927 #define TCR_TG1_GRANULE_BITS (2ULL) 928 #define TCR_TG1_GRANULE_MASK ((1ULL << TCR_TG1_GRANULE_BITS) - 1ULL) 929 930 #define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT) 931 #define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT) 932 #define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT) 933 934 #if __ARM_16K_PG__ 935 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB) 936 #else 937 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB) 938 #endif 939 940 #define TCR_IPS_SHIFT 32ULL 941 #define TCR_IPS_BITS 3ULL 942 #define TCR_IPS_MASK ((1ULL << TCR_IPS_BITS) - 1ULL) 943 #define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT) 944 #define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT) 945 #define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT) 946 #define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT) 947 #define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT) 948 #define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT) 949 950 #define TCR_AS_16BIT_ASID (1ULL << 36) 951 #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37) 952 #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38) 953 #define TCR_TBID0_TBI_DATA_ONLY (1ULL << 51) 954 #define TCR_TBID1_TBI_DATA_ONLY (1ULL << 52) 955 956 #if defined(HAS_APPLE_PAC) 957 #define TCR_TBID0_ENABLE TCR_TBID0_TBI_DATA_ONLY 958 #define TCR_TBID1_ENABLE TCR_TBID1_TBI_DATA_ONLY 959 #else 960 #define TCR_TBID0_ENABLE 0 961 #define TCR_TBID1_ENABLE 0 962 #endif 963 964 #define TCR_E0PD0_BIT (1ULL << 55) 965 #define TCR_E0PD1_BIT (1ULL << 56) 966 967 #if defined(HAS_E0PD) 968 #define TCR_E0PD_VALUE (TCR_E0PD1_BIT) 969 #else 970 #define TCR_E0PD_VALUE 0 971 #endif 972 973 974 #define TCR_EL1_EXTRA 0 975 976 977 /* 978 * Multiprocessor Affinity Register (MPIDR_EL1) 979 * 980 * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+ 981 * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0| 982 * +---------------------------------+--+-----+--+-----+----+----+ 983 * 984 * where 985 * U: Uniprocessor 986 * MT: Multi-threading at lowest affinity level 987 * Aff2: "1" - PCORE, "0" - ECORE 988 * Aff1: Cluster ID 989 * Aff0: CPU ID 990 */ 991 #define MPIDR_AFF0_SHIFT 0 992 #define MPIDR_AFF0_WIDTH 8 993 #define MPIDR_AFF0_MASK (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT) 994 #define MPIDR_AFF1_SHIFT 8 995 #define MPIDR_AFF1_WIDTH 8 996 #define MPIDR_AFF1_MASK (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT) 997 #define MPIDR_AFF2_SHIFT 16 998 #define MPIDR_AFF2_WIDTH 8 999 #define MPIDR_AFF2_MASK (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT) 1000 1001 /* 1002 * TXSZ indicates the size of the range a TTBR covers. Currently, 1003 * we support the following: 1004 * 1005 * 4KB pages, full page L1: 39 bit range. 1006 * 4KB pages, sub-page L1: 38 bit range. 1007 * 16KB pages, full page L1: 47 bit range. 1008 * 16KB pages, sub-page L1: 39 bit range. 1009 * 16KB pages, two level page tables: 36 bit range. 1010 */ 1011 #if __ARM_KERNEL_PROTECT__ 1012 /* 1013 * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the 1014 * address space is used for the mappings that will remain in place when in EL0. 1015 * As a result, 1 bit less of address space is available to the rest of the 1016 * the kernel. 1017 */ 1018 #endif /* __ARM_KERNEL_PROTECT__ */ 1019 #ifdef __ARM_16K_PG__ 1020 #if __ARM64_PMAP_SUBPAGE_L1__ 1021 #define T0SZ_BOOT 25ULL 1022 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ 1023 #define T0SZ_BOOT 17ULL 1024 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ 1025 #else /* __ARM_16K_PG__ */ 1026 #if __ARM64_PMAP_SUBPAGE_L1__ 1027 #define T0SZ_BOOT 26ULL 1028 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 1029 #define T0SZ_BOOT 25ULL 1030 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 1031 #endif /* __ARM_16K_PG__ */ 1032 1033 #if defined(APPLE_ARM64_ARCH_FAMILY) 1034 /* T0SZ must be the same as T1SZ */ 1035 #define T1SZ_BOOT T0SZ_BOOT 1036 #else /* defined(APPLE_ARM64_ARCH_FAMILY) */ 1037 #ifdef __ARM_16K_PG__ 1038 #if __ARM64_PMAP_SUBPAGE_L1__ 1039 #define T1SZ_BOOT 25ULL 1040 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ 1041 #define T1SZ_BOOT 17ULL 1042 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ 1043 #else /* __ARM_16K_PG__ */ 1044 #if __ARM64_PMAP_SUBPAGE_L1__ 1045 #define T1SZ_BOOT 26ULL 1046 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 1047 #define T1SZ_BOOT 25ULL 1048 #endif /*__ARM64_PMAP_SUBPAGE_L1__*/ 1049 #endif /* __ARM_16K_PG__ */ 1050 #endif /* defined(APPLE_ARM64_ARCH_FAMILY) */ 1051 1052 #if __ARM_42BIT_PA_SPACE__ 1053 #define TCR_IPS_VALUE TCR_IPS_42BITS 1054 #else /* !__ARM_42BIT_PA_SPACE__ */ 1055 #define TCR_IPS_VALUE TCR_IPS_40BITS 1056 #endif /* !__ARM_42BIT_PA_SPACE__ */ 1057 1058 #if CONFIG_KERNEL_TBI 1059 #define TCR_EL1_DTBI (TCR_TBI1_TOPBYTE_IGNORED | TCR_TBID1_ENABLE) 1060 #else /* CONFIG_KERNEL_TBI */ 1061 #define TCR_EL1_DTBI 0 1062 #endif /* CONFIG_KERNEL_TBI */ 1063 1064 #if HAS_16BIT_ASID 1065 #define TCR_EL1_ASID TCR_AS_16BIT_ASID 1066 #else /* HAS_16BIT_ASID */ 1067 #define TCR_EL1_ASID 0 1068 #endif /* HAS_16BIT_ASID */ 1069 1070 #define TCR_EL1_BASE \ 1071 (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | \ 1072 TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | \ 1073 TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \ 1074 TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) | \ 1075 TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE) | TCR_E0PD_VALUE | \ 1076 TCR_EL1_DTBI | TCR_EL1_ASID | TCR_EL1_EXTRA) 1077 1078 #if __ARM_KERNEL_PROTECT__ 1079 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1080 #define T1SZ_USER (T1SZ_BOOT + 1) 1081 #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1082 #else 1083 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1084 #endif /* __ARM_KERNEL_PROTECT__ */ 1085 1086 #define TCR_EL1_4KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_4KB)) 1087 #define TCR_EL1_16KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_16KB)) 1088 1089 /* 1090 * Bit 55 of the VA is used to select which TTBR to use during a translation table walk. 1091 */ 1092 #define TTBR_SELECTOR (1ULL << 55) 1093 1094 1095 1096 1097 /* 1098 * Monitor Debug System Control Register (MDSCR) 1099 */ 1100 1101 #define MDSCR_TFO_SHIFT 31 1102 #define MDSCR_TFO (1ULL << MDSCR_TFO_SHIFT) 1103 #define MDSCR_RXFULL_SHIFT 30 1104 #define MDSCR_RXFULL (1ULL << MDSCR_RXFULL_SHIFT) 1105 #define MDSCR_TXFULL_SHIFT 29 1106 #define MDSCR_TXFULL (1ULL << MDSCR_TXFULL_SHIFT) 1107 #define MDSCR_RXO_SHIFT 27 1108 #define MDSCR_RXO (1ULL << MDSCR_RXO_SHIFT) 1109 #define MDSCR_TXU_SHIFT 26 1110 #define MDSCR_TXU (1ULL << MDSCR_TXU_SHIFT) 1111 #define MDSCR_INTDIS_SHIFT 22 1112 #define MDSCR_INTDIS_MASK (0x2U << MDSCR_INTDIS_SHIFT) 1113 #define MDSCR_TDA_SHIFT 21 1114 #define MDSCR_TDA (1ULL << MDSCR_TDA_SHIFT) 1115 #define MDSCR_SC2_SHIFT 19 1116 #define MDSCR_SC2 (1ULL << MDSCR_SC2_SHIFT) 1117 #define MDSCR_MDE_SHIFT 15 1118 #define MDSCR_MDE (1ULL << MDSCR_MDE_SHIFT) 1119 #define MDSCR_HDE_SHIFT 14 1120 #define MDSCR_HDE (1ULL << MDSCR_HDE_SHIFT) 1121 #define MDSCR_KDE_SHIFT 13 1122 #define MDSCR_KDE (1ULL << MDSCR_KDE_SHIFT) 1123 #define MDSCR_TDCC_SHIFT 12 1124 #define MDSCR_TDCC (1ULL << MDSCR_TDCC_SHIFT) 1125 #define MDSCR_ERR_SHIFT 6 1126 #define MDSCR_ERR (1ULL << MDSCR_ERR_SHIFT) 1127 #define MDSCR_SS_SHIFT 0 1128 #define MDSCR_SS (1ULL << MDSCR_SS_SHIFT) 1129 1130 /* 1131 * Translation Table Base Register (TTBR) 1132 * 1133 * 63 48 47 x x-1 0 1134 * +--------+------------------+------+ 1135 * | ASID | Base Address | zero | 1136 * +--------+------------------+------+ 1137 * 1138 */ 1139 #define TTBR_ASID_SHIFT 48 1140 #define TTBR_ASID_MASK 0xffff000000000000 1141 1142 #define TTBR_BADDR_MASK 0x0000ffffffffffff 1143 1144 /* 1145 * Memory Attribute Indirection Register 1146 * 1147 * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0 1148 * +-------+-------+-------+-------+-------+-------+-------+-------+ 1149 * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 | 1150 * +-------+-------+-------+-------+-------+-------+-------+-------+ 1151 * 1152 */ 1153 1154 #define MAIR_ATTR_SHIFT(x) (8*(x)) 1155 1156 /* Strongly ordered or device memory attributes */ 1157 #define MAIR_OUTER_STRONGLY_ORDERED 0x0 1158 #define MAIR_OUTER_DEVICE 0x0 1159 1160 #define MAIR_INNER_STRONGLY_ORDERED 0x0 1161 #define MAIR_INNER_DEVICE 0x4 1162 1163 /* Normal memory attributes */ 1164 #define MAIR_OUTER_NON_CACHEABLE 0x40 1165 #define MAIR_OUTER_WRITE_THROUGH 0x80 1166 #define MAIR_OUTER_WRITE_BACK 0xc0 1167 1168 #define MAIR_INNER_NON_CACHEABLE 0x4 1169 #define MAIR_INNER_WRITE_THROUGH 0x8 1170 #define MAIR_INNER_WRITE_BACK 0xc 1171 1172 /* Allocate policy for cacheable memory */ 1173 #define MAIR_OUTER_WRITE_ALLOCATE 0x10 1174 #define MAIR_OUTER_READ_ALLOCATE 0x20 1175 1176 #define MAIR_INNER_WRITE_ALLOCATE 0x1 1177 #define MAIR_INNER_READ_ALLOCATE 0x2 1178 1179 /* Memory Atribute Encoding */ 1180 1181 /* 1182 * Device memory types: 1183 * G (gathering): multiple reads/writes can be combined 1184 * R (reordering): reads or writes may reach device out of program order 1185 * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes) 1186 */ 1187 #if HAS_FEAT_XS 1188 1189 #define MAIR_DISABLE_XS 0x00 /* Device Memory, nGnRnE (strongly ordered), XS=1 */ 1190 #define MAIR_DISABLE 0x01 /* Device Memory, nGnRnE (strongly ordered), XS=0 */ 1191 #define MAIR_POSTED_COMBINED_REORDERED_XS 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=1 */ 1192 #define MAIR_POSTED_COMBINED_REORDERED 0x0D /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=0 */ 1193 #define MAIR_WRITECOMB 0x40 /* Normal Memory, Non-Cacheable, XS=0 */ 1194 #define MAIR_WRITETHRU 0xA0 /* Normal Memory, Write-through, XS=0 */ 1195 #define MAIR_WRITEBACK 0xFF /* Normal Memory, Write-back, XS=0 */ 1196 1197 1198 /* 1199 * Memory Attribute Index. If these values change, please also update the pmap 1200 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). 1201 */ 1202 #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ 1203 #define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */ 1204 #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ 1205 #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ 1206 #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory), XS = 0 */ 1207 #define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */ 1208 #define CACHE_ATTRINDX_DISABLE_XS 0x5 /* no cache, no buffer (device memory), XS = 1 */ 1209 /** 1210 * Posted mappings use XS by default, and on newer Apple SoCs there is no fabric-level distinction 1211 * between early-ack and non-early-ack, so just alias POSTED to DISABLE_XS to save a MAIR index. 1212 */ 1213 #define CACHE_ATTRINDX_POSTED CACHE_ATTRINDX_DISABLE_XS 1214 #define CACHE_ATTRINDX_POSTED_REORDERED CACHE_ATTRINDX_DISABLE /* no need for device-nGRE on newer SoCs, fallback to nGnRnE */ 1215 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x6 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=0 */ 1216 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED_XS 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=1 */ 1217 #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK 1218 #define CACHE_ATTRINDX_N_INDICES (8ULL) 1219 1220 #else 1221 1222 #define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */ 1223 #define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */ 1224 #define MAIR_POSTED_REORDERED 0x08 /* Device Memory, nGRE (reorderable, posted writes) */ 1225 #define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */ 1226 #define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */ 1227 #define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */ 1228 #define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */ 1229 1230 /* 1231 * Memory Attribute Index. If these values change, please also update the pmap 1232 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). 1233 */ 1234 #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ 1235 #define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */ 1236 #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ 1237 #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ 1238 #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */ 1239 #define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */ 1240 #define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes (device memory) */ 1241 #define CACHE_ATTRINDX_POSTED_REORDERED 0x6 /* no cache, reorderable access, posted writes (device memory) */ 1242 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */ 1243 #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK 1244 #define CACHE_ATTRINDX_N_INDICES (8ULL) 1245 1246 #endif /* HAS_FEAT_XS */ 1247 1248 #if HAS_UCNORMAL_MEM || APPLEVIRTUALPLATFORM 1249 #define CACHE_ATTRINDX_RT CACHE_ATTRINDX_WRITECOMB 1250 #else 1251 #define CACHE_ATTRINDX_RT CACHE_ATTRINDX_DISABLE 1252 #endif /* HAS_UCNORMAL_MEM || APPLEVIRTUALPLATFORM */ 1253 1254 1255 1256 /* 1257 * Access protection bit values (TTEs and PTEs), stage 1 1258 * 1259 * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access) 1260 */ 1261 #define AP_RWNA 0x0 /* priv=read-write, user=no-access */ 1262 #define AP_RWRW 0x1 /* priv=read-write, user=read-write */ 1263 #define AP_RONA 0x2 /* priv=read-only, user=no-access */ 1264 #define AP_RORO 0x3 /* priv=read-only, user=read-only */ 1265 #define AP_MASK 0x3 /* mask to find ap bits */ 1266 1267 /* 1268 * Shareability attributes 1269 */ 1270 #define SH_NONE 0x0 /* Non shareable */ 1271 #define SH_NONE 0x0 /* Device shareable */ 1272 #define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */ 1273 #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */ 1274 #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */ 1275 1276 1277 /* 1278 * ARM Page Granule 1279 */ 1280 #ifdef __ARM_16K_PG__ 1281 #define ARM_PGSHIFT 14 1282 #else 1283 #define ARM_PGSHIFT 12 1284 #endif 1285 #define ARM_PGBYTES (1 << ARM_PGSHIFT) 1286 #define ARM_PGMASK (ARM_PGBYTES-1) 1287 1288 /* 1289 * L0 Translation table 1290 * 1291 * 4KB granule size: 1292 * Each translation table is 4KB 1293 * 512 64-bit entries of 512GB (2^39) of address space. 1294 * Covers 256TB (2^48) of address space. 1295 * 1296 * 16KB granule size: 1297 * Each translation table is 16KB 1298 * 2 64-bit entries of 128TB (2^47) of address space. 1299 * Covers 256TB (2^48) of address space. 1300 */ 1301 1302 /* 16K L0 */ 1303 #define ARM_16K_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */ 1304 #define ARM_16K_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */ 1305 #define ARM_16K_TT_L0_SHIFT 47 /* page descriptor shift */ 1306 #define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */ 1307 1308 /* 4K L0 */ 1309 #define ARM_4K_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */ 1310 #define ARM_4K_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */ 1311 #define ARM_4K_TT_L0_SHIFT 39 /* page descriptor shift */ 1312 #define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */ 1313 1314 /* 1315 * L1 Translation table 1316 * 1317 * 4KB granule size: 1318 * Each translation table is 4KB 1319 * 512 64-bit entries of 1GB (2^30) of address space. 1320 * Covers 512GB (2^39) of address space. 1321 * 1322 * 16KB granule size: 1323 * Each translation table is 16KB 1324 * 2048 64-bit entries of 64GB (2^36) of address space. 1325 * Covers 128TB (2^47) of address space. 1326 */ 1327 1328 /* 16K L1 */ 1329 #define ARM_16K_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */ 1330 #define ARM_16K_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */ 1331 #define ARM_16K_TT_L1_SHIFT 36 /* page descriptor shift */ 1332 #if __ARM64_PMAP_SUBPAGE_L1__ && __ARM_16K_PG__ 1333 /* This config supports 512GB per TTBR. */ 1334 #define ARM_16K_TT_L1_INDEX_MASK 0x0000007000000000ULL /* mask for getting index into L1 table from virtual address */ 1335 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 1336 #define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL /* mask for getting index into L1 table from virtual address */ 1337 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 1338 1339 /* 4K L1 */ 1340 #define ARM_4K_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */ 1341 #define ARM_4K_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */ 1342 #define ARM_4K_TT_L1_SHIFT 30 /* page descriptor shift */ 1343 #if __ARM64_PMAP_SUBPAGE_L1__ && !__ARM_16K_PG__ 1344 /* This config supports 256GB per TTBR. */ 1345 #define ARM_4K_TT_L1_INDEX_MASK 0x0000003fc0000000ULL /* mask for getting index into L1 table from virtual address */ 1346 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 1347 /* IPA[38:30] mask for getting index into L1 table from virtual address */ 1348 #define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL 1349 #if __ARM_42BIT_PA_SPACE__ 1350 /* IPA[39:30] mask for getting index into L1 concatenated table from virtual address */ 1351 #define ARM_4K_TT_L1_40_BIT_CONCATENATED_INDEX_MASK 0x000000ffc0000000ULL 1352 #endif /* __ARM_42BIT_PA_SPACE__ */ 1353 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 1354 1355 /* some sugar for getting pointers to page tables and entries */ 1356 1357 #define L1_TABLE_INDEX(va) (((va) & ARM_TT_L1_INDEX_MASK) >> ARM_TT_L1_SHIFT) 1358 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) 1359 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT) 1360 1361 #define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK)) 1362 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK)) 1363 1364 /* 1365 * L2 Translation table 1366 * 1367 * 4KB granule size: 1368 * Each translation table is 4KB 1369 * 512 64-bit entries of 2MB (2^21) of address space. 1370 * Covers 1GB (2^30) of address space. 1371 * 1372 * 16KB granule size: 1373 * Each translation table is 16KB 1374 * 2048 64-bit entries of 32MB (2^25) of address space. 1375 * Covers 64GB (2^36) of address space. 1376 */ 1377 1378 /* 16K L2 */ 1379 #define ARM_16K_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */ 1380 #define ARM_16K_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */ 1381 #define ARM_16K_TT_L2_SHIFT 25 /* page descriptor shift */ 1382 #define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */ 1383 1384 /* 4K L2 */ 1385 #define ARM_4K_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */ 1386 #define ARM_4K_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */ 1387 #define ARM_4K_TT_L2_SHIFT 21 /* page descriptor shift */ 1388 #define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */ 1389 1390 /* 1391 * L3 Translation table 1392 * 1393 * 4KB granule size: 1394 * Each translation table is 4KB 1395 * 512 64-bit entries of 4KB (2^12) of address space. 1396 * Covers 2MB (2^21) of address space. 1397 * 1398 * 16KB granule size: 1399 * Each translation table is 16KB 1400 * 2048 64-bit entries of 16KB (2^14) of address space. 1401 * Covers 32MB (2^25) of address space. 1402 */ 1403 1404 /* 16K L3 */ 1405 #define ARM_16K_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ 1406 #define ARM_16K_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */ 1407 #define ARM_16K_TT_L3_SHIFT 14 /* page descriptor shift */ 1408 #define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */ 1409 1410 /* 4K L3 */ 1411 #define ARM_4K_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ 1412 #define ARM_4K_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */ 1413 #define ARM_4K_TT_L3_SHIFT 12 /* page descriptor shift */ 1414 #define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */ 1415 1416 #ifdef __ARM_16K_PG__ 1417 1418 /* Native L0 defines */ 1419 #define ARM_TT_L0_SIZE ARM_16K_TT_L0_SIZE 1420 #define ARM_TT_L0_OFFMASK ARM_16K_TT_L0_OFFMASK 1421 #define ARM_TT_L0_SHIFT ARM_16K_TT_L0_SHIFT 1422 #define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK 1423 1424 /* Native L1 defines */ 1425 #define ARM_TT_L1_SIZE ARM_16K_TT_L1_SIZE 1426 #define ARM_TT_L1_OFFMASK ARM_16K_TT_L1_OFFMASK 1427 #define ARM_TT_L1_SHIFT ARM_16K_TT_L1_SHIFT 1428 #define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK 1429 1430 /* Native L2 defines */ 1431 #define ARM_TT_L2_SIZE ARM_16K_TT_L2_SIZE 1432 #define ARM_TT_L2_OFFMASK ARM_16K_TT_L2_OFFMASK 1433 #define ARM_TT_L2_SHIFT ARM_16K_TT_L2_SHIFT 1434 #define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK 1435 1436 /* Native L3 defines */ 1437 #define ARM_TT_L3_SIZE ARM_16K_TT_L3_SIZE 1438 #define ARM_TT_L3_OFFMASK ARM_16K_TT_L3_OFFMASK 1439 #define ARM_TT_L3_SHIFT ARM_16K_TT_L3_SHIFT 1440 #define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK 1441 1442 #else /* !__ARM_16K_PG__ */ 1443 1444 /* Native L0 defines */ 1445 #define ARM_TT_L0_SIZE ARM_4K_TT_L0_SIZE 1446 #define ARM_TT_L0_OFFMASK ARM_4K_TT_L0_OFFMASK 1447 #define ARM_TT_L0_SHIFT ARM_4K_TT_L0_SHIFT 1448 #define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK 1449 1450 /* Native L1 defines */ 1451 #define ARM_TT_L1_SIZE ARM_4K_TT_L1_SIZE 1452 #define ARM_TT_L1_OFFMASK ARM_4K_TT_L1_OFFMASK 1453 #define ARM_TT_L1_SHIFT ARM_4K_TT_L1_SHIFT 1454 #define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK 1455 1456 /* Native L2 defines */ 1457 #define ARM_TT_L2_SIZE ARM_4K_TT_L2_SIZE 1458 #define ARM_TT_L2_OFFMASK ARM_4K_TT_L2_OFFMASK 1459 #define ARM_TT_L2_SHIFT ARM_4K_TT_L2_SHIFT 1460 #define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK 1461 1462 /* Native L3 defines */ 1463 #define ARM_TT_L3_SIZE ARM_4K_TT_L3_SIZE 1464 #define ARM_TT_L3_OFFMASK ARM_4K_TT_L3_OFFMASK 1465 #define ARM_TT_L3_SHIFT ARM_4K_TT_L3_SHIFT 1466 #define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK 1467 1468 #endif /* !__ARM_16K_PG__ */ 1469 1470 /* 1471 * Convenience definitions for: 1472 * ARM_TT_LEAF: The last level of the configured page table format. 1473 * ARM_TT_TWIG: The second to last level of the configured page table format. 1474 * ARM_TT_ROOT: The first level of the configured page table format. 1475 * 1476 * My apologies to any botanists who may be reading this. 1477 */ 1478 #define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE 1479 #define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK 1480 #define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT 1481 #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK 1482 1483 #define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE 1484 #define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK 1485 #define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT 1486 #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK 1487 1488 #define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE 1489 #define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK 1490 #define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT 1491 #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK 1492 1493 /* 1494 * 4KB granule size: 1495 * 1496 * Level 0 Translation Table Entry 1497 * 1498 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1499 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1500 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| 1501 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1502 * 1503 * Level 1 Translation Table Entry 1504 * 1505 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1506 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1507 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| 1508 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1509 * 1510 * Level 1 Translation Block Entry 1511 * 1512 * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0 1513 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1514 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1515 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1516 * 1517 * Level 2 Translation Table Entry 1518 * 1519 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 1520 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1521 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| 1522 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1523 * 1524 * Level 2 Translation Block Entry 1525 * 1526 * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0 1527 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1528 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1529 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1530 * 1531 * 16KB granule size: 1532 * 1533 * Level 0 Translation Table Entry 1534 * 1535 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1536 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1537 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| 1538 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1539 * 1540 * Level 1 Translation Table Entry 1541 * 1542 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1543 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1544 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| 1545 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1546 * 1547 * Level 2 Translation Table Entry 1548 * 1549 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 1550 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1551 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| 1552 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 1553 * 1554 * Level 2 Translation Block Entry 1555 * 1556 * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0 1557 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1558 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 1559 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 1560 * 1561 * where: 1562 * nG: notGlobal bit 1563 * SH: Shareability field 1564 * AP: access protection 1565 * XN: eXecute Never bit 1566 * PXN: Privilege eXecute Never bit 1567 * NS: Non-Secure bit 1568 * HINT: 16 entry continuguous output hint 1569 * AttrIdx: Memory Attribute Index 1570 */ 1571 1572 #define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */ 1573 #ifdef __ARM_16K_PG__ 1574 #define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */ 1575 #else 1576 #define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */ 1577 #endif 1578 1579 #define ARM_TTE_MAX (TTE_PGENTRIES) 1580 1581 #define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ 1582 #define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */ 1583 1584 #define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */ 1585 1586 #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ 1587 #define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */ 1588 #define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */ 1589 #define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL 1590 1591 #ifdef __ARM_16K_PG__ 1592 /* 1593 * Note that L0/L1 block entries are disallowed for the 16KB granule size; what 1594 * are we doing with these? 1595 */ 1596 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */ 1597 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */ 1598 #define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */ 1599 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */ 1600 #define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */ 1601 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */ 1602 #else 1603 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */ 1604 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */ 1605 #define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */ 1606 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */ 1607 #define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */ 1608 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */ 1609 #endif 1610 1611 #define ARM_TTE_BLOCK_APSHIFT 6 1612 #define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */ 1613 #define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT) 1614 1615 #define ARM_TTE_BLOCK_ATTRINDX(x) ((x) << 2) /* memory attributes index */ 1616 #define ARM_TTE_BLOCK_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ 1617 1618 #define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */ 1619 #define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */ 1620 1621 #define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */ 1622 #define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */ 1623 1624 #define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */ 1625 #define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ 1626 1627 #define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */ 1628 #define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ 1629 1630 #define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ 1631 #define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ 1632 1633 #define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */ 1634 #define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */ 1635 1636 #define ARM_TTE_BLOCK_WIRED 0x0400000000000000ULL /* value for software wired bit */ 1637 #define ARM_TTE_BLOCK_WIREDMASK 0x0400000000000000ULL /* software wired mask */ 1638 1639 #define ARM_TTE_BLOCK_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ 1640 #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL /* software writeable mask */ 1641 1642 #define ARM_TTE_TABLE_MASK 0x0000fffffffff000ULL /* mask for extracting pointer to next table (works at any level) */ 1643 1644 #define ARM_TTE_TABLE_APSHIFT 61 1645 #define ARM_TTE_TABLE_AP_NO_EFFECT 0x0ULL 1646 #define ARM_TTE_TABLE_AP_USER_NA 0x1ULL 1647 #define ARM_TTE_TABLE_AP_RO 0x2ULL 1648 #define ARM_TTE_TABLE_AP_KERN_RO 0x3ULL 1649 #define ARM_TTE_TABLE_AP(x) ((x) << ARM_TTE_TABLE_APSHIFT) /* access protection */ 1650 1651 #define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */ 1652 #define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */ 1653 1654 #define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */ 1655 #define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */ 1656 1657 #define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */ 1658 #define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */ 1659 1660 #if __ARM_KERNEL_PROTECT__ 1661 #define ARM_TTE_BOOT_BLOCK_LOWER \ 1662 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ 1663 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG) 1664 #else /* __ARM_KERNEL_PROTECT__ */ 1665 #define ARM_TTE_BOOT_BLOCK_LOWER \ 1666 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ 1667 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF) 1668 #endif /* __ARM_KERNEL_PROTECT__ */ 1669 #define ARM_TTE_BOOT_BLOCK_UPPER ARM_TTE_BLOCK_NX 1670 1671 #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID ) 1672 /* 1673 * L3 Translation table 1674 * 1675 * 4KB granule size: 1676 * Each translation table is 4KB 1677 * 512 64-bit entries of 4KB (2^12) of address space. 1678 * Covers 2MB (2^21) of address space. 1679 * 1680 * 16KB granule size: 1681 * Each translation table is 16KB 1682 * 2048 64-bit entries of 16KB (2^14) of address space. 1683 * Covers 32MB (2^25) of address space. 1684 */ 1685 1686 #ifdef __ARM_16K_PG__ 1687 #define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ 1688 #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */ 1689 #define ARM_PTE_SHIFT 14 /* page descriptor shift */ 1690 #define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */ 1691 #else 1692 #define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ 1693 #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */ 1694 #define ARM_PTE_SHIFT 12 /* page descriptor shift */ 1695 #define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */ 1696 #endif 1697 1698 #define ARM_TTE_PA_MASK 0x0000fffffffff000ULL 1699 1700 /* 1701 * L3 Page table entries 1702 * 1703 * The following page table entry types are possible: 1704 * 1705 * fault page entry 1706 * 63 2 0 1707 * +------------------------------+--+ 1708 * | ignored |00| 1709 * +------------------------------+--+ 1710 * 1711 * 1712 * 63 59 58 55 54 53 52 51 50 47 48 12 11 10 9 8 7 6 5 4 2 1 0 1713 * +-----+------+--+---+----+---+--+----+----------------------+--+--+----+----+--+-------+-+-+ 1714 * | ign |sw use|XN|PXN|HINT|DBM|GP|zero| OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V| 1715 * +-----+------+--+---+----+---+--+----+----------------------+--+--+----+----+--+-------+-+-+ 1716 * 1717 * where: 1718 * nG: notGlobal bit 1719 * SH: Shareability field 1720 * AP: access protection 1721 * XN: eXecute Never bit 1722 * PXN: Privilege eXecute Never bit 1723 * NS: Non-Secure bit 1724 * HINT: 16 entry continuguous output hint 1725 * DBM: Dirty Bit Modifier 1726 * GP: Guraded Page 1727 * AttrIdx: Memory Attribute Index 1728 */ 1729 1730 #define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */ 1731 #ifdef __ARM_16K_PG__ 1732 #define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */ 1733 #else 1734 #define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */ 1735 #endif 1736 1737 #define ARM_PTE_EMPTY 0x0000000000000000ULL /* unassigned - invalid entry */ 1738 1739 /* markers for (invalid) PTE for a page sent to compressor */ 1740 #define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */ 1741 #define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */ 1742 #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL 1743 1744 #define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ 1745 #define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */ 1746 #define ARM_PTE_TYPE_MASK 0x0000000000000003ULL /* mask to get pte type */ 1747 1748 /* This mask works for both 16K and 4K pages because bits 12-13 will be zero in 16K pages */ 1749 #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* output address mask for page */ 1750 #define ARM_PTE_PAGE_SHIFT 12 /* page shift for the output address in the entry */ 1751 1752 #define ARM_PTE_AP(x) ((x) << 6) /* access protections */ 1753 #define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */ 1754 #define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */ 1755 1756 #define ARM_PTE_ATTRINDX(x) (uint64_t)((x) << 2) /* memory attributes index */ 1757 #define ARM_PTE_ATTRINDXMASK (0x7ULL << 2) /* mask memory attributes index */ 1758 #define ARM_PTE_EXTRACT_ATTRINDX(x) (((x) >> 2) & 0x7ULL) /* extract memory attributes index */ 1759 1760 #define ARM_PTE_SH(x) ((x) << 8) /* access shared */ 1761 #define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */ 1762 1763 #define ARM_PTE_AF 0x0000000000000400ULL /* value for access */ 1764 #define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */ 1765 1766 #define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */ 1767 #define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ 1768 1769 #define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */ 1770 #define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ 1771 1772 #define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */ 1773 #define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */ 1774 1775 #define ARM_PTE_GP 0x0004000000000000ULL /* value marking a guarded page */ 1776 #define ARM_PTE_GP_MASK 0x0004000000000000ULL /* mask for a guarded page mark */ 1777 1778 #if __ARM_16K_PG__ 1779 #define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */ 1780 #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */ 1781 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */ 1782 #define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */ 1783 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */ 1784 #else 1785 #define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */ 1786 #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */ 1787 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */ 1788 #define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */ 1789 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */ 1790 #endif 1791 1792 #define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ 1793 #define ARM_PTE_PXN ARM_PTE_PNX 1794 #define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ 1795 1796 #define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */ 1797 #define ARM_PTE_XN ARM_PTE_NX 1798 #define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */ 1799 1800 #define ARM_PTE_XMASK (ARM_PTE_PNXMASK | ARM_PTE_NXMASK) 1801 1802 #define ARM_PTE_GUARDED 0x0004000000000000ULL /* value for "guarded"/BTI enforcing code page */ 1803 #define ARM_PTE_GUARDED_MASK (PTE_GUARDED) 1804 1805 #define ARM_PTE_WIRED 0x0400000000000000ULL /* value for software wired bit */ 1806 #define ARM_PTE_WIRED_MASK 0x0400000000000000ULL /* software wired mask */ 1807 1808 #define ARM_PTE_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ 1809 #define ARM_PTE_WRITABLE ARM_PTE_WRITEABLE 1810 #define ARM_PTE_WRITEABLE_MASK 0x0800000000000000ULL /* software writeable mask */ 1811 1812 #define ARM_PTE_SW_RESERVED_MASK (ARM_PTE_WIRED_MASK | ARM_PTE_WRITEABLE_MASK) 1813 1814 #define ARM_PTE_BOOT_PAGE_BASE \ 1815 (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | \ 1816 ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF) 1817 1818 #if __ARM_KERNEL_PROTECT__ 1819 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG) 1820 #else /* __ARM_KERNEL_PROTECT__ */ 1821 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE) 1822 #endif /* __ARM_KERNEL_PROTECT__ */ 1823 1824 /* 1825 * TLBI appers to only deal in 4KB page addresses, so give 1826 * it an explicit shift of 12. 1827 */ 1828 #define TLBI_ADDR_SHIFT (0) 1829 #define TLBI_ADDR_SIZE (44) 1830 #define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) 1831 #define TLBI_IPA_SHIFT (0) 1832 #define TLBI_IPA_SIZE (36) 1833 #define TLBI_IPA_MASK ((1ULL << TLBI_IPA_SIZE) - 1) 1834 #define TLBI_ASID_SHIFT (48) 1835 #define TLBI_ASID_SIZE (16) 1836 #define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1)) 1837 1838 #define RTLBI_ADDR_SIZE (37) 1839 #define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1) 1840 #define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT 1841 #define RTLBI_TG(_page_shift_) ((uint64_t)((((_page_shift_) - 12) >> 1) + 1) << 46) 1842 #define RTLBI_SCALE_SHIFT (44) 1843 #define RTLBI_NUM_SHIFT (39) 1844 1845 /* 1846 * RCTX instruction operand fields. 1847 */ 1848 #define RCTX_EL_SHIFT (24) 1849 #define RCTX_EL_SIZE (2) 1850 #define RCTX_EL_MASK (((1ULL << RCTX_EL_SIZE) - 1) << RCTX_EL_SHIFT) 1851 #define RCTX_EL(x) ((x << RCTX_EL_SHIFT) & RCTX_EL_MASK) 1852 #define RCTX_ASID_SHIFT (0) 1853 #define RCTX_ASID_SIZE (16) 1854 #define RCTX_ASID_MASK (((1ULL << RCTX_ASID_SIZE) - 1) << RCTX_ASID_SHIFT) 1855 #define RCTX_ASID(x) ((x << RCTX_ASID_SHIFT) & RCTX_ASID_MASK) 1856 1857 /* 1858 * Exception Syndrome Register 1859 * 1860 * 63 56 55 32 31 26 25 24 0 1861 * +------+------+------+--+------------------+ 1862 * | RES0 | ISS2 | EC |IL| ISS | 1863 * +------+------+------+--+------------------+ 1864 * 1865 * RES0 - Reserved bits. 1866 * ISS2 - Instruction Specific Syndrome 2. 1867 * EC - Exception Class 1868 * IL - Instruction Length 1869 * ISS - Instruction Specific Syndrome 1870 * 1871 * Note: The ISS can have many forms. These are defined separately below. 1872 */ 1873 1874 #define ESR_EC_SHIFT 26 1875 #define ESR_EC_WIDTH 6 1876 #define ESR_EC_MASK (0x3FULL << ESR_EC_SHIFT) 1877 #define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT) 1878 1879 #define ESR_IL_SHIFT 25 1880 #define ESR_IL (1 << ESR_IL_SHIFT) 1881 1882 #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL)) 1883 1884 #define ESR_ISS_MASK 0x01FFFFFF 1885 #define ESR_ISS(x) (x & ESR_ISS_MASK) 1886 1887 1888 #ifdef __ASSEMBLER__ 1889 /* Define only the classes we need to test in the exception vectors. */ 1890 #define ESR_EC_UNCATEGORIZED 0x00 1891 #define ESR_EC_BTI_FAIL 0x0D 1892 #define ESR_EC_SVC_64 0x15 1893 #define ESR_EC_HVC_64 0x16 1894 #define ESR_EC_PAC_FAIL 0x1C 1895 #define ESR_EC_IABORT_EL1 0x21 1896 #define ESR_EC_DABORT_EL1 0x25 1897 #define ESR_EC_SP_ALIGN 0x26 1898 #define ESR_EC_BRK_AARCH64 0x3C 1899 #else 1900 typedef enum { 1901 ESR_EC_UNCATEGORIZED = 0x00, 1902 ESR_EC_WFI_WFE = 0x01, 1903 ESR_EC_MCR_MRC_CP15_TRAP = 0x03, 1904 ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04, 1905 ESR_EC_MCR_MRC_CP14_TRAP = 0x05, 1906 ESR_EC_LDC_STC_CP14_TRAP = 0x06, 1907 ESR_EC_TRAP_SIMD_FP = 0x07, 1908 ESR_EC_PTRAUTH_INSTR_TRAP = 0x09, 1909 ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c, 1910 ESR_EC_BTI_FAIL = 0x0d, 1911 ESR_EC_ILLEGAL_INSTR_SET = 0x0e, 1912 ESR_EC_SVC_32 = 0x11, 1913 ESR_EC_HVC_32 = 0x12, 1914 ESR_EC_SVC_64 = 0x15, 1915 ESR_EC_HVC_64 = 0x16, 1916 ESR_EC_MSR_TRAP = 0x18, 1917 #if __has_feature(ptrauth_calls) 1918 ESR_EC_PAC_FAIL = 0x1C, 1919 #endif /* __has_feature(ptrauth_calls) */ 1920 #if HAS_ARM_FEAT_SME 1921 ESR_EC_SME = 0x1D, 1922 #endif 1923 ESR_EC_IABORT_EL0 = 0x20, 1924 ESR_EC_IABORT_EL1 = 0x21, 1925 ESR_EC_PC_ALIGN = 0x22, 1926 ESR_EC_DABORT_EL0 = 0x24, 1927 ESR_EC_DABORT_EL1 = 0x25, 1928 ESR_EC_SP_ALIGN = 0x26, 1929 ESR_EC_FLOATING_POINT_32 = 0x28, 1930 ESR_EC_FLOATING_POINT_64 = 0x2C, 1931 ESR_EC_SERROR_INTERRUPT = 0x2F, 1932 ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL. 1933 ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL. 1934 ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL. 1935 ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL. 1936 ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL. 1937 ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL. 1938 ESR_EC_BKPT_AARCH32 = 0x38, 1939 ESR_EC_BRK_AARCH64 = 0x3C, 1940 } esr_exception_class_t; 1941 1942 typedef enum { 1943 FSC_TRANSLATION_FAULT_L0 = 0x04, 1944 FSC_TRANSLATION_FAULT_L1 = 0x05, 1945 FSC_TRANSLATION_FAULT_L2 = 0x06, 1946 FSC_TRANSLATION_FAULT_L3 = 0x07, 1947 FSC_ACCESS_FLAG_FAULT_L1 = 0x09, 1948 FSC_ACCESS_FLAG_FAULT_L2 = 0x0A, 1949 FSC_ACCESS_FLAG_FAULT_L3 = 0x0B, 1950 FSC_PERMISSION_FAULT_L1 = 0x0D, 1951 FSC_PERMISSION_FAULT_L2 = 0x0E, 1952 FSC_PERMISSION_FAULT_L3 = 0x0F, 1953 FSC_SYNC_EXT_ABORT = 0x10, 1954 FSC_SYNC_EXT_ABORT_TT_L1 = 0x15, 1955 FSC_SYNC_EXT_ABORT_TT_L2 = 0x16, 1956 FSC_SYNC_EXT_ABORT_TT_L3 = 0x17, 1957 FSC_SYNC_PARITY = 0x18, 1958 FSC_ASYNC_PARITY = 0x19, 1959 FSC_SYNC_PARITY_TT_L1 = 0x1D, 1960 FSC_SYNC_PARITY_TT_L2 = 0x1E, 1961 FSC_SYNC_PARITY_TT_L3 = 0x1F, 1962 FSC_ALIGNMENT_FAULT = 0x21, 1963 FSC_DEBUG_FAULT = 0x22, 1964 } fault_status_t; 1965 #endif /* ASSEMBLER */ 1966 1967 /* 1968 * SVC event 1969 * 24 16 15 0 1970 * +---------+-----+ 1971 * |000000000| IMM | 1972 * +---------+-----+ 1973 * 1974 * where: 1975 * IMM: Immediate value 1976 */ 1977 1978 #define ISS_SVC_IMM_MASK 0xffff 1979 #define ISS_SVC_IMM(x) ((x) & ISS_SVC_IMM_MASK) 1980 1981 /* 1982 * HVC event 1983 * 24 16 15 0 1984 * +---------+-----+ 1985 * |000000000| IMM | 1986 * +---------+-----+ 1987 * 1988 * where: 1989 * IMM: Immediate value 1990 */ 1991 1992 #define ISS_HVC_IMM_MASK 0xffff 1993 #define ISS_HVC_IMM(x) ((x) & ISS_HVC_IMM_MASK) 1994 1995 1996 /* 1997 * Software step debug event ISS (EL1) 1998 * 24 23 6 5 0 1999 * +---+-----------------+--+------+ 2000 * |ISV|00000000000000000|EX| IFSC | 2001 * +---+-----------------+--+------+ 2002 * 2003 * where: 2004 * ISV: Instruction syndrome valid 2005 * EX: Exclusive access 2006 * IFSC: Instruction Fault Status Code 2007 */ 2008 2009 #define ISS_SSDE_ISV_SHIFT 24 2010 #define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT) 2011 2012 #define ISS_SSDE_EX_SHIFT 6 2013 #define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT) 2014 2015 #define ISS_SSDE_FSC_MASK 0x3F 2016 #define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK) 2017 2018 /* 2019 * Instruction Abort ISS (EL1) 2020 * 24 10 9 5 0 2021 * +--------------+---+--+---+------+ 2022 * |00000000000000|FnV|EA|000| IFSC | 2023 * +--------------+---+--+---+------+ 2024 * 2025 * where: 2026 * FnV: FAR not Valid 2027 * EA: External Abort type 2028 * IFSC: Instruction Fault Status Code 2029 */ 2030 2031 #define ISS_IA_FNV_SHIFT 10 2032 #define ISS_IA_FNV (0x1 << ISS_IA_FNV_SHIFT) 2033 2034 #define ISS_IA_EA_SHIFT 9 2035 #define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT) 2036 2037 #define ISS_IA_FSC_MASK 0x3F 2038 #define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK) 2039 2040 2041 /* 2042 * Data Abort ISS (EL1) 2043 * 2044 * 24 10 9 8 7 6 5 0 2045 * +--------------+---+--+--+-----+---+----+ 2046 * |00000000000000|FnV|EA|CM|S1PTW|WnR|DFSC| 2047 * +--------------+---+--+--+-----+---+----+ 2048 * 2049 * where: 2050 * FnV: FAR not Valid 2051 * EA: External Abort type 2052 * CM: Cache Maintenance operation 2053 * WnR: Write not Read 2054 * S1PTW: Stage 2 exception on Stage 1 page table walk 2055 * DFSC: Data Fault Status Code 2056 */ 2057 #define ISS_DA_FNV_SHIFT 10 2058 #define ISS_DA_FNV (0x1 << ISS_DA_FNV_SHIFT) 2059 2060 #define ISS_DA_EA_SHIFT 9 2061 #define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT) 2062 2063 #define ISS_DA_CM_SHIFT 8 2064 #define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT) 2065 2066 #define ISS_DA_WNR_SHIFT 6 2067 #define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT) 2068 2069 #define ISS_DA_S1PTW_SHIFT 7 2070 #define ISS_DA_S1PTW (0x1 << ISS_DA_S1PTW_SHIFT) 2071 2072 #define ISS_DA_FSC_MASK 0x3F 2073 #define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK) 2074 2075 /* 2076 * Floating Point Exception ISS (EL1) 2077 * 2078 * 24 23 22 8 7 4 3 2 1 0 2079 * +-+---+---------------+---+--+---+---+---+---+---+ 2080 * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF| 2081 * +-+---+---------------+---+--+---+---+---+---+---+ 2082 * 2083 * where: 2084 * TFV: Trapped Fault Valid 2085 * IDF: Input Denormal Exception 2086 * IXF: Input Inexact Exception 2087 * UFF: Underflow Exception 2088 * OFF: Overflow Exception 2089 * DZF: Divide by Zero Exception 2090 * IOF: Invalid Operation Exception 2091 */ 2092 #define ISS_FP_TFV_SHIFT 23 2093 #define ISS_FP_TFV (0x1 << ISS_FP_TFV_SHIFT) 2094 2095 #define ISS_FP_IDF_SHIFT 7 2096 #define ISS_FP_IDF (0x1 << ISS_FP_IDF_SHIFT) 2097 2098 #define ISS_FP_IXF_SHIFT 4 2099 #define ISS_FP_IXF (0x1 << ISS_FP_IXF_SHIFT) 2100 2101 #define ISS_FP_UFF_SHIFT 3 2102 #define ISS_FP_UFF (0x1 << ISS_FP_UFF_SHIFT) 2103 2104 #define ISS_FP_OFF_SHIFT 2 2105 #define ISS_FP_OFF (0x1 << ISS_FP_OFF_SHIFT) 2106 2107 #define ISS_FP_DZF_SHIFT 1 2108 #define ISS_FP_DZF (0x1 << ISS_FP_DZF_SHIFT) 2109 2110 #define ISS_FP_IOF_SHIFT 0 2111 #define ISS_FP_IOF (0x1 << ISS_FP_IOF_SHIFT) 2112 2113 /* 2114 * Breakpoint Exception ISS (EL1) 2115 * 24 16 0 2116 * +---------+---------+ 2117 * |000000000| Comment | 2118 * +---------+---------+ 2119 * 2120 * where: 2121 * Comment: Instruction Comment Field Value 2122 */ 2123 #define ISS_BRK_COMMENT_MASK 0xFFFF 2124 #define ISS_BRK_COMMENT(x) (x & ISS_BRK_COMMENT_MASK) 2125 2126 2127 2128 /* 2129 * SError Interrupt, IDS=1 2130 * 24 23 0 2131 * +---+------------------------+ 2132 * |IDS| IMPLEMENTATION DEFINED | 2133 * +---+------------------------+ 2134 * 2135 * where: 2136 * IDS: Implementation-defined syndrome (1) 2137 */ 2138 2139 #define ISS_SEI_IDS_SHIFT 24 2140 #define ISS_SEI_IDS (0x1 << ISS_SEI_IDS_SHIFT) 2141 2142 2143 #if HAS_UCNORMAL_MEM 2144 #define ISS_UC 0x11 2145 #endif /* HAS_UCNORMAL_MEM */ 2146 2147 2148 2149 #if HAS_ARM_FEAT_SME 2150 2151 /* 2152 * SME ISS (EL1) 2153 * 2154 * 24 3 2 0 2155 * +----------------------+----+ 2156 * |0000000000000000000000|SMTC| 2157 * +----------------------+----+ 2158 * 2159 * where: 2160 * SMTC: SME Trap Code 2161 */ 2162 #define ISS_SME_SMTC_CAPCR 0x0 2163 #define ISS_SME_SMTC_MASK 0x7 2164 #define ISS_SME_SMTC(x) ((x) & ISS_SME_SMTC_MASK) 2165 2166 2167 /* 2168 * SME Control Register (EL1) 2169 * 31 30 29 4 3 0 2170 * +----+----+--------------------------+---+ 2171 * |FA64|EZT0|00000000000000000000000000|LEN| 2172 * +----+----+--------------------------+---+ 2173 * 2174 * where: 2175 * FA64: Enable FEAT_SME_FA64 2176 * EZT0: Enable ZT0 2177 * LEN: Effective SVL = (LEN + 1) * 128 2178 */ 2179 2180 #define SMCR_EL1_LEN_MASK 0xf 2181 #if HAS_ARM_FEAT_SME2 2182 #define SMCR_EL1_EZT0 (1ULL << 30) 2183 #endif 2184 #define SMCR_EL1_LEN(x) ((x) & SMCR_EL1_LEN_MASK) 2185 2186 #define SMPRI_EL1_PRIORITY_MASK 0xf 2187 #define SMPRI_EL1_PRIORITY(x) ((x) & SMPRI_EL1_PRIORITY_MASK) 2188 2189 /* 2190 * Streaming Vector Control Register (SVCR) 2191 */ 2192 #define SVCR_ZA_SHIFT (1) 2193 #define SVCR_ZA (1ULL << SVCR_ZA_SHIFT) 2194 #define SVCR_SM_SHIFT (0) 2195 #define SVCR_SM (1ULL << SVCR_SM_SHIFT) 2196 2197 #endif /* HAS_ARM_FEAT_SME */ 2198 2199 /* 2200 * Branch Target Indication Exception ISS 2201 * 24 3 2 0 2202 * +----+-----+ 2203 * |res0|BTYPE| 2204 * +----+-----+ 2205 */ 2206 #define ISS_BTI_BTYPE_SHIFT (0) 2207 #define ISS_BTI_BTYPE_MASK (0x3 << ISS_BTI_BTYPE_SHIFT) 2208 2209 /* 2210 * Physical Address Register (EL1) 2211 */ 2212 #define PAR_F_SHIFT 0 2213 #define PAR_F (0x1 << PAR_F_SHIFT) 2214 2215 #define PLATFORM_SYSCALL_TRAP_NO 0x80000000 2216 2217 #define ARM64_SYSCALL_CODE_REG_NUM (16) 2218 2219 #define ARM64_CLINE_SHIFT 6 2220 2221 #if defined(APPLE_ARM64_ARCH_FAMILY) 2222 #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */ 2223 #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */ 2224 #endif 2225 2226 /* 2227 * Timer definitions. 2228 */ 2229 #define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */ 2230 #define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */ 2231 #define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */ 2232 #define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */ 2233 #define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */ 2234 #define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */ 2235 #define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to virtual timebase + frequency reg enabled */ 2236 #define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to physical timebase + frequency reg enabled */ 2237 2238 #define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */ 2239 #define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */ 2240 #define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */ 2241 2242 #define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS 2243 #define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED 2244 #define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE 2245 2246 #define MIDR_EL1_REV_SHIFT 0 2247 #define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT) 2248 #define MIDR_EL1_PNUM_SHIFT 4 2249 #define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT) 2250 #define MIDR_EL1_ARCH_SHIFT 16 2251 #define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT) 2252 #define MIDR_EL1_VAR_SHIFT 20 2253 #define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT) 2254 #define MIDR_EL1_IMP_SHIFT 24 2255 #define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT) 2256 2257 #define MIDR_FIJI (0x002 << MIDR_EL1_PNUM_SHIFT) 2258 #define MIDR_CAPRI (0x003 << MIDR_EL1_PNUM_SHIFT) 2259 #define MIDR_MAUI (0x004 << MIDR_EL1_PNUM_SHIFT) 2260 #define MIDR_ELBA (0x005 << MIDR_EL1_PNUM_SHIFT) 2261 #define MIDR_CAYMAN (0x006 << MIDR_EL1_PNUM_SHIFT) 2262 #define MIDR_MYST (0x007 << MIDR_EL1_PNUM_SHIFT) 2263 #define MIDR_SKYE_MONSOON (0x008 << MIDR_EL1_PNUM_SHIFT) 2264 #define MIDR_SKYE_MISTRAL (0x009 << MIDR_EL1_PNUM_SHIFT) 2265 #define MIDR_CYPRUS_VORTEX (0x00B << MIDR_EL1_PNUM_SHIFT) 2266 #define MIDR_CYPRUS_TEMPEST (0x00C << MIDR_EL1_PNUM_SHIFT) 2267 #define MIDR_M9 (0x00F << MIDR_EL1_PNUM_SHIFT) 2268 #define MIDR_ARUBA_VORTEX (0x010 << MIDR_EL1_PNUM_SHIFT) 2269 #define MIDR_ARUBA_TEMPEST (0x011 << MIDR_EL1_PNUM_SHIFT) 2270 2271 #ifdef APPLELIGHTNING 2272 #define MIDR_CEBU_LIGHTNING (0x012 << MIDR_EL1_PNUM_SHIFT) 2273 #define MIDR_CEBU_THUNDER (0x013 << MIDR_EL1_PNUM_SHIFT) 2274 #define MIDR_TURKS (0x026 << MIDR_EL1_PNUM_SHIFT) 2275 #endif 2276 2277 #ifdef APPLEFIRESTORM 2278 #define MIDR_SICILY_ICESTORM (0x020 << MIDR_EL1_PNUM_SHIFT) 2279 #define MIDR_SICILY_FIRESTORM (0x021 << MIDR_EL1_PNUM_SHIFT) 2280 #define MIDR_TONGA_ICESTORM (0x022 << MIDR_EL1_PNUM_SHIFT) 2281 #define MIDR_TONGA_FIRESTORM (0x023 << MIDR_EL1_PNUM_SHIFT) 2282 #define MIDR_JADE_CHOP_ICESTORM (0x024 << MIDR_EL1_PNUM_SHIFT) 2283 #define MIDR_JADE_CHOP_FIRESTORM (0x025 << MIDR_EL1_PNUM_SHIFT) 2284 #define MIDR_JADE_DIE_ICESTORM (0x028 << MIDR_EL1_PNUM_SHIFT) 2285 #define MIDR_JADE_DIE_FIRESTORM (0x029 << MIDR_EL1_PNUM_SHIFT) 2286 #endif 2287 2288 #ifdef APPLEAVALANCHE 2289 #define MIDR_ELLIS_BLIZZARD (0x030 << MIDR_EL1_PNUM_SHIFT) 2290 #define MIDR_ELLIS_AVALANCHE (0x031 << MIDR_EL1_PNUM_SHIFT) 2291 #endif 2292 #define MIDR_STATEN_BLIZZARD (0x032 << MIDR_EL1_PNUM_SHIFT) 2293 #define MIDR_STATEN_AVALANCHE (0x033 << MIDR_EL1_PNUM_SHIFT) 2294 #define MIDR_RHODES_CHOP_BLIZZARD (0x034 << MIDR_EL1_PNUM_SHIFT) 2295 #define MIDR_RHODES_CHOP_AVALANCHE (0x035 << MIDR_EL1_PNUM_SHIFT) 2296 #define MIDR_RHODES_DIE_BLIZZARD (0x038 << MIDR_EL1_PNUM_SHIFT) 2297 #define MIDR_RHODES_DIE_AVALANCHE (0x039 << MIDR_EL1_PNUM_SHIFT) 2298 2299 #if defined(APPLEEVEREST) 2300 #define MIDR_CRETE_SAWTOOTH (0x040 << MIDR_EL1_PNUM_SHIFT) 2301 #define MIDR_CRETE_EVEREST (0x041 << MIDR_EL1_PNUM_SHIFT) 2302 #define MIDR_IBIZA_ACCE (0x042 << MIDR_EL1_PNUM_SHIFT) 2303 #define MIDR_IBIZA_ACCP (0x043 << MIDR_EL1_PNUM_SHIFT) 2304 #define MIDR_LOBOS_ACCE (0x044 << MIDR_EL1_PNUM_SHIFT) 2305 #define MIDR_LOBOS_ACCP (0x045 << MIDR_EL1_PNUM_SHIFT) 2306 #define MIDR_CAICOS_ACCE (0x046 << MIDR_EL1_PNUM_SHIFT) 2307 #define MIDR_PALMA_ACCE (0x048 << MIDR_EL1_PNUM_SHIFT) 2308 #define MIDR_PALMA_ACCP (0x049 << MIDR_EL1_PNUM_SHIFT) 2309 #define MIDR_COLL_ACCE (0x050 << MIDR_EL1_PNUM_SHIFT) 2310 #define MIDR_COLL_ACCP (0x051 << MIDR_EL1_PNUM_SHIFT) 2311 #endif /* defined(APPLEEVEREST) */ 2312 2313 /*Donan*/ 2314 #define MIDR_DONAN_ACCE (0x052 << MIDR_EL1_PNUM_SHIFT) 2315 #define MIDR_DONAN_ACCP (0x053 << MIDR_EL1_PNUM_SHIFT) 2316 /*Brava*/ 2317 #define MIDR_BRAVA_ACCE (0x054 << MIDR_EL1_PNUM_SHIFT) 2318 #define MIDR_BRAVA_ACCP (0x055 << MIDR_EL1_PNUM_SHIFT) 2319 2320 2321 /* 2322 * Apple-ISA-Extensions ID Register. 2323 */ 2324 #define AIDR_MUL53 (1ULL << 0) 2325 #define AIDR_WKDM (1ULL << 1) 2326 #define AIDR_ARCHRETENTION (1ULL << 2) 2327 2328 2329 2330 2331 /* 2332 * CoreSight debug registers 2333 */ 2334 #define CORESIGHT_ED 0 2335 #define CORESIGHT_CTI 1 2336 #define CORESIGHT_PMU 2 2337 #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */ 2338 2339 #define CORESIGHT_OFFSET(x) ((x) * 0x10000) 2340 #define CORESIGHT_REGIONS 4 2341 #define CORESIGHT_SIZE 0x1000 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 /* 2353 * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0 2354 * 2355 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2356 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ 2357 * | rndr | tlb | ts | fhm | dp | sm4 | sm3 | sha3 | rdm | res0 | atomic | crc32 | sha2 | sha1 | aes | res0 | 2358 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ 2359 */ 2360 2361 #define ID_AA64ISAR0_EL1_TS_OFFSET 52 2362 #define ID_AA64ISAR0_EL1_TS_MASK (0xfull << ID_AA64ISAR0_EL1_TS_OFFSET) 2363 #define ID_AA64ISAR0_EL1_TS_FLAGM_EN (1ull << ID_AA64ISAR0_EL1_TS_OFFSET) 2364 #define ID_AA64ISAR0_EL1_TS_FLAGM2_EN (2ull << ID_AA64ISAR0_EL1_TS_OFFSET) 2365 2366 #define ID_AA64ISAR0_EL1_FHM_OFFSET 48 2367 #define ID_AA64ISAR0_EL1_FHM_MASK (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET) 2368 #define ID_AA64ISAR0_EL1_FHM_8_2 (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET) 2369 2370 #define ID_AA64ISAR0_EL1_DP_OFFSET 44 2371 #define ID_AA64ISAR0_EL1_DP_MASK (0xfull << ID_AA64ISAR0_EL1_DP_OFFSET) 2372 #define ID_AA64ISAR0_EL1_DP_EN (1ull << ID_AA64ISAR0_EL1_DP_OFFSET) 2373 2374 #define ID_AA64ISAR0_EL1_SHA3_OFFSET 32 2375 #define ID_AA64ISAR0_EL1_SHA3_MASK (0xfull << ID_AA64ISAR0_EL1_SHA3_OFFSET) 2376 #define ID_AA64ISAR0_EL1_SHA3_EN (1ull << ID_AA64ISAR0_EL1_SHA3_OFFSET) 2377 2378 #define ID_AA64ISAR0_EL1_RDM_OFFSET 28 2379 #define ID_AA64ISAR0_EL1_RDM_MASK (0xfull << ID_AA64ISAR0_EL1_RDM_OFFSET) 2380 #define ID_AA64ISAR0_EL1_RDM_EN (1ull << ID_AA64ISAR0_EL1_RDM_OFFSET) 2381 2382 #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20 2383 #define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) 2384 #define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) 2385 2386 #define ID_AA64ISAR0_EL1_CRC32_OFFSET 16 2387 #define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET) 2388 #define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET) 2389 2390 #define ID_AA64ISAR0_EL1_SHA2_OFFSET 12 2391 #define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 2392 #define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 2393 #define ID_AA64ISAR0_EL1_SHA2_512_EN (2ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 2394 2395 #define ID_AA64ISAR0_EL1_SHA1_OFFSET 8 2396 #define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET) 2397 #define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET) 2398 2399 #define ID_AA64ISAR0_EL1_AES_OFFSET 4 2400 #define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET) 2401 #define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET) 2402 #define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET) 2403 2404 /* 2405 * ID_AA64ISAR1_EL1 - AArch64 Instruction Set Attribute Register 1 2406 * 2407 * 63 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2408 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ 2409 * | res0 | i8mm | dgh | bf16 | specres | sb | frintts | gpi | gpa | lrcpc | fcma | jscvt | api | apa | dpb | 2410 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ 2411 */ 2412 2413 #define ID_AA64ISAR1_EL1_I8MM_OFFSET 52 2414 #define ID_AA64ISAR1_EL1_I8MM_MASK (0xfull << ID_AA64ISAR1_EL1_I8MM_OFFSET) 2415 #define ID_AA64ISAR1_EL1_I8MM_EN (1ull << ID_AA64ISAR1_EL1_I8MM_OFFSET) 2416 2417 #define ID_AA64ISAR1_EL1_DGH_OFFSET 48 2418 #define ID_AA64ISAR1_EL1_DGH_MASK (0xfull << ID_AA64ISAR1_EL1_DGH_OFFSET) 2419 2420 #define ID_AA64ISAR1_EL1_BF16_OFFSET 44 2421 #define ID_AA64ISAR1_EL1_BF16_MASK (0xfull << ID_AA64ISAR1_EL1_BF16_OFFSET) 2422 #define ID_AA64ISAR1_EL1_BF16_EN (1ull << ID_AA64ISAR1_EL1_BF16_OFFSET) 2423 #define ID_AA64ISAR1_EL1_EBF16_EN (2ull << ID_AA64ISAR1_EL1_BF16_OFFSET) 2424 2425 #define ID_AA64ISAR1_EL1_SPECRES_OFFSET 40 2426 #define ID_AA64ISAR1_EL1_SPECRES_MASK (0xfull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 2427 #define ID_AA64ISAR1_EL1_SPECRES_EN (1ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 2428 #define ID_AA64ISAR1_EL1_SPECRES2_EN (2ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 2429 2430 #define ID_AA64ISAR1_EL1_SB_OFFSET 36 2431 #define ID_AA64ISAR1_EL1_SB_MASK (0xfull << ID_AA64ISAR1_EL1_SB_OFFSET) 2432 #define ID_AA64ISAR1_EL1_SB_EN (1ull << ID_AA64ISAR1_EL1_SB_OFFSET) 2433 2434 #define ID_AA64ISAR1_EL1_FRINTTS_OFFSET 32 2435 #define ID_AA64ISAR1_EL1_FRINTTS_MASK (0xfull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) 2436 #define ID_AA64ISAR1_EL1_FRINTTS_EN (1ull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) 2437 2438 #define ID_AA64ISAR1_EL1_GPI_OFFSET 28 2439 #define ID_AA64ISAR1_EL1_GPI_MASK (0xfull << ID_AA64ISAR1_EL1_GPI_OFFSET) 2440 #define ID_AA64ISAR1_EL1_GPI_EN (1ull << ID_AA64ISAR1_EL1_GPI_OFFSET) 2441 2442 #define ID_AA64ISAR1_EL1_GPA_OFFSET 24 2443 #define ID_AA64ISAR1_EL1_GPA_MASK (0xfull << ID_AA64ISAR1_EL1_GPA_OFFSET) 2444 2445 #define ID_AA64ISAR1_EL1_LRCPC_OFFSET 20 2446 #define ID_AA64ISAR1_EL1_LRCPC_MASK (0xfull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 2447 #define ID_AA64ISAR1_EL1_LRCPC_EN (1ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 2448 #define ID_AA64ISAR1_EL1_LRCP2C_EN (2ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 2449 2450 #define ID_AA64ISAR1_EL1_FCMA_OFFSET 16 2451 #define ID_AA64ISAR1_EL1_FCMA_MASK (0xfull << ID_AA64ISAR1_EL1_FCMA_OFFSET) 2452 #define ID_AA64ISAR1_EL1_FCMA_EN (1ull << ID_AA64ISAR1_EL1_FCMA_OFFSET) 2453 2454 #define ID_AA64ISAR1_EL1_JSCVT_OFFSET 12 2455 #define ID_AA64ISAR1_EL1_JSCVT_MASK (0xfull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) 2456 #define ID_AA64ISAR1_EL1_JSCVT_EN (1ull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) 2457 2458 #define ID_AA64ISAR1_EL1_API_OFFSET 8 2459 #define ID_AA64ISAR1_EL1_API_MASK (0xfull << ID_AA64ISAR1_EL1_API_OFFSET) 2460 #define ID_AA64ISAR1_EL1_API_PAuth_EN (1ull << ID_AA64ISAR1_EL1_API_OFFSET) 2461 #define ID_AA64ISAR1_EL1_API_PAuth2_EN (3ull << ID_AA64ISAR1_EL1_API_OFFSET) 2462 #define ID_AA64ISAR1_EL1_API_FPAC_EN (4ull << ID_AA64ISAR1_EL1_API_OFFSET) 2463 #define ID_AA64ISAR1_EL1_API_FPACCOMBINE (5ull << ID_AA64ISAR1_EL1_API_OFFSET) 2464 2465 #define ID_AA64ISAR1_EL1_APA_OFFSET 4 2466 #define ID_AA64ISAR1_EL1_APA_MASK (0xfull << ID_AA64ISAR1_EL1_APA_OFFSET) 2467 2468 #define ID_AA64ISAR1_EL1_DPB_OFFSET 0 2469 #define ID_AA64ISAR1_EL1_DPB_MASK (0xfull << ID_AA64ISAR1_EL1_DPB_OFFSET) 2470 #define ID_AA64ISAR1_EL1_DPB_EN (1ull << ID_AA64ISAR1_EL1_DPB_OFFSET) 2471 #define ID_AA64ISAR1_EL1_DPB2_EN (2ull << ID_AA64ISAR1_EL1_DPB_OFFSET) 2472 2473 /* 2474 * ID_AA64ISAR2_EL1 - AArch64 Instruction Set Attribute Register 2 2475 * 2476 * 63 56 55 52 51 24 23 20 19 8 7 4 3 0 2477 * +------+------+------+------+-------+-------+------+ 2478 * | res2 | CSSC | res1 | BC | res0 | RPRES | WFxT | 2479 * +------+------+------+------+-------+-------+------+ 2480 */ 2481 2482 #define ID_AA64ISAR2_EL1_CSSC_OFFSET 52 2483 #define ID_AA64ISAR2_EL1_CSSC_MASK (0xfull << ID_AA64ISAR2_EL1_CSSC_OFFSET) 2484 #define ID_AA64ISAR2_EL1_CSSC_EN (1ull << ID_AA64ISAR2_EL1_CSSC_OFFSET) 2485 2486 #define ID_AA64ISAR2_EL1_BC_OFFSET 20 2487 #define ID_AA64ISAR2_EL1_BC_MASK (0xfull << ID_AA64ISAR2_EL1_BC_OFFSET) 2488 #define ID_AA64ISAR2_EL1_BC_EN (1ull << ID_AA64ISAR2_EL1_BC_OFFSET) 2489 2490 #define ID_AA64ISAR2_EL1_RPRES_OFFSET 4 2491 #define ID_AA64ISAR2_EL1_RPRES_MASK (0xfull << ID_AA64ISAR2_EL1_RPRES_OFFSET) 2492 #define ID_AA64ISAR2_EL1_RPRES_EN (1ull << ID_AA64ISAR2_EL1_RPRES_OFFSET) 2493 2494 #define ID_AA64ISAR2_EL1_WFxT_OFFSET 0 2495 #define ID_AA64ISAR2_EL1_WFxT_MASK (0xfull << ID_AA64ISAR2_EL1_WFxT_OFFSET) 2496 #define ID_AA64ISAR2_EL1_WFxT_EN (1ull << ID_AA64ISAR2_EL1_WFxT_OFFSET) 2497 2498 /* 2499 * ID_AA64MMFR0_EL1 - AArch64 Memory Model Feature Register 0 2500 * 63 60 59 56 55 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2501 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ 2502 * | ECV | FGT | RES0 | ExS | TGran4_2 | TGran64_2 | TGran16_2 | TGran4 | TGran64 | TGran16 | BigEndEL0 | SNSMem | BigEnd | ASIDBits | PARange | 2503 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ 2504 */ 2505 2506 #define ID_AA64MMFR0_EL1_ECV_OFFSET 60 2507 #define ID_AA64MMFR0_EL1_ECV_MASK (0xfull << ID_AA64MMFR0_EL1_ECV_OFFSET) 2508 #define ID_AA64MMFR0_EL1_ECV_EN (1ull << ID_AA64MMFR0_EL1_ECV_OFFSET) 2509 2510 /* 2511 * ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2 2512 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 14 8 7 4 3 0 2513 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ 2514 * | E0PD | EVT | BBM | TTL | RES0 | FWB | IDS | AT | ST | NV | CCIDX | VARANGE | IESB | LSM | UAO | CnP | 2515 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ 2516 */ 2517 2518 #define ID_AA64MMFR2_EL1_AT_OFFSET 32 2519 #define ID_AA64MMFR2_EL1_AT_MASK (0xfull << ID_AA64MMFR2_EL1_AT_OFFSET) 2520 #define ID_AA64MMFR2_EL1_AT_LSE2_EN (1ull << ID_AA64MMFR2_EL1_AT_OFFSET) 2521 #define ID_AA64MMFR2_EL1_VARANGE_OFFSET 16 2522 #define ID_AA64MMFR2_EL1_VARANGE_MASK (0xfull << ID_AA64MMFR2_EL1_VARANGE_OFFSET) 2523 2524 /* 2525 * ID_AA64PFR0_EL1 - AArch64 Processor Feature Register 0 2526 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2527 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ 2528 * | CSV3 | CSV2 | RES0 | DIT | AMU | MPAM | SEL2 | SVE | RAS | GIC | AdvSIMD | FP | EL3 | EL2 | EL1 | EL0 | 2529 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ 2530 */ 2531 2532 #define ID_AA64PFR0_EL1_CSV3_OFFSET 60 2533 #define ID_AA64PFR0_EL1_CSV3_MASK (0xfull << ID_AA64PFR0_EL1_CSV3_OFFSET) 2534 #define ID_AA64PFR0_EL1_CSV3_EN (1ull << ID_AA64PFR0_EL1_CSV3_OFFSET) 2535 2536 #define ID_AA64PFR0_EL1_CSV2_OFFSET 56 2537 #define ID_AA64PFR0_EL1_CSV2_MASK (0xfull << ID_AA64PFR0_EL1_CSV2_OFFSET) 2538 #define ID_AA64PFR0_EL1_CSV2_EN (1ull << ID_AA64PFR0_EL1_CSV2_OFFSET) 2539 #define ID_AA64PFR0_EL1_CSV2_2 (2ull << ID_AA64PFR0_EL1_CSV2_OFFSET) 2540 2541 #define ID_AA64PFR0_EL1_DIT_OFFSET 48 2542 #define ID_AA64PFR0_EL1_DIT_MASK (0xfull << ID_AA64PFR0_EL1_DIT_OFFSET) 2543 #define ID_AA64PFR0_EL1_DIT_EN (1ull << ID_AA64PFR0_EL1_DIT_OFFSET) 2544 2545 #define ID_AA64PFR0_EL1_AdvSIMD_OFFSET 20 2546 #define ID_AA64PFR0_EL1_AdvSIMD_MASK (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 2547 #define ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT (0x0ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 2548 #define ID_AA64PFR0_EL1_AdvSIMD_FP16 (0x1ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 2549 #define ID_AA64PFR0_EL1_AdvSIMD_DIS (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 2550 2551 /* 2552 * ID_AA64PFR1_EL1 - AArch64 Processor Feature Register 1 2553 * 63 20 19 16 15 12 11 8 7 4 3 0 2554 * +----------------------------------+-----------+----------+-------+------+------+ 2555 * | RES0 | MPAM_frac | RAS_frac | MTE | SSBS | BT | 2556 * +----------------------------------+-----------+----------+-------+------+------+ 2557 */ 2558 2559 2560 2561 2562 #define ID_AA64PFR1_EL1_SME_OFFSET 24 2563 #define ID_AA64PFR1_EL1_SME_MASK (0xfull << ID_AA64PFR1_EL1_SME_OFFSET) 2564 #define ID_AA64PFR1_EL1_SME_EN (1ull << ID_AA64PFR1_EL1_SME_OFFSET) 2565 #define ID_AA64PFR1_EL1_CSV2_frac_OFFSET 32 2566 #define ID_AA64PFR1_EL1_CSV2_frac_MASK (0xfull << ID_AA64PFR1_EL1_CSV2_frac_OFFSET) 2567 #define ID_AA64PFR1_EL1_CSV2_frac_1p1 (1ull << ID_AA64PFR1_EL1_CSV2_frac_OFFSET) 2568 #define ID_AA64PFR1_EL1_CSV2_frac_1p2 (2ull << ID_AA64PFR1_EL1_CSV2_frac_OFFSET) 2569 2570 2571 #define ID_AA64PFR1_EL1_SSBS_OFFSET 4 2572 #define ID_AA64PFR1_EL1_SSBS_MASK (0xfull << ID_AA64PFR1_EL1_SSBS_OFFSET) 2573 #define ID_AA64PFR1_EL1_SSBS_EN (1ull << ID_AA64PFR1_EL1_SSBS_OFFSET) 2574 2575 #define ID_AA64PFR1_EL1_BT_OFFSET 0 2576 #define ID_AA64PFR1_EL1_BT_MASK (0xfull << ID_AA64PFR1_EL1_BT_OFFSET) 2577 #define ID_AA64PFR1_EL1_BT_EN (1ull << ID_AA64PFR1_EL1_BT_OFFSET) 2578 2579 /* 2580 * ID_AA64PFR2_EL1 - AArch64 Processor Feature Register 2 2581 */ 2582 2583 2584 2585 2586 /* 2587 * ID_AA64MMFR1_EL1 - AArch64 Memory Model Feature Register 1 2588 * 2589 * 63 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 2590 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ 2591 * | res0 | nTLBPA | AFP | HCX | ETS | TWED | XNX | SpecSEI | PAN | LO | HPDS | VH | VMIDBits | HAFDBS | 2592 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ 2593 */ 2594 2595 #define ID_AA64MMFR1_EL1_AFP_OFFSET 44 2596 #define ID_AA64MMFR1_EL1_AFP_MASK (0xfull << ID_AA64MMFR1_EL1_AFP_OFFSET) 2597 #define ID_AA64MMFR1_EL1_AFP_EN (1ull << ID_AA64MMFR1_EL1_AFP_OFFSET) 2598 2599 #define ID_AA64MMFR1_EL1_HCX_OFFSET 40 2600 #define ID_AA64MMFR1_EL1_HCX_MASK (0xfull << ID_AA64MMFR1_EL1_HCX_OFFSET) 2601 #define ID_AA64MMFR1_EL1_HCX_EN (1ull << ID_AA64MMFR1_EL1_HCX_OFFSET) 2602 2603 /* 2604 * ID_AA64SMFR0_EL1 - SME Feature ID Register 0 2605 * 2606 * 63 62 60 59 56 55 52 51 49 48 47 44 43 40 39 36 35 34 33 32 31 0 2607 * +------+------+--------+--------+------+--------+--------+------+-------+--------+--------+---------+--------+------+ 2608 * | FA64 | res0 | SMEver | I16I64 | res0 | F64F64 | I16I32 | res0 | I8I32 | F16F32 | B16F32 | BI32I32 | F32F32 | res0 | 2609 * +------+------+--------+--------+------+--------+--------+------+-------+--------+--------+---------+--------+------+ 2610 */ 2611 2612 #define ID_AA64SMFR0_EL1_SMEver_OFFSET 56 2613 #define ID_AA64SMFR0_EL1_SMEver_MASK (0xfull << ID_AA64SMFR0_EL1_SMEver_OFFSET) 2614 #define ID_AA64SMFR0_EL1_SMEver_SME (0ull << ID_AA64SMFR0_EL1_SMEver_OFFSET) 2615 #define ID_AA64SMFR0_EL1_SMEver_SME2 (1ull << ID_AA64SMFR0_EL1_SMEver_OFFSET) 2616 2617 #define ID_AA64SMFR0_EL1_I16I64_OFFSET 52 2618 #define ID_AA64SMFR0_EL1_I16I64_MASK (0xfull << ID_AA64SMFR0_EL1_I16I64_OFFSET) 2619 #define ID_AA64SMFR0_EL1_I16I64_EN (0xfull << ID_AA64SMFR0_EL1_I16I64_OFFSET) 2620 2621 #define ID_AA64SMFR0_EL1_F64F64_OFFSET 48 2622 #define ID_AA64SMFR0_EL1_F64F64_MASK (1ull << ID_AA64SMFR0_EL1_F64F64_OFFSET) 2623 #define ID_AA64SMFR0_EL1_F64F64_EN (1ull << ID_AA64SMFR0_EL1_F64F64_OFFSET) 2624 2625 #define ID_AA64SMFR0_EL1_I16I32_OFFSET 44 2626 #define ID_AA64SMFR0_EL1_I16I32_MASK (0xfull << ID_AA64SMFR0_EL1_I16I32_OFFSET) 2627 #define ID_AA64SMFR0_EL1_I16I32_EN (0x5ull << ID_AA64SMFR0_EL1_I16I32_OFFSET) 2628 2629 2630 #define ID_AA64SMFR0_EL1_I8I32_OFFSET 36 2631 #define ID_AA64SMFR0_EL1_I8I32_MASK (0xfull << ID_AA64SMFR0_EL1_I8I32_OFFSET) 2632 #define ID_AA64SMFR0_EL1_I8I32_EN (0xfull << ID_AA64SMFR0_EL1_I8I32_OFFSET) 2633 2634 #define ID_AA64SMFR0_EL1_F16F32_OFFSET 35 2635 #define ID_AA64SMFR0_EL1_F16F32_MASK (1ull << ID_AA64SMFR0_EL1_F16F32_OFFSET) 2636 #define ID_AA64SMFR0_EL1_F16F32_EN (1ull << ID_AA64SMFR0_EL1_F16F32_OFFSET) 2637 2638 #define ID_AA64SMFR0_EL1_B16F32_OFFSET 34 2639 #define ID_AA64SMFR0_EL1_B16F32_MASK (1ull << ID_AA64SMFR0_EL1_B16F32_OFFSET) 2640 #define ID_AA64SMFR0_EL1_B16F32_EN (1ull << ID_AA64SMFR0_EL1_B16F32_OFFSET) 2641 2642 #define ID_AA64SMFR0_EL1_BI32I32_OFFSET 33 2643 #define ID_AA64SMFR0_EL1_BI32I32_MASK (1ull << ID_AA64SMFR0_EL1_BI32I32_OFFSET) 2644 #define ID_AA64SMFR0_EL1_BI32I32_EN (1ull << ID_AA64SMFR0_EL1_BI32I32_OFFSET) 2645 2646 #define ID_AA64SMFR0_EL1_F32F32_OFFSET 32 2647 #define ID_AA64SMFR0_EL1_F32F32_MASK (1ull << ID_AA64SMFR0_EL1_F32F32_OFFSET) 2648 #define ID_AA64SMFR0_EL1_F32F32_EN (1ull << ID_AA64SMFR0_EL1_F32F32_OFFSET) 2649 2650 2651 2652 2653 #define APSTATE_G_SHIFT (0) 2654 #define APSTATE_P_SHIFT (1) 2655 #define APSTATE_A_SHIFT (2) 2656 #define APSTATE_AP_MASK ((1ULL << APSTATE_A_SHIFT) | (1ULL << APSTATE_P_SHIFT)) 2657 2658 2659 #define ACTLR_EL1_EnTSO (1ULL << 1) 2660 #define ACTLR_EL1_EnAPFLG (1ULL << 4) 2661 #define ACTLR_EL1_EnAFP (1ULL << 5) 2662 #define ACTLR_EL1_EnPRSV (1ULL << 6) 2663 2664 2665 #if HAS_USAT_BIT 2666 #define ACTLR_EL1_USAT_OFFSET 0 2667 #define ACTLR_EL1_USAT_MASK (1ULL << ACTLR_EL1_USAT_OFFSET) 2668 #define ACTLR_EL1_USAT ACTLR_EL1_USAT_MASK 2669 #endif 2670 2671 2672 2673 2674 2675 2676 #ifdef HAS_DISDDHWP0 2677 #define ACTLR_EL1_DisDDHWP0_OFFSET 17 2678 #define ACTLR_EL1_DisDDHWP0_MASK (1ULL << ACTLR_EL1_DisDDHWP0_OFFSET) 2679 #define ACTLR_EL1_DisDDHWP0 ACTLR_EL1_DisDDHWP0_MASK 2680 #endif /* HAS_DISDDDHWP0 */ 2681 2682 2683 #if defined(HAS_APPLE_PAC) 2684 // The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code 2685 #define PAC_DISCRIMINATOR_RECOVER 0x1e02 2686 #endif 2687 2688 2689 #define CTR_EL0_L1Ip_OFFSET 14 2690 #define CTR_EL0_L1Ip_VIPT (2ULL << CTR_EL0_L1Ip_OFFSET) 2691 #define CTR_EL0_L1Ip_PIPT (3ULL << CTR_EL0_L1Ip_OFFSET) 2692 #define CTR_EL0_L1Ip_MASK (3ULL << CTR_EL0_L1Ip_OFFSET) 2693 2694 2695 #ifdef __ASSEMBLER__ 2696 2697 /* 2698 * Conditionally write to system/special-purpose register. 2699 * The register is written to only when the first two arguments 2700 * do not match. If they do match, the macro jumps to a 2701 * caller-provided label. 2702 * The _ISB variant also conditionally issues an ISB after the MSR. 2703 * 2704 * $0 - System/special-purpose register to modify 2705 * $1 - Register containing current FPCR value 2706 * $2 - Register containing expected value 2707 * $3 - Label to jump to when register is already set to expected value 2708 */ 2709 .macro CMSR 2710 cmp $1, $2 2711 2712 /* Skip expensive MSR if not required */ 2713 b.eq $3f 2714 msr $0, $2 2715 .endmacro 2716 2717 .macro CMSR_ISB 2718 CMSR $0, $1, $2, $3 2719 isb sy 2720 .endmacro 2721 2722 /* 2723 * Modify FPCR only if it does not contain the XNU default value. 2724 * $0 - Register containing current FPCR value 2725 * $1 - Scratch register 2726 * $2 - Label to jump to when FPCR is already set to default value 2727 */ 2728 .macro SANITIZE_FPCR 2729 mov $1, #FPCR_DEFAULT 2730 CMSR FPCR, $0, $1, $2 2731 .endmacro 2732 2733 /* 2734 * Family of macros that can be used to protect code sections such that they 2735 * are only executed on a particular SoC/Revision/CPU, and skipped otherwise. 2736 * All macros will forward-jump to 1f when the condition is not matched. 2737 * This label may be defined manually, or implicitly through the use of 2738 * the EXEC_END macro. 2739 * For cores, XX can be: EQ (equal), ALL (don't care). 2740 * For revisions, XX can be: EQ (equal), LO (lower than), HS (higher or same), ALL (don't care). 2741 */ 2742 2743 /* 2744 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX 2745 * $1 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2746 * $2 - GPR containing MIDR_EL1 value 2747 * $3 - Scratch register 2748 */ 2749 .macro EXEC_COREEQ_REVEQ 2750 and $3, $2, #MIDR_EL1_PNUM_MASK 2751 cmp $3, $0 2752 b.ne 1f 2753 2754 mov $3, $2 2755 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2756 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2757 cmp $3, $1 2758 b.ne 1f 2759 .endmacro 2760 2761 .macro EXEC_COREEQ_REVLO 2762 and $3, $2, #MIDR_EL1_PNUM_MASK 2763 cmp $3, $0 2764 b.ne 1f 2765 2766 mov $3, $2 2767 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2768 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2769 cmp $3, $1 2770 b.pl 1f 2771 .endmacro 2772 2773 .macro EXEC_COREEQ_REVHS 2774 and $3, $2, #MIDR_EL1_PNUM_MASK 2775 cmp $3, $0 2776 b.ne 1f 2777 2778 mov $3, $2 2779 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 2780 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 2781 cmp $3, $1 2782 b.mi 1f 2783 .endmacro 2784 2785 /* 2786 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2787 * $1 - GPR containing MIDR_EL1 value 2788 * $2 - Scratch register 2789 */ 2790 .macro EXEC_COREALL_REVEQ 2791 mov $2, $1 2792 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2793 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2794 cmp $2, $0 2795 b.ne 1f 2796 .endmacro 2797 2798 .macro EXEC_COREALL_REVLO 2799 mov $2, $1 2800 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2801 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2802 cmp $2, $0 2803 b.pl 1f 2804 .endmacro 2805 2806 .macro EXEC_COREALL_REVHS 2807 mov $2, $1 2808 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2809 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2810 cmp $2, $0 2811 b.mi 1f 2812 .endmacro 2813 2814 .macro CMP_FOREACH reg, cc, label, car, cdr:vararg 2815 cmp \reg, \car 2816 b.\cc \label 2817 .ifnb \cdr 2818 CMP_FOREACH \reg, \cc, \label, \cdr 2819 .endif 2820 .endm 2821 2822 .macro EXEC_COREIN_REVALL midr_el1, scratch, midr_list:vararg 2823 and \scratch, \midr_el1, #MIDR_EL1_PNUM_MASK 2824 CMP_FOREACH \scratch, eq, Lmatch\@, \midr_list 2825 b 1f 2826 Lmatch\@: 2827 .endm 2828 2829 /* 2830 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX 2831 * $1 - GPR containing MIDR_EL1 value 2832 * $2 - Scratch register 2833 */ 2834 .macro EXEC_COREEQ_REVALL 2835 and $2, $1, #MIDR_EL1_PNUM_MASK 2836 cmp $2, $0 2837 b.ne 1f 2838 .endmacro 2839 2840 /* 2841 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 2842 * $1 - GPR containing MIDR_EL1 value 2843 * $2 - Scratch register 2844 */ 2845 .macro EXEC_PCORE_REVEQ 2846 ARM64_IS_PCORE $2 2847 cbz $2, 1f 2848 2849 mov $2, $1 2850 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2851 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2852 cmp $2, $0 2853 b.ne 1f 2854 .endmacro 2855 2856 .macro EXEC_PCORE_REVLO 2857 ARM64_IS_PCORE $2 2858 cbz $2, 1f 2859 2860 mov $2, $1 2861 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2862 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2863 cmp $2, $0 2864 b.pl 1f 2865 .endmacro 2866 2867 .macro EXEC_PCORE_REVHS 2868 ARM64_IS_PCORE $2 2869 cbz $2, 1f 2870 2871 mov $2, $1 2872 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2873 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2874 cmp $2, $0 2875 b.mi 1f 2876 .endmacro 2877 2878 .macro EXEC_ECORE_REVEQ 2879 ARM64_IS_ECORE $2 2880 cbz $2, 1f 2881 2882 mov $2, $1 2883 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2884 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2885 cmp $2, $0 2886 b.ne 1f 2887 .endmacro 2888 2889 .macro EXEC_ECORE_REVLO 2890 ARM64_IS_ECORE $2 2891 cbz $2, 1f 2892 2893 mov $2, $1 2894 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2895 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2896 cmp $2, $0 2897 b.pl 1f 2898 .endmacro 2899 2900 .macro EXEC_ECORE_REVHS 2901 ARM64_IS_ECORE $2 2902 cbz $2, 1f 2903 2904 mov $2, $1 2905 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 2906 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 2907 cmp $2, $0 2908 b.mi 1f 2909 .endmacro 2910 2911 /* 2912 * $0 - GPR containing MIDR_EL1 value 2913 * $1 - Scratch register 2914 */ 2915 .macro EXEC_PCORE_REVALL 2916 ARM64_IS_PCORE $1 2917 cbz $1, 1f 2918 .endmacro 2919 2920 .macro EXEC_ECORE_REVALL 2921 ARM64_IS_ECORE $1 2922 cbz $1, 1f 2923 .endmacro 2924 2925 /* 2926 * Macro that defines the label that all EXEC_COREXX_REVXX macros jump to. 2927 */ 2928 .macro EXEC_END 2929 1: 2930 .endmacro 2931 2932 /* 2933 * Wedges CPUs with a specified core that are below a specified revision. This 2934 * macro is intended for CPUs that have been deprecated in iBoot and may have 2935 * incorrect behavior if they continue running xnu. 2936 */ 2937 .macro DEPRECATE_COREEQ_REVLO core, rev, midr_el1, scratch 2938 EXEC_COREEQ_REVLO \core, \rev, \midr_el1, \scratch 2939 /* BEGIN IGNORE CODESTYLE */ 2940 b . 2941 /* END IGNORE CODESTYLE */ 2942 EXEC_END 2943 .endmacro 2944 2945 /* 2946 * Sets bits in an SPR register. 2947 * arg0: Name of the register to be accessed. 2948 * arg1: Mask of bits to be set. 2949 * arg2: Scratch register 2950 */ 2951 .macro HID_SET_BITS 2952 mrs $2, $0 2953 orr $2, $2, $1 2954 msr $0, $2 2955 .endmacro 2956 2957 /* 2958 * Clears bits in an SPR register. 2959 * arg0: Name of the register to be accessed. 2960 * arg1: Mask of bits to be cleared. 2961 * arg2: Scratch register 2962 */ 2963 .macro HID_CLEAR_BITS 2964 mrs $2, $0 2965 bic $2, $2, $1 2966 msr $0, $2 2967 .endmacro 2968 2969 /* 2970 * Combines the functionality of HID_CLEAR_BITS followed by HID_SET_BITS into 2971 * a single read-modify-write sequence. 2972 * arg0: Name of the register to be accessed. 2973 * arg1: Mask of bits to be cleared. 2974 * arg2: Value to insert 2975 * arg3: Scratch register 2976 */ 2977 .macro HID_INSERT_BITS 2978 mrs $3, $0 2979 bic $3, $3, $1 2980 orr $3, $3, $2 2981 msr $0, $3 2982 .endmacro 2983 2984 /* 2985 * Replaces the value of a field in an implementation-defined system register. 2986 * sreg: system register name 2987 * field: field name within the sysreg, where the assembler symbols 2988 * ARM64_REG_<field>_{shift,width} specify the bounds of the field 2989 * (note that preprocessor macros will not work here) 2990 * value: the value to insert 2991 * scr{1,2}: scratch regs 2992 */ 2993 .macro HID_WRITE_FIELD sreg, field, val, scr1, scr2 2994 mrs \scr1, \sreg 2995 mov \scr2, \val 2996 bfi \scr1, \scr2, ARM64_REG_\sreg\()_\field\()_shift, ARM64_REG_\sreg\()_\field\()_width 2997 msr \sreg, \scr1 2998 .endmacro 2999 3000 /* 3001 * This macro is a replacement for ERET with better security properties. 3002 * 3003 * It prevents "straight-line speculation" (an Arm term) past the ERET. 3004 */ 3005 .macro ERET_NO_STRAIGHT_LINE_SPECULATION 3006 eret 3007 #if __ARM_SB_AVAILABLE__ 3008 sb // Technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures 3009 #else /* __ARM_SB_AVAILABLE__ */ 3010 isb // ISB technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures 3011 nop // Sequence of six NOPs to pad out and terminate instruction decode group */ 3012 nop 3013 nop 3014 nop 3015 nop 3016 nop 3017 #endif /* !__ARM_SB_AVAILABLE__ */ 3018 .endmacro 3019 3020 3021 #endif /* __ASSEMBLER__ */ 3022 3023 #define MSR(reg, src) __asm__ volatile ("msr " reg ", %0" :: "r" (src)) 3024 #define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest)) 3025 3026 #if XNU_MONITOR 3027 #define __ARM_PTE_PHYSMAP__ 1 3028 #define PPL_STATE_KERNEL 0 3029 #define PPL_STATE_DISPATCH 1 3030 #define PPL_STATE_PANIC 2 3031 #define PPL_STATE_EXCEPTION 3 3032 #endif 3033 3034 3035 #endif /* _ARM64_PROC_REG_H_ */ 3036