1 /* 2 * Copyright (c) 2007-2024 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 /* CMU_ENDHIST */ 32 /* 33 * Mach Operating System 34 * Copyright (c) 1991,1990 Carnegie Mellon University 35 * All Rights Reserved. 36 * 37 * Permission to use, copy, modify and distribute this software and its 38 * documentation is hereby granted, provided that both the copyright 39 * notice and this permission notice appear in all copies of the 40 * software, derivative works or modified versions, and any portions 41 * thereof, and that both notices appear in supporting documentation. 42 * 43 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 44 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 45 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 46 * 47 * Carnegie Mellon requests users of this software to return to 48 * 49 * Software Distribution Coordinator or [email protected] 50 * School of Computer Science 51 * Carnegie Mellon University 52 * Pittsburgh PA 15213-3890 53 * 54 * any improvements or extensions that they make and grant Carnegie Mellon 55 * the rights to redistribute these changes. 56 */ 57 58 /* 59 * Processor registers for ARM/ARM64 60 */ 61 #ifndef _ARM64_PROC_REG_H_ 62 #define _ARM64_PROC_REG_H_ 63 64 #if !defined(KERNEL_PRIVATE) && !defined(SPTM_TESTING_PRIVATE) 65 /** 66 * This file is only exported into the internal userspace SDK exclusively for 67 * usage by the SPTM userspace testing system. Let's enforce this by error'ing 68 * the build if an SPTM-specific define is not set. If your userspace project is 69 * not the SPTM testing system, then do not use these files! 70 * 71 * This check does not apply to the kernel itself, or when this file is exported 72 * into Kernel.framework. 73 */ 74 #error This file is only included in the userspace internal SDK for the SPTM project 75 #endif /* !defined(KERNEL_PRIVATE) && !defined(SPTM_TESTING_PRIVATE) */ 76 77 #if defined (__arm64__) 78 #include <pexpert/arm64/board_config.h> 79 #elif defined (__arm__) 80 #include <pexpert/arm/board_config.h> 81 #endif 82 83 #if !CONFIG_SPTM 84 /* 85 * Processor registers for ARM 86 */ 87 #if __ARM_42BIT_PA_SPACE__ 88 /** 89 * On PPL, the identity map requires a smaller T0SZ value because DRAM starts 90 * at a PA not mappable by only 3 bits in L1 table on platforms with 42-bit 91 * PA space. On SPTM, this is overcome by boot with a smaller T0SZ and resize 92 * to the __ARM64_PMAP_SUBPAGE_L1__ T0SZ when the identity map is no longer 93 * used. 94 */ 95 #undef __ARM64_PMAP_SUBPAGE_L1__ 96 #undef __ARM64_PMAP_KERN_SUBPAGE_L1__ 97 #endif /* __ARM_42BIT_PA_SPACE__ */ 98 #endif /* !CONFIG_SPTM */ 99 100 /* For arm platforms, create one pset per cluster */ 101 #define MAX_PSETS MAX_CPU_CLUSTERS 102 103 104 /* Thread groups are enabled on all ARM platforms (irrespective of scheduler) */ 105 #define CONFIG_THREAD_GROUPS 1 106 107 #ifdef XNU_KERNEL_PRIVATE 108 109 #if __ARM_VFP__ 110 #define ARM_VFP_DEBUG 0 111 #endif /* __ARM_VFP__ */ 112 113 #endif /* XNU_KERNEL_PRIVATE */ 114 115 /* 116 * FSR registers 117 * 118 * CPSR: Current Program Status Register 119 * SPSR: Saved Program Status Registers 120 * 121 * 31 30 29 28 27 24 19 16 9 8 7 6 5 4 0 122 * +-----------------------------------------------------------+ 123 * | N| Z| C| V| Q|...| J|...|GE[3:0]|...| E| A| I| F| T| MODE | 124 * +-----------------------------------------------------------+ 125 */ 126 127 /* 128 * Flags 129 */ 130 #define PSR_NF 0x80000000 /* Negative/Less than */ 131 #define PSR_ZF 0x40000000 /* Zero */ 132 #define PSR_CF 0x20000000 /* Carry/Borrow/Extend */ 133 #define PSR_VF 0x10000000 /* Overflow */ 134 135 /* 136 * Modified execution mode flags 137 */ 138 #define PSR_TF 0x00000020 /* thumb flag (BX ARMv4T) */ 139 140 /* 141 * CPU mode 142 */ 143 #define PSR_USER_MODE 0x00000010 /* User mode */ 144 145 #define PSR_MODE_MASK 0x0000001F 146 #define PSR_IS_KERNEL(psr) (((psr) & PSR_MODE_MASK) != PSR_USER_MODE) 147 #define PSR_IS_USER(psr) (((psr) & PSR_MODE_MASK) == PSR_USER_MODE) 148 149 #define PSR_USERDFLT PSR_USER_MODE 150 151 #define PSR_BTYPE_SHIFT (10) 152 #define PSR_BTYPE_MASK (0x3 << PSR_BTYPE_SHIFT) 153 154 /* 155 * Cache configuration 156 */ 157 158 #if defined (APPLETYPHOON) 159 160 /* I-Cache */ 161 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 162 163 /* D-Cache */ 164 #define MMU_CLINE 6 /* cache line size as 1<<MMU_CLINE (64) */ 165 166 #elif defined (APPLETWISTER) 167 168 /* I-Cache */ 169 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 170 171 /* D-Cache */ 172 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 173 174 #elif defined (APPLEHURRICANE) 175 176 /* I-Cache */ 177 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 178 179 /* D-Cache */ 180 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 181 182 #elif defined (APPLEMONSOON) 183 184 /* I-Cache, 96KB for Monsoon, 48KB for Mistral, 6-way. */ 185 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 186 187 /* D-Cache, 64KB for Monsoon, 32KB for Mistral, 4-way. */ 188 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 189 190 #elif defined (APPLEVORTEX) 191 192 /* I-Cache, 128KB 8-way for Vortex, 48KB 6-way for Tempest. */ 193 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 194 195 /* D-Cache, 128KB 8-way for Vortex, 32KB 4-way for Tempest. */ 196 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 197 198 #elif defined (APPLELIGHTNING) 199 200 /* I-Cache, 192KB for Lightning, 96KB for Thunder, 6-way. */ 201 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 202 203 /* D-Cache, 128KB for Lightning, 8-way. 48KB for Thunder, 6-way. */ 204 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 205 206 #elif defined (APPLEFIRESTORM) 207 208 /* I-Cache, 256KB for Firestorm, 128KB for Icestorm, 6-way. */ 209 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 210 211 /* D-Cache, 160KB for Firestorm, 8-way. 64KB for Icestorm, 6-way. */ 212 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 213 214 #elif defined (APPLEAVALANCHE) 215 216 /* I-Cache, 192KB for Avalanche, 128KB for Blizzard, 6-way. */ 217 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 218 219 /* D-Cache, 128KB for Avalanche, 8-way. 64KB for Blizzard, 8-way. */ 220 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 221 222 #elif defined (APPLEEVEREST) 223 224 /* I-Cache, 192KB for Everest, 128KB for SawTooth, 6-way. */ 225 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 226 227 /* D-Cache, 128KB for Everest, 8-way. 64KB for SawTooth, 8-way. */ 228 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 229 230 #elif defined (APPLEH16) 231 232 /* I-Cache, 192KB for AppleH16 PCore, 128KB for ECore, 6-way. */ 233 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 234 235 /* D-Cache, 128KB for AppleH16 PCore, 8-way. 64KB for ECore, 8-way. */ 236 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 237 238 #elif defined (APPLEACC8) 239 240 /* I-Cache, 192KB for Acc8 PCore, 128KB for ECore, 6-way. */ 241 #define MMU_I_CLINE 6 /* cache line size as 1<<MMU_I_CLINE (64) */ 242 243 /* D-Cache, 128KB for Acc8 PCore, 8-way. 64KB for ECore, 8-way. */ 244 #define MMU_CLINE 6 /* cache line size is 1<<MMU_CLINE (64) */ 245 246 #elif defined (VMAPPLE) 247 248 /* I-Cache. */ 249 #define MMU_I_CLINE 6 250 251 /* D-Cache. */ 252 #define MMU_CLINE 6 253 254 #else 255 #error processor not supported 256 #endif 257 258 #define MAX_L2_CLINE_BYTES (1 << MAX_L2_CLINE) 259 260 /* 261 * Format of the Debug & Watchpoint Breakpoint Value and Control Registers 262 */ 263 #define ARM_DBG_VR_ADDRESS_MASK 0xFFFFFFFC /* BVR & WVR */ 264 #define ARM_DBG_VR_ADDRESS_MASK64 0xFFFFFFFFFFFFFFFCull /* BVR & WVR */ 265 266 #define ARM_DBG_CR_ADDRESS_MASK_MASK 0x1F000000 /* BCR & WCR */ 267 #define ARM_DBGBCR_MATCH_MASK (1 << 22) /* BCR only */ 268 #define ARM_DBGBCR_TYPE_MASK (1 << 21) /* BCR only */ 269 #define ARM_DBGBCR_TYPE_IVA (0 << 21) 270 #define ARM_DBG_CR_LINKED_MASK (1 << 20) /* BCR & WCR */ 271 #define ARM_DBG_CR_LINKED_UNLINKED (0 << 20) 272 #define ARM_DBG_CR_SECURITY_STATE_BOTH (0 << 14) 273 #define ARM_DBG_CR_HIGHER_MODE_ENABLE (1 << 13) 274 #define ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK 0x00001FE0 /* WCR only */ 275 #define ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK 0x000001E0 /* BCR & WCR */ 276 #define ARM_DBGWCR_ACCESS_CONTROL_MASK (3 << 3) /* WCR only */ 277 #define ARM_DBG_CR_MODE_CONTROL_PRIVILEGED (1 << 1) /* BCR & WCR */ 278 #define ARM_DBG_CR_MODE_CONTROL_USER (2 << 1) /* BCR & WCR */ 279 #define ARM_DBG_CR_ENABLE_MASK (1 << 0) /* BCR & WCR */ 280 #define ARM_DBG_CR_ENABLE_ENABLE (1 << 0) 281 282 /* 283 * Format of the OS Lock Access (DBGOSLAR) and Lock Access Registers (DBGLAR) 284 */ 285 #define ARM_DBG_LOCK_ACCESS_KEY 0xC5ACCE55 286 287 /* ARM Debug registers of interest */ 288 #define ARM_DEBUG_OFFSET_DBGPRCR (0x310) 289 #define ARM_DEBUG_OFFSET_DBGLAR (0xFB0) 290 291 /* 292 * Main ID Register (MIDR) 293 * 294 * 31 24 23 20 19 16 15 4 3 0 295 * +-----+-----+------+------+-----+ 296 * | IMP | VAR | ARCH | PNUM | REV | 297 * +-----+-----+------+------+-----+ 298 * 299 * where: 300 * IMP: Implementor code 301 * VAR: Variant number 302 * ARCH: Architecture code 303 * PNUM: Primary part number 304 * REV: Minor revision number 305 */ 306 #define MIDR_REV_SHIFT 0 307 #define MIDR_REV_MASK (0xf << MIDR_REV_SHIFT) 308 #define MIDR_VAR_SHIFT 20 309 #define MIDR_VAR_MASK (0xf << MIDR_VAR_SHIFT) 310 311 312 #if __ARM_KERNEL_PROTECT__ 313 /* 314 * __ARM_KERNEL_PROTECT__ is a feature intended to guard against potential 315 * architectural or microarchitectural vulnerabilities that could allow cores to 316 * read/access EL1-only mappings while in EL0 mode. This is achieved by 317 * removing as many mappings as possible when the core transitions to EL0 mode 318 * from EL1 mode, and restoring those mappings when the core transitions to EL1 319 * mode from EL0 mode. 320 * 321 * At the moment, this is achieved through use of ASIDs and TCR_EL1. TCR_EL1 is 322 * used to map and unmap the ordinary kernel mappings, by contracting and 323 * expanding translation zone size for TTBR1 when exiting and entering EL1, 324 * respectively: 325 * 326 * Kernel EL0 Mappings: TTBR1 mappings that must remain mapped while the core is 327 * is in EL0. 328 * Kernel EL1 Mappings: TTBR1 mappings that must be mapped while the core is in 329 * EL1. 330 * 331 * T1SZ_USER: T1SZ_BOOT + 1 332 * TTBR1_EL1_BASE_BOOT: (2^64) - (2^(64 - T1SZ_BOOT) 333 * TTBR1_EL1_BASE_USER: (2^64) - (2^(64 - T1SZ_USER) 334 * TTBR1_EL1_MAX: (2^64) - 1 335 * 336 * When in EL1, we program TCR_EL1 (specifically, TCR_EL1.T1SZ) to give the 337 * the following TTBR1 layout: 338 * 339 * TTBR1_EL1_BASE_BOOT TTBR1_EL1_BASE_USER TTBR1_EL1_MAX 340 * +---------------------------------------------------------+ 341 * | Kernel EL0 Mappings | Kernel EL1 Mappings | 342 * +---------------------------------------------------------+ 343 * 344 * And when in EL0, we program TCR_EL1 to give the following TTBR1 layout: 345 * 346 * TTBR1_EL1_BASE_USER TTBR1_EL1_MAX 347 * +---------------------------------------------------------+ 348 * | Kernel EL0 Mappings | 349 * +---------------------------------------------------------+ 350 * 351 * With the current implementation, both the EL0 and EL1 mappings for the kernel 352 * use otherwise empty translation tables for mapping the exception vectors (so 353 * that we do not need to TLB flush the exception vector address when switching 354 * between EL0 and EL1). The rationale here is that the TLBI would require a 355 * DSB, and DSBs can be extremely expensive. 356 * 357 * Each pmap is given two ASIDs: (n & ~1) as an EL0 ASID, and (n | 1) as an EL1 358 * ASID. The core switches between ASIDs on EL transitions, so that the TLB 359 * does not need to be fully invalidated on an EL transition. 360 * 361 * Most kernel mappings will be marked non-global in this configuration, as 362 * global mappings would be visible to userspace unless we invalidate them on 363 * eret. 364 */ 365 #if XNU_MONITOR 366 /* 367 * Please note that because we indirect through the thread register in order to 368 * locate the kernel, and because we unmap most of the kernel, the security 369 * model of the PPL is undermined by __ARM_KERNEL_PROTECT__, as we rely on 370 * kernel controlled data to direct codeflow in the exception vectors. 371 * 372 * If we want to ship XNU_MONITOR paired with __ARM_KERNEL_PROTECT__, we will 373 * need to find a performant solution to this problem. 374 */ 375 #endif 376 #endif /* __ARM_KERNEL_PROTECT */ 377 378 #if ARM_PARAMETERIZED_PMAP 379 /* 380 * ARM_PARAMETERIZED_PMAP configures the kernel to get the characteristics of 381 * the page tables (number of levels, size of the root allocation) from the 382 * pmap data structure, rather than treating them as compile-time constants. 383 * This allows the pmap code to dynamically adjust how it deals with page 384 * tables. 385 */ 386 #endif /* ARM_PARAMETERIZED_PMAP */ 387 388 #if __ARM_MIXED_PAGE_SIZE__ 389 /* 390 * __ARM_MIXED_PAGE_SIZE__ configures the kernel to support page tables that do 391 * not use the kernel page size. This is primarily meant to support running 392 * 4KB page processes on a 16KB page kernel. 393 * 394 * This only covers support in the pmap/machine dependent layers. Any support 395 * elsewhere in the kernel must be managed separately. 396 */ 397 #if !ARM_PARAMETERIZED_PMAP 398 /* 399 * Page tables that use non-kernel page sizes require us to reprogram TCR based 400 * on the page tables we are switching to. This means that the parameterized 401 * pmap support is required. 402 */ 403 #error __ARM_MIXED_PAGE_SIZE__ requires ARM_PARAMETERIZED_PMAP 404 #endif /* !ARM_PARAMETERIZED_PMAP */ 405 #if __ARM_KERNEL_PROTECT__ 406 /* 407 * Because switching the page size requires updating TCR based on the pmap, and 408 * __ARM_KERNEL_PROTECT__ relies on TCR being programmed with constants, XNU 409 * does not currently support support configurations that use both 410 * __ARM_KERNEL_PROTECT__ and __ARM_MIXED_PAGE_SIZE__. 411 */ 412 #error __ARM_MIXED_PAGE_SIZE__ and __ARM_KERNEL_PROTECT__ are mutually exclusive 413 #endif /* __ARM_KERNEL_PROTECT__ */ 414 #endif /* __ARM_MIXED_PAGE_SIZE__ */ 415 416 /* 417 * 64-bit Program Status Register (PSR64) 418 * 419 * 31 27 23 22 21 20 19 10 9 5 4 0 420 * +-+-+-+-+-----+---+--+--+----------+-+-+-+-+-+-----+ 421 * |N|Z|C|V|00000|PAN|SS|IL|0000000000|D|A|I|F|0| M | 422 * +-+-+-+-+-+---+---+--+--+----------+-+-+-+-+-+-----+ 423 * 424 * where: 425 * NZCV: Comparison flags 426 * PAN: Privileged Access Never 427 * SS: Single step 428 * IL: Illegal state 429 * DAIF: Interrupt masks 430 * M: Mode field 431 */ 432 433 #define PSR64_NZCV_SHIFT 28 434 #define PSR64_NZCV_WIDTH 4 435 #define PSR64_NZCV_MASK (0xF << PSR64_NZCV_SHIFT) 436 437 #define PSR64_N_SHIFT 31 438 #define PSR64_N (1 << PSR64_N_SHIFT) 439 440 #define PSR64_Z_SHIFT 30 441 #define PSR64_Z (1 << PSR64_Z_SHIFT) 442 443 #define PSR64_C_SHIFT 29 444 #define PSR64_C (1 << PSR64_C_SHIFT) 445 446 #define PSR64_V_SHIFT 28 447 #define PSR64_V (1 << PSR64_V_SHIFT) 448 449 #define PSR64_TCO_SHIFT 25 450 #define PSR64_TCO (1 << PSR64_TCO_SHIFT) 451 452 #define PSR64_DIT_SHIFT 24 453 #define PSR64_DIT (1 << PSR64_DIT_SHIFT) 454 455 #define PSR64_UAO_SHIFT 23 456 #define PSR64_UAO (1 << PSR64_UAO_SHIFT) 457 458 #define PSR64_PAN_SHIFT 22 459 #define PSR64_PAN (1 << PSR64_PAN_SHIFT) 460 461 #define PSR64_SS_SHIFT 21 462 #define PSR64_SS (1 << PSR64_SS_SHIFT) 463 464 #define PSR64_IL_SHIFT 20 465 #define PSR64_IL (1 << PSR64_IL_SHIFT) 466 467 /* 468 * SSBS is bit 12 for A64 SPSR and bit 23 for A32 SPSR 469 * I do not want to talk about it! 470 */ 471 #define PSR64_SSBS_SHIFT_32 23 472 #define PSR64_SSBS_SHIFT_64 12 473 #define PSR64_SSBS_32 (1 << PSR64_SSBS_SHIFT_32) 474 #define PSR64_SSBS_64 (1 << PSR64_SSBS_SHIFT_64) 475 476 /* 477 * msr DAIF, Xn and mrs Xn, DAIF transfer into 478 * and out of bits 9:6 479 */ 480 #define DAIF_DEBUG_SHIFT 9 481 #define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT) 482 483 #define DAIF_ASYNC_SHIFT 8 484 #define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT) 485 486 #define DAIF_IRQF_SHIFT 7 487 #define DAIF_IRQF (1 << DAIF_IRQF_SHIFT) 488 489 #define DAIF_FIQF_SHIFT 6 490 #define DAIF_FIQF (1 << DAIF_FIQF_SHIFT) 491 492 #define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) 493 #define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF) 494 495 #define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF)) 496 497 #if HAS_ARM_FEAT_SSBS2 498 #define PSR64_SSBS_U32_DEFAULT PSR64_SSBS_32 499 #define PSR64_SSBS_U64_DEFAULT PSR64_SSBS_64 500 #define PSR64_SSBS_KRN_DEFAULT PSR64_SSBS_64 501 #else 502 #define PSR64_SSBS_U32_DEFAULT (0) 503 #define PSR64_SSBS_U64_DEFAULT (0) 504 #define PSR64_SSBS_KRN_DEFAULT (0) 505 #endif 506 507 /* 508 * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer 509 * from bits 3:0. 510 */ 511 #define DAIFSC_DEBUGF (1 << 3) 512 #define DAIFSC_ASYNCF (1 << 2) 513 #define DAIFSC_IRQF (1 << 1) 514 #define DAIFSC_FIQF (1 << 0) 515 #define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) 516 #define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF) 517 #define DAIFSC_NOASYNC (DAIFSC_DEBUGF | DAIFSC_IRQF | DAIFSC_FIQF) 518 519 /* 520 * ARM64_TODO: unify with ARM? 521 */ 522 #define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */ 523 524 #define PSR64_MODE_MASK 0x1F 525 526 #define PSR64_USER_MASK PSR64_NZCV_MASK 527 528 #define PSR64_MODE_USER32_THUMB 0x20 529 530 #define PSR64_MODE_RW_SHIFT 4 531 #define PSR64_MODE_RW_64 0 532 #define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT) 533 534 #define PSR64_MODE_EL_SHIFT 2 535 #define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT) 536 #define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT) 537 #define PSR64_MODE_EL2 (0x2 << PSR64_MODE_EL_SHIFT) 538 #define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT) 539 #define PSR64_MODE_EL0 0 540 541 #define PSR64_MODE_EL_KERNEL (PSR64_MODE_EL1) 542 543 #define PSR64_MODE_SPX 0x1 544 #define PSR64_MODE_SP0 0 545 546 #define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U32_DEFAULT) 547 #define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0 | PSR64_SSBS_U64_DEFAULT) 548 #define PSR64_KERNEL_STANDARD (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0 | PSR64_SSBS_KRN_DEFAULT) 549 #if __ARM_PAN_AVAILABLE__ 550 #define PSR64_KERNEL_DEFAULT (PSR64_KERNEL_STANDARD | PSR64_PAN) 551 #else 552 #define PSR64_KERNEL_DEFAULT PSR64_KERNEL_STANDARD 553 #endif 554 555 #define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) > PSR64_MODE_EL0) 556 #define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0) 557 558 #define PSR64_IS_USER32(x) (PSR64_IS_USER(x) && (x & PSR64_MODE_RW_32)) 559 #define PSR64_IS_USER64(x) (PSR64_IS_USER(x) && !(x & PSR64_MODE_RW_32)) 560 561 562 563 /* 564 * System Control Register (SCTLR) 565 */ 566 567 #if HAS_MTE 568 #define SCTLR_TCSO_ENABLED (1ULL << 59) 569 #define SCTLR_TCSO0_ENABLED (1ULL << 58) 570 #endif /* HAS_MTE */ 571 572 #if HAS_ARM_FEAT_SME 573 // 60 EnTP2 Enable TPIDR2_EL0 at EL0 574 #define SCTLR_TP2_ENABLED (1ULL << 60) 575 #endif 576 577 #define SCTLR_EPAN_ENABLED (1ULL << 57) 578 579 #define SCTLR_DSSBS (1ULL << 44) 580 581 #if HAS_MTE 582 583 #define SCTLR_ATA_ENABLED (1ULL << 43) 584 #define SCTLR_ATA0_ENABLED (1ULL << 42) 585 586 #define SCTLR_TCF_SHIFT (40) 587 #define SCTLR_TCF_NOP (0b00ULL << SCTLR_TCF_SHIFT) 588 #define SCTLR_TCF_SYNC (0b01ULL << SCTLR_TCF_SHIFT) 589 #define SCTLR_TCF_ASYNC (0b10ULL << SCTLR_TCF_SHIFT) 590 #define SCTLR_TCF_ASYMM (0b11ULL << SCTLR_TCF_SHIFT) 591 #define SCTLR_TCF_MASK (0b11ULL << SCTLR_TCF_SHIFT) 592 593 #define SCTLR_TCF0_SHIFT (38) 594 #define SCTLR_TCF0_NOP (0b00ULL << SCTLR_TCF0_SHIFT) 595 #define SCTLR_TCF0_SYNC (0b01ULL << SCTLR_TCF0_SHIFT) 596 #define SCTLR_TCF0_ASYNC (0b10ULL << SCTLR_TCF0_SHIFT) 597 #define SCTLR_TCF0_ASYMM (0b11ULL << SCTLR_TCF0_SHIFT) 598 #define SCTLR_TCF0_MASK (0b11ULL << SCTLR_TCF0_SHIFT) 599 600 #define SCTLR_EXTRA (SCTLR_ATA_ENABLED | SCTLR_ATA0_ENABLED | SCTLR_TCF_SYNC | SCTLR_TCF0_SYNC) 601 #define SCTLR_MTE_CONFIG SCTLR_EXTRA 602 603 #else /* !HAS_MTE */ 604 605 #define SCTLR_EXTRA (0) 606 607 #endif /* HAS_MTE */ 608 609 #define SCTLR_RESERVED ((3ULL << 28) | (1ULL << 20)) 610 #if defined(HAS_APPLE_PAC) 611 612 // 31 PACIA_ENABLED AddPACIA and AuthIA functions enabled 613 #define SCTLR_PACIA_ENABLED_SHIFT 31 614 #define SCTLR_PACIA_ENABLED (1ULL << SCTLR_PACIA_ENABLED_SHIFT) 615 // 30 PACIB_ENABLED AddPACIB and AuthIB functions enabled 616 #define SCTLR_PACIB_ENABLED (1ULL << 30) 617 // 29:28 RES1 11 618 // 27 PACDA_ENABLED AddPACDA and AuthDA functions enabled 619 #define SCTLR_PACDA_ENABLED (1ULL << 27) 620 // 13 PACDB_ENABLED AddPACDB and AuthDB functions enabled 621 #define SCTLR_PACDB_ENABLED (1ULL << 13) 622 623 #define SCTLR_PAC_KEYS_ENABLED (SCTLR_PACIA_ENABLED | SCTLR_PACIB_ENABLED | SCTLR_PACDA_ENABLED | SCTLR_PACDB_ENABLED) 624 #endif /* defined(HAS_APPLE_PAC) */ 625 626 // 36 BT1 PACIxSP acts as a BTI C landing pad rather than BTI JC at EL1 627 #define SCTLR_BT1_ENABLED (1ULL << 36) 628 629 // 35 BT0 PACIxSP acts as a BTI C landing pad rather than BTI JC at EL0 630 #define SCTLR_BT0_ENABLED (1ULL << 35) 631 632 633 // 26 UCI User Cache Instructions 634 #define SCTLR_UCI_ENABLED (1ULL << 26) 635 636 // 25 EE Exception Endianness 637 #define SCTLR_EE_BIG_ENDIAN (1ULL << 25) 638 639 // 24 E0E EL0 Endianness 640 #define SCTLR_E0E_BIG_ENDIAN (1ULL << 24) 641 642 // 23 SPAN Set PAN 643 #define SCTLR_PAN_UNCHANGED (1ULL << 23) 644 645 // 22 EIS Taking an exception is a context synchronization event 646 #define SCTLR_EIS (1ULL << 22) 647 648 // 21 RES0 0 649 // 20 RES1 1 650 651 // 19 WXN Writeable implies eXecute Never 652 #define SCTLR_WXN_ENABLED (1ULL << 19) 653 654 // 18 nTWE Not trap WFE from EL0 655 #define SCTLR_nTWE_WFE_ENABLED (1ULL << 18) 656 657 // 17 RES0 0 658 659 // 16 nTWI Not trap WFI from EL0 660 #define SCTRL_nTWI_WFI_ENABLED (1ULL << 16) 661 662 // 15 UCT User Cache Type register (CTR_EL0) 663 #define SCTLR_UCT_ENABLED (1ULL << 15) 664 665 // 14 DZE User Data Cache Zero (DC ZVA) 666 #define SCTLR_DZE_ENABLED (1ULL << 14) 667 668 // 12 I Instruction cache enable 669 #define SCTLR_I_ENABLED (1ULL << 12) 670 671 // 11 EOS Exception return is a context synchronization event 672 #define SCTLR_EOS (1ULL << 11) 673 674 // 10 EnRCTX EL0 Access to FEAT_SPECRES speculation restriction instructions 675 #define SCTLR_EnRCTX (1ULL << 10) 676 677 // 9 UMA User Mask Access 678 #define SCTLR_UMA_ENABLED (1ULL << 9) 679 680 // 8 SED SETEND Disable 681 #define SCTLR_SED_DISABLED (1ULL << 8) 682 683 // 7 ITD IT Disable 684 #define SCTLR_ITD_DISABLED (1ULL << 7) 685 686 // 6 RES0 0 687 688 // 5 CP15BEN CP15 Barrier ENable 689 #define SCTLR_CP15BEN_ENABLED (1ULL << 5) 690 691 // 4 SA0 Stack Alignment check for EL0 692 #define SCTLR_SA0_ENABLED (1ULL << 4) 693 694 // 3 SA Stack Alignment check 695 #define SCTLR_SA_ENABLED (1ULL << 3) 696 697 // 2 C Cache enable 698 #define SCTLR_C_ENABLED (1ULL << 2) 699 700 // 1 A Alignment check 701 #define SCTLR_A_ENABLED (1ULL << 1) 702 703 // 0 M MMU enable 704 #define SCTLR_M_ENABLED (1ULL << 0) 705 706 #if APPLEVIRTUALPLATFORM 707 #define SCTLR_EPAN_DEFAULT 0 708 /* xnu tries to set SCTLR_EL1.EPAN = 1, but it may be RaZ/WI on some hosts */ 709 #define SCTLR_EPAN_OPTIONAL SCTLR_EPAN_ENABLED 710 #elif HAS_ARM_FEAT_PAN3 711 #define SCTLR_EPAN_DEFAULT SCTLR_EPAN_ENABLED 712 #define SCTLR_EPAN_OPTIONAL 0 713 #else 714 #define SCTLR_EPAN_DEFAULT 0 715 #define SCTLR_EPAN_OPTIONAL 0 716 #endif 717 718 #if __ARM_ARCH_8_5__ 719 #define SCTLR_EIS_DEFAULT (0) 720 #define SCTLR_DSSBS_DEFAULT SCTLR_DSSBS 721 #else 722 #define SCTLR_EIS_DEFAULT (SCTLR_EIS) 723 #define SCTLR_DSSBS_DEFAULT (0) 724 #endif 725 726 #if ERET_IS_NOT_CONTEXT_SYNCHRONIZING 727 #define SCTLR_EOS_DEFAULT (0) 728 #else 729 #define SCTLR_EOS_DEFAULT (SCTLR_EOS) 730 #endif 731 732 #if HAS_APPLE_PAC 733 #define SCTLR_PAC_KEYS_DEFAULT SCTLR_PAC_KEYS_ENABLED 734 #else /* !HAS_APPLE_PAC */ 735 #define SCTLR_PAC_KEYS_DEFAULT 0 736 #endif 737 738 #if BTI_ENFORCED 739 /* In the kernel, we want PACIxSP to behave only as a BTI C */ 740 #define SCTLR_BT_DEFAULT SCTLR_BT1_ENABLED 741 #else 742 #define SCTLR_BT_DEFAULT 0 743 #endif /* BTI_ENFORCED */ 744 745 #if HAS_ARM_FEAT_SME 746 #define SCTLR_TP2_DEFAULT SCTLR_TP2_ENABLED 747 #else 748 #define SCTLR_TP2_DEFAULT 0 749 #endif 750 751 #define SCTLR_OTHER 0 752 753 #define SCTLR_EL1_REQUIRED \ 754 (SCTLR_RESERVED | SCTLR_UCI_ENABLED | SCTLR_nTWE_WFE_ENABLED | SCTLR_DZE_ENABLED | \ 755 SCTLR_I_ENABLED | SCTLR_SED_DISABLED | SCTLR_CP15BEN_ENABLED | SCTLR_BT_DEFAULT | \ 756 SCTLR_SA0_ENABLED | SCTLR_SA_ENABLED | SCTLR_C_ENABLED | SCTLR_M_ENABLED | \ 757 SCTLR_EPAN_DEFAULT | SCTLR_EIS_DEFAULT | SCTLR_EOS_DEFAULT | SCTLR_DSSBS_DEFAULT | \ 758 SCTLR_PAC_KEYS_DEFAULT | SCTLR_TP2_DEFAULT | SCTLR_OTHER) 759 760 #define SCTLR_EL1_OPTIONAL \ 761 (SCTLR_EPAN_OPTIONAL) 762 763 #define SCTLR_EL1_DEFAULT \ 764 (SCTLR_EL1_REQUIRED | SCTLR_EL1_OPTIONAL) 765 766 767 /* 768 * Coprocessor Access Control Register (CPACR) 769 * 770 * 31 28 27 22 21 20 19 0 771 * +---+---+------+------+--------------------+ 772 * |000|TTA|000000| FPEN |00000000000000000000| 773 * +---+---+------+------+--------------------+ 774 * 775 * where: 776 * TTA: Trace trap 777 * FPEN: Floating point enable 778 */ 779 #define CPACR_TTA_SHIFT 28 780 #define CPACR_TTA (1 << CPACR_TTA_SHIFT) 781 782 #if HAS_ARM_FEAT_SME 783 #define CPACR_SMEN_SHIFT 24 784 #define CPACR_SMEN_MASK (0x3 << CPACR_SMEN_SHIFT) 785 #define CPACR_SMEN_EL0_TRAP (0x1 << CPACR_SMEN_SHIFT) 786 #define CPACR_SMEN_ENABLE (0x3 << CPACR_SMEN_SHIFT) 787 #endif /* HAS_ARM_FEAT_SME */ 788 789 #define CPACR_FPEN_SHIFT 20 790 #define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT) 791 #define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT) 792 793 #if HAS_ARM_FEAT_SME 794 #define CPACR_ZEN_SHIFT 16 795 #define CPACR_ZEN_MASK (0x3 << CPACR_ZEN_SHIFT) 796 #define CPACR_ZEN_EL0_TRAP (0x1 << CPACR_ZEN_SHIFT) 797 #define CPACR_ZEN_ENABLE (0x3 << CPACR_ZEN_SHIFT) 798 #endif /* HAS_ARM_FEAT_SME */ 799 800 /* 801 * FPSR: Floating Point Status Register 802 * 803 * 31 30 29 28 27 26 7 6 4 3 2 1 0 804 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ 805 * | N| Z| C| V|QC|0000000000000000000|IDC|00|IXC|UFC|OFC|DZC|IOC| 806 * +--+--+--+--+--+-------------------+---+--+---+---+---+---+---+ 807 */ 808 809 #define FPSR_N_SHIFT 31 810 #define FPSR_Z_SHIFT 30 811 #define FPSR_C_SHIFT 29 812 #define FPSR_V_SHIFT 28 813 #define FPSR_QC_SHIFT 27 814 #define FPSR_IDC_SHIFT 7 815 #define FPSR_IXC_SHIFT 4 816 #define FPSR_UFC_SHIFT 3 817 #define FPSR_OFC_SHIFT 2 818 #define FPSR_DZC_SHIFT 1 819 #define FPSR_IOC_SHIFT 0 820 #define FPSR_N (1 << FPSR_N_SHIFT) 821 #define FPSR_Z (1 << FPSR_Z_SHIFT) 822 #define FPSR_C (1 << FPSR_C_SHIFT) 823 #define FPSR_V (1 << FPSR_V_SHIFT) 824 #define FPSR_QC (1 << FPSR_QC_SHIFT) 825 #define FPSR_IDC (1 << FPSR_IDC_SHIFT) 826 #define FPSR_IXC (1 << FPSR_IXC_SHIFT) 827 #define FPSR_UFC (1 << FPSR_UFC_SHIFT) 828 #define FPSR_OFC (1 << FPSR_OFC_SHIFT) 829 #define FPSR_DZC (1 << FPSR_DZC_SHIFT) 830 #define FPSR_IOC (1 << FPSR_IOC_SHIFT) 831 832 /* 833 * A mask for all for all of the bits that are not RAZ for FPSR; this 834 * is primarily for converting between a 32-bit view of NEON state 835 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). 836 */ 837 #define FPSR_MASK \ 838 (FPSR_N | FPSR_Z | FPSR_C | FPSR_V | FPSR_QC | FPSR_IDC | FPSR_IXC | \ 839 FPSR_UFC | FPSR_OFC | FPSR_DZC | FPSR_IOC) 840 841 /* 842 * FPCR: Floating Point Control Register 843 * 844 * 31 26 25 24 23 21 19 18 15 14 12 11 10 9 8 7 0 845 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ 846 * |00000|AHP|DN|FZ|RMODE|STRIDE| 0|LEN|IDE|00|IXE|UFE|OFE|DZE|IOE|00000000| 847 * +-----+---+--+--+-----+------+--+---+---+--+---+---+---+---+---+--------+ 848 */ 849 850 #define FPCR_AHP_SHIFT 26 851 #define FPCR_DN_SHIFT 25 852 #define FPCR_FZ_SHIFT 24 853 #define FPCR_RMODE_SHIFT 22 854 #define FPCR_STRIDE_SHIFT 20 855 #define FPCR_LEN_SHIFT 16 856 #define FPCR_IDE_SHIFT 15 857 #define FPCR_IXE_SHIFT 12 858 #define FPCR_UFE_SHIFT 11 859 #define FPCR_OFE_SHIFT 10 860 #define FPCR_DZE_SHIFT 9 861 #define FPCR_IOE_SHIFT 8 862 #define FPCR_AHP (1 << FPCR_AHP_SHIFT) 863 #define FPCR_DN (1 << FPCR_DN_SHIFT) 864 #define FPCR_FZ (1 << FPCR_FZ_SHIFT) 865 #define FPCR_RMODE (0x3 << FPCR_RMODE_SHIFT) 866 #define FPCR_STRIDE (0x3 << FPCR_STRIDE_SHIFT) 867 #define FPCR_LEN (0x7 << FPCR_LEN_SHIFT) 868 #define FPCR_IDE (1 << FPCR_IDE_SHIFT) 869 #define FPCR_IXE (1 << FPCR_IXE_SHIFT) 870 #define FPCR_UFE (1 << FPCR_UFE_SHIFT) 871 #define FPCR_OFE (1 << FPCR_OFE_SHIFT) 872 #define FPCR_DZE (1 << FPCR_DZE_SHIFT) 873 #define FPCR_IOE (1 << FPCR_IOE_SHIFT) 874 #define FPCR_DEFAULT (0) 875 #define FPCR_DEFAULT_32 (FPCR_DN|FPCR_FZ) 876 877 /* 878 * A mask for all for all of the bits that are not RAZ for FPCR; this 879 * is primarily for converting between a 32-bit view of NEON state 880 * (FPSCR) and a 64-bit view of NEON state (FPSR, FPCR). 881 */ 882 #define FPCR_MASK \ 883 (FPCR_AHP | FPCR_DN | FPCR_FZ | FPCR_RMODE | FPCR_STRIDE | FPCR_LEN | \ 884 FPCR_IDE | FPCR_IXE | FPCR_UFE | FPCR_OFE | FPCR_DZE | FPCR_IOE) 885 886 /* 887 * Translation Control Register (TCR) 888 * 889 * Legacy: 890 * 891 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 892 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ 893 * | zero |TBI1|TBI0|AS|z| IPS |z|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |z|TG0| SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| 894 * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+ 895 * 896 * Current (with 16KB granule support): 897 * 898 * 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0 899 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ 900 * | zero |TBI1|TBI0|AS|z| IPS | TG1 | SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ | TG0 | SH0 |ORGN0|IRGN0|EPD0|z|T0SZ| 901 * +------+----+----+--+-+-----+-----+-----+-----+-----+----+--+------+-----+-----+-----+-----+----+-+----+ 902 * 903 * TBI1: Top Byte Ignored for TTBR1 region 904 * TBI0: Top Byte Ignored for TTBR0 region 905 * AS: ASID Size 906 * IPS: Physical Address Size limit 907 * TG1: Granule Size for TTBR1 region 908 * SH1: Shareability for TTBR1 region 909 * ORGN1: Outer Cacheability for TTBR1 region 910 * IRGN1: Inner Cacheability for TTBR1 region 911 * EPD1: Translation table walk disable for TTBR1 912 * A1: ASID selection from TTBR1 enable 913 * T1SZ: Virtual address size for TTBR1 914 * TG0: Granule Size for TTBR0 region 915 * SH0: Shareability for TTBR0 region 916 * ORGN0: Outer Cacheability for TTBR0 region 917 * IRGN0: Inner Cacheability for TTBR0 region 918 * T0SZ: Virtual address size for TTBR0 919 */ 920 921 #define TCR_T0SZ_SHIFT 0ULL 922 #define TCR_T0SZ_MASK 0x3FULL 923 #define TCR_TSZ_BITS 6ULL 924 #define TCR_TSZ_MASK ((1ULL << TCR_TSZ_BITS) - 1ULL) 925 926 #define TCR_IRGN0_SHIFT 8ULL 927 #define TCR_IRGN0_DISABLED (0ULL << TCR_IRGN0_SHIFT) 928 #define TCR_IRGN0_WRITEBACK (1ULL << TCR_IRGN0_SHIFT) 929 #define TCR_IRGN0_WRITETHRU (2ULL << TCR_IRGN0_SHIFT) 930 #define TCR_IRGN0_WRITEBACKNO (3ULL << TCR_IRGN0_SHIFT) 931 932 #define TCR_ORGN0_SHIFT 10ULL 933 #define TCR_ORGN0_DISABLED (0ULL << TCR_ORGN0_SHIFT) 934 #define TCR_ORGN0_WRITEBACK (1ULL << TCR_ORGN0_SHIFT) 935 #define TCR_ORGN0_WRITETHRU (2ULL << TCR_ORGN0_SHIFT) 936 #define TCR_ORGN0_WRITEBACKNO (3ULL << TCR_ORGN0_SHIFT) 937 938 #define TCR_SH0_SHIFT 12ULL 939 #define TCR_SH0_NONE (0ULL << TCR_SH0_SHIFT) 940 #define TCR_SH0_OUTER (2ULL << TCR_SH0_SHIFT) 941 #define TCR_SH0_INNER (3ULL << TCR_SH0_SHIFT) 942 943 #define TCR_TG0_GRANULE_SHIFT (14ULL) 944 #define TCR_TG0_GRANULE_BITS (2ULL) 945 #define TCR_TG0_GRANULE_MASK ((1ULL << TCR_TG0_GRANULE_BITS) - 1ULL) 946 947 #define TCR_TG0_GRANULE_4KB (0ULL << TCR_TG0_GRANULE_SHIFT) 948 #define TCR_TG0_GRANULE_64KB (1ULL << TCR_TG0_GRANULE_SHIFT) 949 #define TCR_TG0_GRANULE_16KB (2ULL << TCR_TG0_GRANULE_SHIFT) 950 951 #if __ARM_16K_PG__ 952 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_16KB) 953 #else 954 #define TCR_TG0_GRANULE_SIZE (TCR_TG0_GRANULE_4KB) 955 #endif 956 957 #define TCR_T1SZ_SHIFT 16ULL 958 #define TCR_T1SZ_MASK 0x3FULL 959 960 #define TCR_A1_ASID1 (1ULL << 22ULL) 961 #define TCR_EPD1_TTBR1_DISABLED (1ULL << 23ULL) 962 963 #define TCR_IRGN1_SHIFT 24ULL 964 #define TCR_IRGN1_DISABLED (0ULL << TCR_IRGN1_SHIFT) 965 #define TCR_IRGN1_WRITEBACK (1ULL << TCR_IRGN1_SHIFT) 966 #define TCR_IRGN1_WRITETHRU (2ULL << TCR_IRGN1_SHIFT) 967 #define TCR_IRGN1_WRITEBACKNO (3ULL << TCR_IRGN1_SHIFT) 968 969 #define TCR_ORGN1_SHIFT 26ULL 970 #define TCR_ORGN1_DISABLED (0ULL << TCR_ORGN1_SHIFT) 971 #define TCR_ORGN1_WRITEBACK (1ULL << TCR_ORGN1_SHIFT) 972 #define TCR_ORGN1_WRITETHRU (2ULL << TCR_ORGN1_SHIFT) 973 #define TCR_ORGN1_WRITEBACKNO (3ULL << TCR_ORGN1_SHIFT) 974 975 #define TCR_SH1_SHIFT 28ULL 976 #define TCR_SH1_NONE (0ULL << TCR_SH1_SHIFT) 977 #define TCR_SH1_OUTER (2ULL << TCR_SH1_SHIFT) 978 #define TCR_SH1_INNER (3ULL << TCR_SH1_SHIFT) 979 980 #define TCR_TG1_GRANULE_SHIFT 30ULL 981 #define TCR_TG1_GRANULE_BITS (2ULL) 982 #define TCR_TG1_GRANULE_MASK ((1ULL << TCR_TG1_GRANULE_BITS) - 1ULL) 983 984 #define TCR_TG1_GRANULE_16KB (1ULL << TCR_TG1_GRANULE_SHIFT) 985 #define TCR_TG1_GRANULE_4KB (2ULL << TCR_TG1_GRANULE_SHIFT) 986 #define TCR_TG1_GRANULE_64KB (3ULL << TCR_TG1_GRANULE_SHIFT) 987 988 #if __ARM_16K_PG__ 989 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_16KB) 990 #else 991 #define TCR_TG1_GRANULE_SIZE (TCR_TG1_GRANULE_4KB) 992 #endif 993 994 #define TCR_IPS_SHIFT 32ULL 995 #define TCR_IPS_BITS 3ULL 996 #define TCR_IPS_MASK ((1ULL << TCR_IPS_BITS) - 1ULL) 997 #define TCR_IPS_32BITS (0ULL << TCR_IPS_SHIFT) 998 #define TCR_IPS_36BITS (1ULL << TCR_IPS_SHIFT) 999 #define TCR_IPS_40BITS (2ULL << TCR_IPS_SHIFT) 1000 #define TCR_IPS_42BITS (3ULL << TCR_IPS_SHIFT) 1001 #define TCR_IPS_44BITS (4ULL << TCR_IPS_SHIFT) 1002 #define TCR_IPS_48BITS (5ULL << TCR_IPS_SHIFT) 1003 1004 #define TCR_AS_16BIT_ASID (1ULL << 36) 1005 #define TCR_TBI0_TOPBYTE_IGNORED (1ULL << 37) 1006 #define TCR_TBI1_TOPBYTE_IGNORED (1ULL << 38) 1007 #define TCR_TBID0_TBI_DATA_ONLY (1ULL << 51) 1008 #define TCR_TBID1_TBI_DATA_ONLY (1ULL << 52) 1009 1010 #if defined(HAS_APPLE_PAC) 1011 #define TCR_TBID0_ENABLE TCR_TBID0_TBI_DATA_ONLY 1012 #define TCR_TBID1_ENABLE TCR_TBID1_TBI_DATA_ONLY 1013 #else 1014 #define TCR_TBID0_ENABLE 0 1015 #define TCR_TBID1_ENABLE 0 1016 #endif 1017 1018 #define TCR_E0PD0_BIT (1ULL << 55) 1019 #define TCR_E0PD1_BIT (1ULL << 56) 1020 1021 #if defined(HAS_E0PD) 1022 #define TCR_E0PD_VALUE (TCR_E0PD1_BIT) 1023 #else 1024 #define TCR_E0PD_VALUE 0 1025 #endif 1026 1027 #if HAS_MTE 1028 1029 #define TCR_MTX0_ENABLE (1ULL << 60) 1030 #define TCR_MTX1_ENABLE (1ULL << 61) 1031 1032 #define TCR_EL1_EXTRA (TCR_MTX0_ENABLE | TCR_MTX1_ENABLE) 1033 1034 #else /* !HAS_MTE */ 1035 1036 #define TCR_EL1_EXTRA 0 1037 1038 #endif /* HAS_MTE */ 1039 1040 1041 /* 1042 * Multiprocessor Affinity Register (MPIDR_EL1) 1043 * 1044 * +64-----------------------------31+30+29-25+24+23-16+15-8+7--0+ 1045 * |000000000000000000000000000000001| U|00000|MT| Aff2|Aff1|Aff0| 1046 * +---------------------------------+--+-----+--+-----+----+----+ 1047 * 1048 * where 1049 * U: Uniprocessor 1050 * MT: Multi-threading at lowest affinity level 1051 * Aff2: "1" - PCORE, "0" - ECORE 1052 * Aff1: Cluster ID 1053 * Aff0: CPU ID 1054 */ 1055 #define MPIDR_AFF0_SHIFT 0 1056 #define MPIDR_AFF0_WIDTH 8 1057 #define MPIDR_AFF0_MASK (((1 << MPIDR_AFF0_WIDTH) - 1) << MPIDR_AFF0_SHIFT) 1058 #define MPIDR_AFF1_SHIFT 8 1059 #define MPIDR_AFF1_WIDTH 8 1060 #define MPIDR_AFF1_MASK (((1 << MPIDR_AFF1_WIDTH) - 1) << MPIDR_AFF1_SHIFT) 1061 #define MPIDR_AFF2_SHIFT 16 1062 #define MPIDR_AFF2_WIDTH 8 1063 #define MPIDR_AFF2_MASK (((1 << MPIDR_AFF2_WIDTH) - 1) << MPIDR_AFF2_SHIFT) 1064 1065 /* 1066 * TXSZ indicates the size of the range a TTBR covers. Currently, 1067 * we support the following: 1068 * 1069 * 4KB pages, full page L1: 39 bit range. 1070 * 4KB pages, sub-page L1: 38 bit range. 1071 * 16KB pages, full page L1: 47 bit range. 1072 * 16KB pages, sub-page L1: 39 bit range. 1073 * 16KB pages, two level page tables: 36 bit range. 1074 */ 1075 #if __ARM_KERNEL_PROTECT__ 1076 /* 1077 * If we are configured to use __ARM_KERNEL_PROTECT__, the first half of the 1078 * address space is used for the mappings that will remain in place when in EL0. 1079 * As a result, 1 bit less of address space is available to the rest of the 1080 * the kernel. 1081 */ 1082 #endif /* __ARM_KERNEL_PROTECT__ */ 1083 #ifdef __ARM_16K_PG__ 1084 #if __ARM64_PMAP_SUBPAGE_L1__ 1085 #define T0SZ_BOOT 25ULL 1086 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ 1087 #define T0SZ_BOOT 17ULL 1088 #endif /* !__ARM64_PMAP_SUBPAGE_L1__ */ 1089 #else /* __ARM_16K_PG__ */ 1090 #if __ARM64_PMAP_SUBPAGE_L1__ 1091 #define T0SZ_BOOT 26ULL 1092 #else /* __ARM64_PMAP_SUBPAGE_L1__ */ 1093 #define T0SZ_BOOT 25ULL 1094 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 1095 #endif /* __ARM_16K_PG__ */ 1096 1097 #if __ARM64_PMAP_SUBPAGE_L1__ && CONFIG_SPTM 1098 #define T0SZ_EARLY_BOOT 17ULL 1099 #endif /*__ARM64_PMAP_SUBPAGE_L1__ && CONFIG_SPTM */ 1100 1101 #if HAS_ARM_INDEPENDENT_TNSZ 1102 #ifdef __ARM_16K_PG__ 1103 #if __ARM64_PMAP_KERN_SUBPAGE_L1__ 1104 #define T1SZ_BOOT 25ULL 1105 #else /* !__ARM64_PMAP_KERN_SUBPAGE_L1__ */ 1106 #define T1SZ_BOOT 17ULL 1107 #endif /* !__ARM64_PMAP_KERN_SUBPAGE_L1__ */ 1108 #else /* __ARM_16K_PG__ */ 1109 #if __ARM64_PMAP_KERN_SUBPAGE_L1__ 1110 #define T1SZ_BOOT 26ULL 1111 #else /* __ARM64_PMAP_KERN_SUBPAGE_L1__ */ 1112 #define T1SZ_BOOT 25ULL 1113 #endif /*__ARM64_PMAP_KERN_SUBPAGE_L1__*/ 1114 #endif /* __ARM_16K_PG__ */ 1115 #else /* HAS_ARM_INDEPENDENT_TNSZ */ 1116 #define T1SZ_BOOT T0SZ_BOOT 1117 #endif /* HAS_ARM_INDEPENDENT_TNSZ */ 1118 1119 #if __ARM_42BIT_PA_SPACE__ 1120 #define TCR_IPS_VALUE TCR_IPS_42BITS 1121 #else /* !__ARM_42BIT_PA_SPACE__ */ 1122 #define TCR_IPS_VALUE TCR_IPS_40BITS 1123 #endif /* !__ARM_42BIT_PA_SPACE__ */ 1124 1125 #if CONFIG_KERNEL_TBI 1126 #define TCR_EL1_DTBI (TCR_TBI1_TOPBYTE_IGNORED | TCR_TBID1_ENABLE) 1127 #else /* CONFIG_KERNEL_TBI */ 1128 #define TCR_EL1_DTBI 0 1129 #endif /* CONFIG_KERNEL_TBI */ 1130 1131 #if HAS_16BIT_ASID 1132 #define TCR_EL1_ASID TCR_AS_16BIT_ASID 1133 #else /* HAS_16BIT_ASID */ 1134 #define TCR_EL1_ASID 0 1135 #endif /* HAS_16BIT_ASID */ 1136 1137 #define TCR_EL1_BASE \ 1138 (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | \ 1139 TCR_IRGN0_WRITEBACK | (T0SZ_BOOT << TCR_T0SZ_SHIFT) | \ 1140 TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \ 1141 TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) | \ 1142 TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE) | TCR_E0PD_VALUE | \ 1143 TCR_EL1_DTBI | TCR_EL1_ASID | TCR_EL1_EXTRA) 1144 1145 #if __ARM64_PMAP_SUBPAGE_L1__ && CONFIG_SPTM 1146 #define TCR_EL1_BASE_BOOT \ 1147 (TCR_IPS_VALUE | TCR_SH0_OUTER | TCR_ORGN0_WRITEBACK | \ 1148 TCR_IRGN0_WRITEBACK | (T0SZ_EARLY_BOOT << TCR_T0SZ_SHIFT) | \ 1149 TCR_SH1_OUTER | TCR_ORGN1_WRITEBACK | \ 1150 TCR_IRGN1_WRITEBACK | (TCR_TG1_GRANULE_SIZE) | \ 1151 TCR_TBI0_TOPBYTE_IGNORED | (TCR_TBID0_ENABLE) | TCR_E0PD_VALUE | \ 1152 TCR_EL1_DTBI | TCR_EL1_ASID | TCR_EL1_EXTRA) 1153 #endif /* __ARM64_PMAP_SUBPAGE_L1__ && CONFIG_SPTM */ 1154 1155 #if __ARM_KERNEL_PROTECT__ 1156 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1157 #define T1SZ_USER (T1SZ_BOOT + 1) 1158 #define TCR_EL1_USER (TCR_EL1_BASE | (T1SZ_USER << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1159 #else 1160 #if CONFIG_SPTM 1161 #if __ARM64_PMAP_SUBPAGE_L1__ 1162 #define TCR_EL1_BOOT (TCR_EL1_BASE_BOOT | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1163 #define TCR_EL1_FINAL (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1164 #else /* !__ARM64_PMAP_SUBPAGE_L1__ */ 1165 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1166 #define TCR_EL1_FINAL TCR_EL1_BOOT 1167 #endif /* __ARM64_PMAP_SUBPAGE_L1__ */ 1168 #else /* !CONFIG_SPTM */ 1169 #define TCR_EL1_BOOT (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_SIZE)) 1170 #endif /* CONFIG_SPTM */ 1171 #endif /* __ARM_KERNEL_PROTECT__ */ 1172 1173 #define TCR_EL1_4KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_4KB)) 1174 #define TCR_EL1_16KB (TCR_EL1_BASE | (T1SZ_BOOT << TCR_T1SZ_SHIFT) | (TCR_TG0_GRANULE_16KB)) 1175 1176 /* 1177 * Bit 55 of the VA is used to select which TTBR to use during a translation table walk. 1178 */ 1179 #define TTBR_SELECTOR (1ULL << 55) 1180 1181 1182 1183 /* 1184 * Hypervisor Fine-Grained Read Trap Register (HFGRTR) 1185 */ 1186 1187 #define HFGRTR_AMAIR2_SHIFT 63 1188 #define HFGRTR_AMAIR2 (1ULL << HFGRTR_AMAIR2_SHIFT) 1189 #define HFGRTR_MAIR2_SHIFT 62 1190 #define HFGRTR_MAIR2 (1ULL << HFGRTR_MAIR2_SHIFT) 1191 #define HFGRTR_S2POR_SHIFT 61 1192 #define HFGRTR_S2POR (1ULL << HFGRTR_S2POR_SHIFT) 1193 #define HFGRTR_POR_EL1_SHIFT 60 1194 #define HFGRTR_POR_EL1 (1ULL << HFGRTR_POR_EL1_SHIFT) 1195 #define HFGRTR_POR_EL0_SHIFT 59 1196 #define HFGRTR_POR_EL0 (1ULL << HFGRTR_POR_EL0_SHIFT) 1197 #define HFGRTR_PIR_SHIFT 58 1198 #define HFGRTR_PIR (1ULL << HFGRTR_PIR_SHIFT) 1199 #define HFGRTR_PIRE0_SHIFT 57 1200 #define HFGRTR_PIRE0 (1ULL << HFGRTR_PIRE0_SHIFT) 1201 #define HFGRTR_RCWMASK_SHIFT 56 1202 #define HFGRTR_RCWMASK (1ULL << HFGRTR_RCWMASK_SHIFT) 1203 #define HFGRTR_TPIDR2_SHIFT 55 1204 #define HFGRTR_TPIDR2 (1ULL << HFGRTR_TPIDR2_SHIFT) 1205 #define HFGRTR_SMPRI_SHIFT 54 1206 #define HFGRTR_SMPRI (1ULL << HFGRTR_SMPRI_SHIFT) 1207 #define HFGRTR_GCS_EL1_SHIFT 53 1208 #define HFGRTR_GCS_EL1 (1ULL << HFGRTR_GCS_EL1_SHIFT) 1209 #define HFGRTR_GCS_EL0_SHIFT 52 1210 #define HFGRTR_GCS_EL0 (1ULL << HFGRTR_GCS_EL0_SHIFT) 1211 #define HFGRTR_ACCDATA_SHIFT 50 1212 #define HFGRTR_ACCDATA (1ULL << HFGRTR_ACCDATA_SHIFT) 1213 #define HFGRTR_ERXADDR_SHIFT 49 1214 #define HFGRTR_ERXADDR (1ULL << HFGRTR_ERXADDR_SHIFT) 1215 #define HFGRTR_ERXPFGCDN_SHIFT 48 1216 #define HFGRTR_ERXPFGCDN (1ULL << HFGRTR_ERXPFGCDN_SHIFT) 1217 #define HFGRTR_ERXPFGCTL_SHIFT 47 1218 #define HFGRTR_ERXPFGCTL (1ULL << HFGRTR_ERXPFGCTL_SHIFT) 1219 #define HFGRTR_ERXPFGF_SHIFT 46 1220 #define HFGRTR_ERXPFGF (1ULL << HFGRTR_ERXPFGF_SHIFT) 1221 #define HFGRTR_ERXMISC_SHIFT 45 1222 #define HFGRTR_ERXMISC (1ULL << HFGRTR_ERXMISC_SHIFT) 1223 #define HFGRTR_ERXSTATUS_SHIFT 44 1224 #define HFGRTR_ERXSTATUS (1ULL << HFGRTR_ERXSTATUS_SHIFT) 1225 #define HFGRTR_ERXCTLR_SHIFT 43 1226 #define HFGRTR_ERXCTLR (1ULL << HFGRTR_ERXCTLR_SHIFT) 1227 #define HFGRTR_ERXFR_SHIFT 42 1228 #define HFGRTR_ERXFR (1ULL << HFGRTR_ERXFR_SHIFT) 1229 #define HFGRTR_ERRSELR_SHIFT 41 1230 #define HFGRTR_ERRSELR (1ULL << HFGRTR_ERRSELR_SHIFT) 1231 #define HFGRTR_ERRIDR_SHIFT 40 1232 #define HFGRTR_ERRIDR (1ULL << HFGRTR_ERRIDR_SHIFT) 1233 #define HFGRTR_ICC_IGRPEN_SHIFT 39 1234 #define HFGRTR_ICC_IGRPEN (1ULL << HFGRTR_ICC_IGRPEN_SHIFT) 1235 #define HFGRTR_VBAR_SHIFT 38 1236 #define HFGRTR_VBAR (1ULL << HFGRTR_VBAR_SHIFT) 1237 #define HFGRTR_TTBR1_SHIFT 37 1238 #define HFGRTR_TTBR1 (1ULL << HFGRTR_TTBR1_SHIFT) 1239 #define HFGRTR_TTBR0_SHIFT 36 1240 #define HFGRTR_TTBR0 (1ULL << HFGRTR_TTBR0_SHIFT) 1241 #define HFGRTR_TPIDR_EL0_SHIFT 35 1242 #define HFGRTR_TPIDR_EL0 (1ULL << HFGRTR_TPIDR_EL0_SHIFT) 1243 #define HFGRTR_TPIDRRO_SHIFT 34 1244 #define HFGRTR_TPIDRRO (1ULL << HFGRTR_TPIDRRO_SHIFT) 1245 #define HFGRTR_TPIDR_EL1_SHIFT 33 1246 #define HFGRTR_TPIDR_EL1 (1ULL << HFGRTR_TPIDR_EL1_SHIFT) 1247 #define HFGRTR_TCR_SHIFT 32 1248 #define HFGRTR_TCR (1ULL << HFGRTR_TCR_SHIFT) 1249 #define HFGRTR_SCXTNUM_EL0_SHIFT 31 1250 #define HFGRTR_SCXTNUM_EL0 (1ULL << HFGRTR_SCXTNUM_EL0_SHIFT) 1251 #define HFGRTR_SCXTNUM_EL1_SHIFT 30 1252 #define HFGRTR_SCXTNUM_EL1 (1ULL << HFGRTR_SCXTNUM_EL1_SHIFT) 1253 #define HFGRTR_SCTLR_SHIFT 29 1254 #define HFGRTR_SCTLR (1ULL << HFGRTR_SCTLR_SHIFT) 1255 #define HFGRTR_REVIDR_SHIFT 28 1256 #define HFGRTR_REVIDR (1ULL << HFGRTR_REVIDR_SHIFT) 1257 #define HFGRTR_PAR_SHIFT 27 1258 #define HFGRTR_PAR (1ULL << HFGRTR_PAR_SHIFT) 1259 #define HFGRTR_MPIDR_SHIFT 26 1260 #define HFGRTR_MPIDR (1ULL << HFGRTR_MPIDR_SHIFT) 1261 #define HFGRTR_MIDR_SHIFT 25 1262 #define HFGRTR_MIDR (1ULL << HFGRTR_MIDR_SHIFT) 1263 #define HFGRTR_MAIR_SHIFT 24 1264 #define HFGRTR_MAIR (1ULL << HFGRTR_MAIR_SHIFT) 1265 #define HFGRTR_LORSA_SHIFT 23 1266 #define HFGRTR_LORSA (1ULL << HFGRTR_LORSA_SHIFT) 1267 #define HFGRTR_LORN_SHIFT 22 1268 #define HFGRTR_LORN (1ULL << HFGRTR_LORN_SHIFT) 1269 #define HFGRTR_LORID_SHIFT 21 1270 #define HFGRTR_LORID (1ULL << HFGRTR_LORID_SHIFT) 1271 #define HFGRTR_LOREA_SHIFT 20 1272 #define HFGRTR_LOREA (1ULL << HFGRTR_LOREA_SHIFT) 1273 #define HFGRTR_LORC_SHIFT 19 1274 #define HFGRTR_LORC (1ULL << HFGRTR_LORC_SHIFT) 1275 #define HFGRTR_ISR_SHIFT 18 1276 #define HFGRTR_ISR (1ULL << HFGRTR_ISR_SHIFT) 1277 #define HFGRTR_FAR_SHIFT 17 1278 #define HFGRTR_FAR (1ULL << HFGRTR_FAR_SHIFT) 1279 #define HFGRTR_ESR_SHIFT 16 1280 #define HFGRTR_ESR (1ULL << HFGRTR_ESR_SHIFT) 1281 #define HFGRTR_DCZID_SHIFT 15 1282 #define HFGRTR_DCZID (1ULL << HFGRTR_DCZID_SHIFT) 1283 #define HFGRTR_CTR_SHIFT 14 1284 #define HFGRTR_CTR (1ULL << HFGRTR_CTR_SHIFT) 1285 #define HFGRTR_CSSELR_SHIFT 13 1286 #define HFGRTR_CSSELR (1ULL << HFGRTR_CSSELR_SHIFT) 1287 #define HFGRTR_CPACR_SHIFT 12 1288 #define HFGRTR_CPACR (1ULL << HFGRTR_CPACR_SHIFT) 1289 #define HFGRTR_CONTEXTIDR_SHIFT 11 1290 #define HFGRTR_CONTEXTIDR (1ULL << HFGRTR_CONTEXTIDR_SHIFT) 1291 #define HFGRTR_CLIDR_SHIFT 10 1292 #define HFGRTR_CLIDR (1ULL << HFGRTR_CLIDR_SHIFT) 1293 #define HFGRTR_CCSIDR_SHIFT 9 1294 #define HFGRTR_CCSIDR (1ULL << HFGRTR_CCSIDR_SHIFT) 1295 #define HFGRTR_APIBKEY_SHIFT 8 1296 #define HFGRTR_APIBKEY (1ULL << HFGRTR_APIBKEY_SHIFT) 1297 #define HFGRTR_APIAKEY_SHIFT 7 1298 #define HFGRTR_APIAKEY (1ULL << HFGRTR_APIAKEY_SHIFT) 1299 #define HFGRTR_APGAKEY_SHIFT 6 1300 #define HFGRTR_APGAKEY (1ULL << HFGRTR_APGAKEY_SHIFT) 1301 #define HFGRTR_APDBKEY_SHIFT 5 1302 #define HFGRTR_APDBKEY (1ULL << HFGRTR_APDBKEY_SHIFT) 1303 #define HFGRTR_APDAKEY_SHIFT 4 1304 #define HFGRTR_APDAKEY (1ULL << HFGRTR_APDAKEY_SHIFT) 1305 #define HFGRTR_AMAIR_SHIFT 3 1306 #define HFGRTR_AMAIR (1ULL << HFGRTR_AMAIR_SHIFT) 1307 #define HFGRTR_AIDR_SHIFT 2 1308 #define HFGRTR_AIDR (1ULL << HFGRTR_AIDR_SHIFT) 1309 #define HFGRTR_AFSR1_SHIFT 1 1310 #define HFGRTR_AFSR1 (1ULL << HFGRTR_AFSR1_SHIFT) 1311 #define HFGRTR_AFSR0_SHIFT 0 1312 #define HFGRTR_AFSR0 (1ULL << HFGRTR_AFSR0_SHIFT) 1313 1314 /* 1315 * Hypervisor Fine-Grained Write Trap Register (HFGWTR) 1316 */ 1317 1318 #define HFGWTR_AMAIR2_SHIFT 63 1319 #define HFGWTR_AMAIR2 (1ULL << HFGWTR_AMAIR2_SHIFT) 1320 #define HFGWTR_MAIR2_SHIFT 62 1321 #define HFGWTR_MAIR2 (1ULL << HFGWTR_MAIR2_SHIFT) 1322 #define HFGWTR_S2POR_SHIFT 61 1323 #define HFGWTR_S2POR (1ULL << HFGWTR_S2POR_SHIFT) 1324 #define HFGWTR_POR_EL1_SHIFT 60 1325 #define HFGWTR_POR_EL1 (1ULL << HFGWTR_POR_EL1_SHIFT) 1326 #define HFGWTR_POR_EL0_SHIFT 59 1327 #define HFGWTR_POR_EL0 (1ULL << HFGWTR_POR_EL0_SHIFT) 1328 #define HFGWTR_PIR_SHIFT 58 1329 #define HFGWTR_PIR (1ULL << HFGWTR_PIR_SHIFT) 1330 #define HFGWTR_PIRE0_SHIFT 57 1331 #define HFGWTR_PIRE0 (1ULL << HFGWTR_PIRE0_SHIFT) 1332 #define HFGWTR_RCWMASK_SHIFT 56 1333 #define HFGWTR_RCWMASK (1ULL << HFGWTR_RCWMASK_SHIFT) 1334 #define HFGWTR_TPIDR2_SHIFT 55 1335 #define HFGWTR_TPIDR2 (1ULL << HFGWTR_TPIDR2_SHIFT) 1336 #define HFGWTR_SMPRI_SHIFT 54 1337 #define HFGWTR_SMPRI (1ULL << HFGWTR_SMPRI_SHIFT) 1338 #define HFGWTR_GCS_EL1_SHIFT 53 1339 #define HFGWTR_GCS_EL1 (1ULL << HFGWTR_GCS_EL1_SHIFT) 1340 #define HFGWTR_GCS_EL0_SHIFT 52 1341 #define HFGWTR_GCS_EL0 (1ULL << HFGWTR_GCS_EL0_SHIFT) 1342 #define HFGWTR_ACCDATA_SHIFT 50 1343 #define HFGWTR_ACCDATA (1ULL << HFGWTR_ACCDATA_SHIFT) 1344 #define HFGWTR_ERXADDR_SHIFT 49 1345 #define HFGWTR_ERXADDR (1ULL << HFGWTR_ERXADDR_SHIFT) 1346 #define HFGWTR_ERXPFGCDN_SHIFT 48 1347 #define HFGWTR_ERXPFGCDN (1ULL << HFGWTR_ERXPFGCDN_SHIFT) 1348 #define HFGWTR_ERXPFGCTL_SHIFT 47 1349 #define HFGWTR_ERXPFGCTL (1ULL << HFGWTR_ERXPFGCTL_SHIFT) 1350 #define HFGWTR_ERXMISC_SHIFT 45 1351 #define HFGWTR_ERXMISC (1ULL << HFGWTR_ERXMISC_SHIFT) 1352 #define HFGWTR_ERXSTATUS_SHIFT 44 1353 #define HFGWTR_ERXSTATUS (1ULL << HFGWTR_ERXSTATUS_SHIFT) 1354 #define HFGWTR_ERXCTLR_SHIFT 43 1355 #define HFGWTR_ERXCTLR (1ULL << HFGWTR_ERXCTLR_SHIFT) 1356 #define HFGWTR_ERRSELR_SHIFT 41 1357 #define HFGWTR_ERRSELR (1ULL << HFGWTR_ERRSELR_SHIFT) 1358 #define HFGWTR_ICC_IGRPEN_SHIFT 39 1359 #define HFGWTR_ICC_IGRPEN (1ULL << HFGWTR_ICC_IGRPEN_SHIFT) 1360 #define HFGWTR_VBAR_SHIFT 38 1361 #define HFGWTR_VBAR (1ULL << HFGWTR_VBAR_SHIFT) 1362 #define HFGWTR_TTBR1_SHIFT 37 1363 #define HFGWTR_TTBR1 (1ULL << HFGWTR_TTBR1_SHIFT) 1364 #define HFGWTR_TTBR0_SHIFT 36 1365 #define HFGWTR_TTBR0 (1ULL << HFGWTR_TTBR0_SHIFT) 1366 #define HFGWTR_TPIDR_EL0_SHIFT 35 1367 #define HFGWTR_TPIDR_EL0 (1ULL << HFGWTR_TPIDR_EL0_SHIFT) 1368 #define HFGWTR_TPIDRRO_SHIFT 34 1369 #define HFGWTR_TPIDRRO (1ULL << HFGWTR_TPIDRRO_SHIFT) 1370 #define HFGWTR_TPIDR_EL1_SHIFT 33 1371 #define HFGWTR_TPIDR_EL1 (1ULL << HFGWTR_TPIDR_EL1_SHIFT) 1372 #define HFGWTR_TCR_SHIFT 32 1373 #define HFGWTR_TCR (1ULL << HFGWTR_TCR_SHIFT) 1374 #define HFGWTR_SCXTNUM_EL0_SHIFT 31 1375 #define HFGWTR_SCXTNUM_EL0 (1ULL << HFGWTR_SCXTNUM_EL0_SHIFT) 1376 #define HFGWTR_SCXTNUM_EL1_SHIFT 30 1377 #define HFGWTR_SCXTNUM_EL1 (1ULL << HFGWTR_SCXTNUM_EL1_SHIFT) 1378 #define HFGWTR_SCXTNUM_SHIFT 30 1379 #define HFGWTR_SCXTNUM (1ULL << HFGWTR_SCXTNUM_SHIFT) 1380 #define HFGWTR_SCTLR_SHIFT 29 1381 #define HFGWTR_SCTLR (1ULL << HFGWTR_SCTLR_SHIFT) 1382 #define HFGWTR_PAR_SHIFT 27 1383 #define HFGWTR_PAR (1ULL << HFGWTR_PAR_SHIFT) 1384 #define HFGWTR_MAIR_SHIFT 24 1385 #define HFGWTR_MAIR (1ULL << HFGWTR_MAIR_SHIFT) 1386 #define HFGWTR_LORSA_SHIFT 23 1387 #define HFGWTR_LORSA (1ULL << HFGWTR_LORSA_SHIFT) 1388 #define HFGWTR_LORN_SHIFT 22 1389 #define HFGWTR_LORN (1ULL << HFGWTR_LORN_SHIFT) 1390 #define HFGWTR_LOREA_SHIFT 20 1391 #define HFGWTR_LOREA (1ULL << HFGWTR_LOREA_SHIFT) 1392 #define HFGWTR_LORC_SHIFT 19 1393 #define HFGWTR_LORC (1ULL << HFGWTR_LORC_SHIFT) 1394 #define HFGWTR_FAR_SHIFT 17 1395 #define HFGWTR_FAR (1ULL << HFGWTR_FAR_SHIFT) 1396 #define HFGWTR_ESR_SHIFT 16 1397 #define HFGWTR_ESR (1ULL << HFGWTR_ESR_SHIFT) 1398 #define HFGWTR_CSSELR_SHIFT 13 1399 #define HFGWTR_CSSELR (1ULL << HFGWTR_CSSELR_SHIFT) 1400 #define HFGWTR_CPACR_SHIFT 12 1401 #define HFGWTR_CPACR (1ULL << HFGWTR_CPACR_SHIFT) 1402 #define HFGWTR_CONTEXTIDR_SHIFT 11 1403 #define HFGWTR_CONTEXTIDR (1ULL << HFGWTR_CONTEXTIDR_SHIFT) 1404 #define HFGWTR_APIBKEY_SHIFT 8 1405 #define HFGWTR_APIBKEY (1ULL << HFGWTR_APIBKEY_SHIFT) 1406 #define HFGWTR_APIAKEY_SHIFT 7 1407 #define HFGWTR_APIAKEY (1ULL << HFGWTR_APIAKEY_SHIFT) 1408 #define HFGWTR_APGAKEY_SHIFT 6 1409 #define HFGWTR_APGAKEY (1ULL << HFGWTR_APGAKEY_SHIFT) 1410 #define HFGWTR_APDBKEY_SHIFT 5 1411 #define HFGWTR_APDBKEY (1ULL << HFGWTR_APDBKEY_SHIFT) 1412 #define HFGWTR_APDAKEY_SHIFT 4 1413 #define HFGWTR_APDAKEY (1ULL << HFGWTR_APDAKEY_SHIFT) 1414 #define HFGWTR_AMAIR_SHIFT 3 1415 #define HFGWTR_AMAIR (1ULL << HFGWTR_AMAIR_SHIFT) 1416 #define HFGWTR_AFSR1_SHIFT 1 1417 #define HFGWTR_AFSR1 (1ULL << HFGWTR_AFSR1_SHIFT) 1418 #define HFGWTR_AFSR0_SHIFT 0 1419 #define HFGWTR_AFSR0 (1ULL << HFGWTR_AFSR0_SHIFT) 1420 1421 /* 1422 * Monitor Debug System Control Register (MDSCR) 1423 */ 1424 1425 #define MDSCR_TFO_SHIFT 31 1426 #define MDSCR_TFO (1ULL << MDSCR_TFO_SHIFT) 1427 #define MDSCR_RXFULL_SHIFT 30 1428 #define MDSCR_RXFULL (1ULL << MDSCR_RXFULL_SHIFT) 1429 #define MDSCR_TXFULL_SHIFT 29 1430 #define MDSCR_TXFULL (1ULL << MDSCR_TXFULL_SHIFT) 1431 #define MDSCR_RXO_SHIFT 27 1432 #define MDSCR_RXO (1ULL << MDSCR_RXO_SHIFT) 1433 #define MDSCR_TXU_SHIFT 26 1434 #define MDSCR_TXU (1ULL << MDSCR_TXU_SHIFT) 1435 #define MDSCR_INTDIS_SHIFT 22 1436 #define MDSCR_INTDIS_MASK (0x2U << MDSCR_INTDIS_SHIFT) 1437 #define MDSCR_TDA_SHIFT 21 1438 #define MDSCR_TDA (1ULL << MDSCR_TDA_SHIFT) 1439 #define MDSCR_SC2_SHIFT 19 1440 #define MDSCR_SC2 (1ULL << MDSCR_SC2_SHIFT) 1441 #define MDSCR_MDE_SHIFT 15 1442 #define MDSCR_MDE (1ULL << MDSCR_MDE_SHIFT) 1443 #define MDSCR_HDE_SHIFT 14 1444 #define MDSCR_HDE (1ULL << MDSCR_HDE_SHIFT) 1445 #define MDSCR_KDE_SHIFT 13 1446 #define MDSCR_KDE (1ULL << MDSCR_KDE_SHIFT) 1447 #define MDSCR_TDCC_SHIFT 12 1448 #define MDSCR_TDCC (1ULL << MDSCR_TDCC_SHIFT) 1449 #define MDSCR_ERR_SHIFT 6 1450 #define MDSCR_ERR (1ULL << MDSCR_ERR_SHIFT) 1451 #define MDSCR_SS_SHIFT 0 1452 #define MDSCR_SS (1ULL << MDSCR_SS_SHIFT) 1453 1454 /* 1455 * Hypervisor Debug Fine-Grained Read Trap Register (HDFGRTR_EL2) 1456 */ 1457 #define HDFGRTR_PMBIDR_SHIFT 63 1458 #define HDFGRTR_PMBIDR (1ULL << HDFGRTR_PMBIDR_SHIFT) 1459 #define HDFGRTR_PMSNEVFR_SHIFT 62 1460 #define HDFGRTR_PMSNEVFR (1ULL << HDFGRTR_PMSNEVFR_SHIFT) 1461 #define HDFGRTR_BRBDATA_SHIFT 61 1462 #define HDFGRTR_BRBDATA (1ULL << HDFGRTR_BRBDATA_SHIFT) 1463 #define HDFGRTR_BRBCTL_SHIFT 60 1464 #define HDFGRTR_BRBCTL (1ULL << HDFGRTR_BRBCTL_SHIFT) 1465 #define HDFGRTR_BRBIDR_SHIFT 59 1466 #define HDFGRTR_BRBIDR (1ULL << HDFGRTR_BRBIDR_SHIFT) 1467 #define HDFGRTR_PMCEID_SHIFT 58 1468 #define HDFGRTR_PMCEID (1ULL << HDFGRTR_PMCEID_SHIFT) 1469 #define HDFGRTR_PMUSERENR_SHIFT 57 1470 #define HDFGRTR_PMUSERENR (1ULL << HDFGRTR_PMUSERENR_SHIFT) 1471 #define HDFGRTR_TRBTRG_SHIFT 56 1472 #define HDFGRTR_TRBTRG (1ULL << HDFGRTR_TRBTRG_SHIFT) 1473 #define HDFGRTR_TRBSR_SHIFT 55 1474 #define HDFGRTR_TRBSR (1ULL << HDFGRTR_TRBSR_SHIFT) 1475 #define HDFGRTR_TRBPTR_SHIFT 54 1476 #define HDFGRTR_TRBPTR (1ULL << HDFGRTR_TRBPTR_SHIFT) 1477 #define HDFGRTR_TRBMAR_SHIFT 53 1478 #define HDFGRTR_TRBMAR (1ULL << HDFGRTR_TRBMAR_SHIFT) 1479 #define HDFGRTR_TRBLIMITR_SHIFT 52 1480 #define HDFGRTR_TRBLIMITR (1ULL << HDFGRTR_TRBLIMITR_SHIFT) 1481 #define HDFGRTR_TRBIDR_SHIFT 51 1482 #define HDFGRTR_TRBIDR (1ULL << HDFGRTR_TRBIDR_SHIFT) 1483 #define HDFGRTR_TRBBASER_SHIFT 50 1484 #define HDFGRTR_TRBBASER (1ULL << HDFGRTR_TRBBASER_SHIFT) 1485 #define HDFGRTR_TRCVICTLR_SHIFT 48 1486 #define HDFGRTR_TRCVICTLR (1ULL << HDFGRTR_TRCVICTLR_SHIFT) 1487 #define HDFGRTR_TRCSTATR_SHIFT 47 1488 #define HDFGRTR_TRCSTATR (1ULL << HDFGRTR_TRCSTATR_SHIFT) 1489 #define HDFGRTR_TRCSSCSR_SHIFT 46 1490 #define HDFGRTR_TRCSSCSR (1ULL << HDFGRTR_TRCSSCSR_SHIFT) 1491 #define HDFGRTR_TRCSEQSTR_SHIFT 45 1492 #define HDFGRTR_TRCSEQSTR (1ULL << HDFGRTR_TRCSEQSTR_SHIFT) 1493 #define HDFGRTR_TRCPRGCTLR_SHIFT 44 1494 #define HDFGRTR_TRCPRGCTLR (1ULL << HDFGRTR_TRCPRGCTLR_SHIFT) 1495 #define HDFGRTR_TRCOSLSR_SHIFT 43 1496 #define HDFGRTR_TRCOSLSR (1ULL << HDFGRTR_TRCOSLSR_SHIFT) 1497 #define HDFGRTR_TRCIMSPEC_SHIFT 41 1498 #define HDFGRTR_TRCIMSPEC (1ULL << HDFGRTR_TRCIMSPEC_SHIFT) 1499 #define HDFGRTR_TRCID_SHIFT 40 1500 #define HDFGRTR_TRCID (1ULL << HDFGRTR_TRCID_SHIFT) 1501 #define HDFGRTR_TRCCNTVR_SHIFT 37 1502 #define HDFGRTR_TRCCNTVR (1ULL << HDFGRTR_TRCCNTVR_SHIFT) 1503 #define HDFGRTR_TRCCLAIM_SHIFT 36 1504 #define HDFGRTR_TRCCLAIM (1ULL << HDFGRTR_TRCCLAIM_SHIFT) 1505 #define HDFGRTR_TRCAUXCTLR_SHIFT 35 1506 #define HDFGRTR_TRCAUXCTLR (1ULL << HDFGRTR_TRCAUXCTLR_SHIFT) 1507 #define HDFGRTR_TRCAUTHSTATUS_SHIFT 34 1508 #define HDFGRTR_TRCAUTHSTATUS (1ULL << HDFGRTR_TRCAUTHSTATUS_SHIFT) 1509 #define HDFGRTR_TRC_SHIFT 33 1510 #define HDFGRTR_TRC (1ULL << HDFGRTR_TRC_SHIFT) 1511 #define HDFGRTR_PMSLATFR_SHIFT 32 1512 #define HDFGRTR_PMSLATFR (1ULL << HDFGRTR_PMSLATFR_SHIFT) 1513 #define HDFGRTR_PMSIRR_SHIFT 31 1514 #define HDFGRTR_PMSIRR (1ULL << HDFGRTR_PMSIRR_SHIFT) 1515 #define HDFGRTR_PMSIDR_SHIFT 30 1516 #define HDFGRTR_PMSIDR (1ULL << HDFGRTR_PMSIDR_SHIFT) 1517 #define HDFGRTR_PMSICR_SHIFT 29 1518 #define HDFGRTR_PMSICR (1ULL << HDFGRTR_PMSICR_SHIFT) 1519 #define HDFGRTR_PMSFCR_SHIFT 28 1520 #define HDFGRTR_PMSFCR (1ULL << HDFGRTR_PMSFCR_SHIFT) 1521 #define HDFGRTR_PMSEVFR_SHIFT 27 1522 #define HDFGRTR_PMSEVFR (1ULL << HDFGRTR_PMSEVFR_SHIFT) 1523 #define HDFGRTR_PMSCR_SHIFT 26 1524 #define HDFGRTR_PMSCR (1ULL << HDFGRTR_PMSCR_SHIFT) 1525 #define HDFGRTR_PMBSR_SHIFT 25 1526 #define HDFGRTR_PMBSR (1ULL << HDFGRTR_PMBSR_SHIFT) 1527 #define HDFGRTR_PMBPTR_SHIFT 24 1528 #define HDFGRTR_PMBPTR (1ULL << HDFGRTR_PMBPTR_SHIFT) 1529 #define HDFGRTR_PMBLIMITR_SHIFT 23 1530 #define HDFGRTR_PMBLIMITR (1ULL << HDFGRTR_PMBLIMITR_SHIFT) 1531 #define HDFGRTR_PMMIR_SHIFT 22 1532 #define HDFGRTR_PMMIR (1ULL << HDFGRTR_PMMIR_SHIFT) 1533 #define HDFGRTR_PMSELR_SHIFT 19 1534 #define HDFGRTR_PMSELR (1ULL << HDFGRTR_PMSELR_SHIFT) 1535 #define HDFGRTR_PMOVS_SHIFT 18 1536 #define HDFGRTR_PMOVS (1ULL << HDFGRTR_PMOVS_SHIFT) 1537 #define HDFGRTR_PMINTEN_SHIFT 17 1538 #define HDFGRTR_PMINTEN (1ULL << HDFGRTR_PMINTEN_SHIFT) 1539 #define HDFGRTR_PMCNTEN_SHIFT 16 1540 #define HDFGRTR_PMCNTEN (1ULL << HDFGRTR_PMCNTEN_SHIFT) 1541 #define HDFGRTR_PMCCNTR_SHIFT 15 1542 #define HDFGRTR_PMCCNTR (1ULL << HDFGRTR_PMCCNTR_SHIFT) 1543 #define HDFGRTR_PMCCFILTR_SHIFT 14 1544 #define HDFGRTR_PMCCFILTR (1ULL << HDFGRTR_PMCCFILTR_SHIFT) 1545 #define HDFGRTR_PMEVTYPER_SHIFT 13 1546 #define HDFGRTR_PMEVTYPER (1ULL << HDFGRTR_PMEVTYPER_SHIFT) 1547 #define HDFGRTR_PMEVCNTR_SHIFT 12 1548 #define HDFGRTR_PMEVCNTR (1ULL << HDFGRTR_PMEVCNTR_SHIFT) 1549 #define HDFGRTR_OSDLR_SHIFT 11 1550 #define HDFGRTR_OSDLR (1ULL << HDFGRTR_OSDLR_SHIFT) 1551 #define HDFGRTR_OSECCR_SHIFT 10 1552 #define HDFGRTR_OSECCR (1ULL << HDFGRTR_OSECCR_SHIFT) 1553 #define HDFGRTR_OSLSR_SHIFT 9 1554 #define HDFGRTR_OSLSR (1ULL << HDFGRTR_OSLSR_SHIFT) 1555 #define HDFGRTR_DBGPRCR_SHIFT 7 1556 #define HDFGRTR_DBGPRCR (1ULL << HDFGRTR_DBGPRCR_SHIFT) 1557 #define HDFGRTR_DBGAUTHSTATUS_SHIFT 6 1558 #define HDFGRTR_DBGAUTHSTATUS (1ULL << HDFGRTR_DBGAUTHSTATUS_SHIFT) 1559 #define HDFGRTR_DBGCLAIM_SHIFT 5 1560 #define HDFGRTR_DBGCLAIM (1ULL << HDFGRTR_DBGCLAIM_SHIFT) 1561 #define HDFGRTR_MDSCR_SHIFT 4 1562 #define HDFGRTR_MDSCR (1ULL << HDFGRTR_MDSCR_SHIFT) 1563 #define HDFGRTR_DBGWVR_SHIFT 3 1564 #define HDFGRTR_DBGWVR (1ULL << HDFGRTR_DBGWVR_SHIFT) 1565 #define HDFGRTR_DBGWCR_SHIFT 2 1566 #define HDFGRTR_DBGWCR (1ULL << HDFGRTR_DBGWCR_SHIFT) 1567 #define HDFGRTR_DBGBVR_SHIFT 1 1568 #define HDFGRTR_DBGBVR (1ULL << HDFGRTR_DBGBVR_SHIFT) 1569 #define HDFGRTR_DBGBCR_SHIFT 0 1570 #define HDFGRTR_DBGBCR (1ULL << HDFGRTR_DBGBCR_SHIFT) 1571 1572 /* 1573 * Hypervisor Debug Fine-Grained Write Trap Register (HDFGWTR_EL2) 1574 */ 1575 #define HDFGWTR_PMSNEVFR_SHIFT 62 1576 #define HDFGWTR_PMSNEVFR (1ULL << HDFGWTR_PMSNEVFR_SHIFT) 1577 #define HDFGWTR_BRBDATA_SHIFT 61 1578 #define HDFGWTR_BRBDATA (1ULL << HDFGWTR_BRBDATA_SHIFT) 1579 #define HDFGWTR_BRBCTL_SHIFT 60 1580 #define HDFGWTR_BRBCTL (1ULL << HDFGWTR_BRBCTL_SHIFT) 1581 #define HDFGWTR_PMUSERENR_SHIFT 57 1582 #define HDFGWTR_PMUSERENR (1ULL << HDFGWTR_PMUSERENR_SHIFT) 1583 #define HDFGWTR_TRBTRG_SHIFT 56 1584 #define HDFGWTR_TRBTRG (1ULL << HDFGWTR_TRBTRG_SHIFT) 1585 #define HDFGWTR_TRBSR_SHIFT 55 1586 #define HDFGWTR_TRBSR (1ULL << HDFGWTR_TRBSR_SHIFT) 1587 #define HDFGWTR_TRBPTR_SHIFT 54 1588 #define HDFGWTR_TRBPTR (1ULL << HDFGWTR_TRBPTR_SHIFT) 1589 #define HDFGWTR_TRBMAR_SHIFT 53 1590 #define HDFGWTR_TRBMAR (1ULL << HDFGWTR_TRBMAR_SHIFT) 1591 #define HDFGWTR_TRBLIMITR_SHIFT 52 1592 #define HDFGWTR_TRBLIMITR (1ULL << HDFGWTR_TRBLIMITR_SHIFT) 1593 #define HDFGWTR_TRBBASER_SHIFT 50 1594 #define HDFGWTR_TRBBASER (1ULL << HDFGWTR_TRBBASER_SHIFT) 1595 #define HDFGWTR_TRFCR_SHIFT 49 1596 #define HDFGWTR_TRFCR (1ULL << HDFGWTR_TRFCR_SHIFT) 1597 #define HDFGWTR_TRCVICTLR_SHIFT 48 1598 #define HDFGWTR_TRCVICTLR (1ULL << HDFGWTR_TRCVICTLR_SHIFT) 1599 #define HDFGWTR_TRCSSCSR_SHIFT 46 1600 #define HDFGWTR_TRCSSCSR (1ULL << HDFGWTR_TRCSSCSR_SHIFT) 1601 #define HDFGWTR_TRCSEQSTR_SHIFT 45 1602 #define HDFGWTR_TRCSEQSTR (1ULL << HDFGWTR_TRCSEQSTR_SHIFT) 1603 #define HDFGWTR_TRCPRGCTLR_SHIFT 44 1604 #define HDFGWTR_TRCPRGCTLR (1ULL << HDFGWTR_TRCPRGCTLR_SHIFT) 1605 #define HDFGWTR_TRCOSLAR_SHIFT 42 1606 #define HDFGWTR_TRCOSLAR (1ULL << HDFGWTR_TRCOSLAR_SHIFT) 1607 #define HDFGWTR_TRCIMSPEC_SHIFT 41 1608 #define HDFGWTR_TRCIMSPEC (1ULL << HDFGWTR_TRCIMSPEC_SHIFT) 1609 #define HDFGWTR_TRCCNTVR_SHIFT 37 1610 #define HDFGWTR_TRCCNTVR (1ULL << HDFGWTR_TRCCNTVR_SHIFT) 1611 #define HDFGWTR_TRCCLAIM_SHIFT 36 1612 #define HDFGWTR_TRCCLAIM (1ULL << HDFGWTR_TRCCLAIM_SHIFT) 1613 #define HDFGWTR_TRCAUXCTLR_SHIFT 35 1614 #define HDFGWTR_TRCAUXCTLR (1ULL << HDFGWTR_TRCAUXCTLR_SHIFT) 1615 #define HDFGWTR_TRC_SHIFT 33 1616 #define HDFGWTR_TRC (1ULL << HDFGWTR_TRC_SHIFT) 1617 #define HDFGWTR_PMSLATFR_SHIFT 32 1618 #define HDFGWTR_PMSLATFR (1ULL << HDFGWTR_PMSLATFR_SHIFT) 1619 #define HDFGWTR_PMSIRR_SHIFT 31 1620 #define HDFGWTR_PMSIRR (1ULL << HDFGWTR_PMSIRR_SHIFT) 1621 #define HDFGWTR_PMSICR_SHIFT 29 1622 #define HDFGWTR_PMSICR (1ULL << HDFGWTR_PMSICR_SHIFT) 1623 #define HDFGWTR_PMSFCR_SHIFT 28 1624 #define HDFGWTR_PMSFCR (1ULL << HDFGWTR_PMSFCR_SHIFT) 1625 #define HDFGWTR_PMSEVFR_SHIFT 27 1626 #define HDFGWTR_PMSEVFR (1ULL << HDFGWTR_PMSEVFR_SHIFT) 1627 #define HDFGWTR_PMSCR_SHIFT 26 1628 #define HDFGWTR_PMSCR (1ULL << HDFGWTR_PMSCR_SHIFT) 1629 #define HDFGWTR_PMBSR_SHIFT 25 1630 #define HDFGWTR_PMBSR (1ULL << HDFGWTR_PMBSR_SHIFT) 1631 #define HDFGWTR_PMBPTR_SHIFT 24 1632 #define HDFGWTR_PMBPTR (1ULL << HDFGWTR_PMBPTR_SHIFT) 1633 #define HDFGWTR_PMBLIMITR_SHIFT 23 1634 #define HDFGWTR_PMBLIMITR (1ULL << HDFGWTR_PMBLIMITR_SHIFT) 1635 #define HDFGWTR_PMCR_SHIFT 21 1636 #define HDFGWTR_PMCR (1ULL << HDFGWTR_PMCR_SHIFT) 1637 #define HDFGWTR_PMSWINC_SHIFT 20 1638 #define HDFGWTR_PMSWINC (1ULL << HDFGWTR_PMSWINC_SHIFT) 1639 #define HDFGWTR_PMSELR_SHIFT 19 1640 #define HDFGWTR_PMSELR (1ULL << HDFGWTR_PMSELR_SHIFT) 1641 #define HDFGWTR_PMOVS_SHIFT 18 1642 #define HDFGWTR_PMOVS (1ULL << HDFGWTR_PMOVS_SHIFT) 1643 #define HDFGWTR_PMINTEN_SHIFT 17 1644 #define HDFGWTR_PMINTEN (1ULL << HDFGWTR_PMINTEN_SHIFT) 1645 #define HDFGWTR_PMCNTEN_SHIFT 16 1646 #define HDFGWTR_PMCNTEN (1ULL << HDFGWTR_PMCNTEN_SHIFT) 1647 #define HDFGWTR_PMCCNTR_SHIFT 15 1648 #define HDFGWTR_PMCCNTR (1ULL << HDFGWTR_PMCCNTR_SHIFT) 1649 #define HDFGWTR_PMCCFILTR_SHIFT 14 1650 #define HDFGWTR_PMCCFILTR (1ULL << HDFGWTR_PMCCFILTR_SHIFT) 1651 #define HDFGWTR_PMEVTYPER_SHIFT 13 1652 #define HDFGWTR_PMEVTYPER (1ULL << HDFGWTR_PMEVTYPER_SHIFT) 1653 #define HDFGWTR_PMEVCNTR_SHIFT 12 1654 #define HDFGWTR_PMEVCNTR (1ULL << HDFGWTR_PMEVCNTR_SHIFT) 1655 #define HDFGWTR_OSDLR_SHIFT 11 1656 #define HDFGWTR_OSDLR (1ULL << HDFGWTR_OSDLR_SHIFT) 1657 #define HDFGWTR_OSECCR_SHIFT 10 1658 #define HDFGWTR_OSECCR (1ULL << HDFGWTR_OSECCR_SHIFT) 1659 #define HDFGWTR_OSLAR_SHIFT 8 1660 #define HDFGWTR_OSLAR (1ULL << HDFGWTR_OSLAR_SHIFT) 1661 #define HDFGWTR_DBGPRCR_SHIFT 7 1662 #define HDFGWTR_DBGPRCR (1ULL << HDFGWTR_DBGPRCR_SHIFT) 1663 #define HDFGWTR_DBGCLAIM_SHIFT 5 1664 #define HDFGWTR_DBGCLAIM (1ULL << HDFGWTR_DBGCLAIM_SHIFT) 1665 #define HDFGWTR_MDSCR_SHIFT 4 1666 #define HDFGWTR_MDSCR (1ULL << HDFGWTR_MDSCR_SHIFT) 1667 #define HDFGWTR_DBGWVR_SHIFT 3 1668 #define HDFGWTR_DBGWVR (1ULL << HDFGWTR_DBGWVR_SHIFT) 1669 #define HDFGWTR_DBGWCR_SHIFT 2 1670 #define HDFGWTR_DBGWCR (1ULL << HDFGWTR_DBGWCR_SHIFT) 1671 #define HDFGWTR_DBGBVR_SHIFT 1 1672 #define HDFGWTR_DBGBVR (1ULL << HDFGWTR_DBGBVR_SHIFT) 1673 #define HDFGWTR_DBGBCR_SHIFT 0 1674 #define HDFGWTR_DBGBCR (1ULL << HDFGWTR_DBGBCR_SHIFT) 1675 1676 /* 1677 * Translation Table Base Register (TTBR) 1678 * 1679 * 63 48 47 x x-1 1 0 1680 * +--------+------------------+------+---+ 1681 * | ASID | Base Address | zero |CnP| 1682 * +--------+------------------+------+---+ 1683 * 1684 */ 1685 #define TTBR_ASID_SHIFT 48 1686 #define TTBR_ASID_MASK 0xffff000000000000 1687 1688 #define TTBR_BADDR_MASK 0x0000fffffffffffe 1689 #define TTBR_CNP 0x0000000000000001 1690 1691 /* 1692 * Memory Attribute Indirection Register 1693 * 1694 * 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0 1695 * +-------+-------+-------+-------+-------+-------+-------+-------+ 1696 * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 | 1697 * +-------+-------+-------+-------+-------+-------+-------+-------+ 1698 * 1699 */ 1700 1701 #define MAIR_ATTR_SHIFT(x) (8*(x)) 1702 1703 /* Strongly ordered or device memory attributes */ 1704 #define MAIR_OUTER_STRONGLY_ORDERED 0x0 1705 #define MAIR_OUTER_DEVICE 0x0 1706 1707 #define MAIR_INNER_STRONGLY_ORDERED 0x0 1708 #define MAIR_INNER_DEVICE 0x4 1709 1710 /* Normal memory attributes */ 1711 #define MAIR_OUTER_NON_CACHEABLE 0x40 1712 #define MAIR_OUTER_WRITE_THROUGH 0x80 1713 #define MAIR_OUTER_WRITE_BACK 0xc0 1714 1715 #define MAIR_INNER_NON_CACHEABLE 0x4 1716 #define MAIR_INNER_WRITE_THROUGH 0x8 1717 #define MAIR_INNER_WRITE_BACK 0xc 1718 1719 /* Allocate policy for cacheable memory */ 1720 #define MAIR_OUTER_WRITE_ALLOCATE 0x10 1721 #define MAIR_OUTER_READ_ALLOCATE 0x20 1722 1723 #define MAIR_INNER_WRITE_ALLOCATE 0x1 1724 #define MAIR_INNER_READ_ALLOCATE 0x2 1725 1726 /* Memory Atribute Encoding */ 1727 1728 /* 1729 * Device memory types: 1730 * G (gathering): multiple reads/writes can be combined 1731 * R (reordering): reads or writes may reach device out of program order 1732 * E (early-acknowledge): writes may return immediately (e.g. PCIe posted writes) 1733 */ 1734 #if HAS_FEAT_XS 1735 1736 #define MAIR_DISABLE_XS 0x00 /* Device Memory, nGnRnE (strongly ordered), XS=1 */ 1737 #define MAIR_DISABLE 0x01 /* Device Memory, nGnRnE (strongly ordered), XS=0 */ 1738 #define MAIR_POSTED_COMBINED_REORDERED_XS 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=1 */ 1739 #define MAIR_POSTED_COMBINED_REORDERED 0x0D /* Device Memory, GRE (reorderable, gathered writes, posted writes), XS=0 */ 1740 #define MAIR_WRITECOMB 0x40 /* Normal Memory, Non-Cacheable, XS=0 */ 1741 #define MAIR_WRITETHRU 0xA0 /* Normal Memory, Write-through, XS=0 */ 1742 #define MAIR_WRITEBACK 0xFF /* Normal Memory, Write-back, XS=0 */ 1743 1744 #if HAS_MTE 1745 #define MAIR_MTE_WRITEBACK 0xF0 /* Normal Tagged Memory, Outer Write-back, Inner Write-back */ 1746 #endif /* HAS_MTE */ 1747 1748 /* 1749 * Memory Attribute Index. If these values change, please also update the pmap 1750 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). 1751 */ 1752 #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ 1753 #define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */ 1754 #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ 1755 #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ 1756 #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory), XS = 0 */ 1757 #define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */ 1758 #define CACHE_ATTRINDX_DISABLE_XS 0x5 /* no cache, no buffer (device memory), XS = 1 */ 1759 /** 1760 * Posted mappings use XS by default, and on newer Apple SoCs there is no fabric-level distinction 1761 * between early-ack and non-early-ack, so just alias POSTED to DISABLE_XS to save a MAIR index. 1762 */ 1763 #define CACHE_ATTRINDX_POSTED CACHE_ATTRINDX_DISABLE_XS 1764 #define CACHE_ATTRINDX_POSTED_REORDERED CACHE_ATTRINDX_DISABLE /* no need for device-nGRE on newer SoCs, fallback to nGnRnE */ 1765 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x6 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=0 */ 1766 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED_XS 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory), XS=1 */ 1767 #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK 1768 #define CACHE_ATTRINDX_N_INDICES (8ULL) 1769 1770 #else 1771 1772 #define MAIR_DISABLE 0x00 /* Device Memory, nGnRnE (strongly ordered) */ 1773 #define MAIR_POSTED 0x04 /* Device Memory, nGnRE (strongly ordered, posted writes) */ 1774 #define MAIR_POSTED_REORDERED 0x08 /* Device Memory, nGRE (reorderable, posted writes) */ 1775 #define MAIR_POSTED_COMBINED_REORDERED 0x0C /* Device Memory, GRE (reorderable, gathered writes, posted writes) */ 1776 #define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */ 1777 #define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */ 1778 #define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */ 1779 #if HAS_MTE 1780 #define MAIR_MTE_WRITEBACK 0xF0 /* Normal Tagged Memory, Outer Write-back, Inner Write-back */ 1781 #endif /* HAS_MTE */ 1782 1783 /* 1784 * Memory Attribute Index. If these values change, please also update the pmap 1785 * LLDB macros that rely on this value (e.g., PmapDecodeTTEARM64). 1786 */ 1787 #define CACHE_ATTRINDX_WRITEBACK 0x0 /* cache enabled, buffer enabled (normal memory) */ 1788 #define CACHE_ATTRINDX_INNERWRITEBACK CACHE_ATTRINDX_WRITEBACK /* legacy compatibility only */ 1789 #define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes (normal memory) */ 1790 #define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled (normal memory) */ 1791 #define CACHE_ATTRINDX_DISABLE 0x3 /* no cache, no buffer (device memory) */ 1792 #define CACHE_ATTRINDX_RESERVED 0x4 /* reserved for internal use */ 1793 #define CACHE_ATTRINDX_POSTED 0x5 /* no cache, no buffer, posted writes (device memory) */ 1794 #define CACHE_ATTRINDX_POSTED_REORDERED 0x6 /* no cache, reorderable access, posted writes (device memory) */ 1795 #define CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 0x7 /* no cache, write gathering, reorderable access, posted writes (device memory) */ 1796 #define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK 1797 #define CACHE_ATTRINDX_N_INDICES (8ULL) 1798 1799 #endif /* HAS_FEAT_XS */ 1800 1801 #if HAS_UCNORMAL_MEM || APPLEVIRTUALPLATFORM 1802 #define CACHE_ATTRINDX_RT CACHE_ATTRINDX_WRITECOMB 1803 #else 1804 #define CACHE_ATTRINDX_RT CACHE_ATTRINDX_POSTED_COMBINED_REORDERED 1805 #endif /* HAS_UCNORMAL_MEM || APPLEVIRTUALPLATFORM */ 1806 1807 #if HAS_MTE 1808 #define CACHE_ATTRINDX_MTE CACHE_ATTRINDX_RESERVED 1809 #endif /* HAS_MTE */ 1810 1811 1812 /* 1813 * Access protection bit values (TTEs and PTEs), stage 1 1814 * 1815 * Bit 1 controls access type (1=RO, 0=RW), bit 0 controls user (1=access, 0=no access) 1816 */ 1817 #define AP_RWNA 0x0 /* priv=read-write, user=no-access */ 1818 #define AP_RWRW 0x1 /* priv=read-write, user=read-write */ 1819 #define AP_RONA 0x2 /* priv=read-only, user=no-access */ 1820 #define AP_RORO 0x3 /* priv=read-only, user=read-only */ 1821 #define AP_MASK 0x3 /* mask to find ap bits */ 1822 1823 /* 1824 * Shareability attributes 1825 */ 1826 #define SH_NONE 0x0 /* Non shareable */ 1827 #define SH_NONE 0x0 /* Device shareable */ 1828 #define SH_DEVICE 0x2 /* Normal memory Inner non shareable - Outer non shareable */ 1829 #define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */ 1830 #define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */ 1831 1832 #if HAS_MTE 1833 /* 1834 * Legacy MTE emulation relies on marking "MTE" pages with 0x1 shareability 1835 * attribute, which is otherwise unused in xnu. This communicates back to 1836 * mSim that the page should be treated specially in the emulation. 1837 */ 1838 #define SH_MTE SH_OUTER_MEMORY 1839 #endif /* HAS_MTE */ 1840 1841 /* 1842 * ARM Page Granule 1843 */ 1844 #ifdef __ARM_16K_PG__ 1845 #define ARM_PGSHIFT 14 1846 #else 1847 #define ARM_PGSHIFT 12 1848 #endif 1849 #define ARM_PGBYTES (1 << ARM_PGSHIFT) 1850 #define ARM_PGMASK (ARM_PGBYTES-1) 1851 1852 /* 1853 * L0 Translation table 1854 * 1855 * 4KB granule size: 1856 * Each translation table is 4KB 1857 * 512 64-bit entries of 512GB (2^39) of address space. 1858 * Covers 256TB (2^48) of address space. 1859 * 1860 * 16KB granule size: 1861 * Each translation table is 16KB 1862 * 2 64-bit entries of 128TB (2^47) of address space. 1863 * Covers 256TB (2^48) of address space. 1864 */ 1865 1866 /* 16K L0 */ 1867 #define ARM_16K_TT_L0_SIZE 0x0000800000000000ULL /* size of area covered by a tte */ 1868 #define ARM_16K_TT_L0_OFFMASK 0x00007fffffffffffULL /* offset within an L0 entry */ 1869 #define ARM_16K_TT_L0_SHIFT 47 /* page descriptor shift */ 1870 #define ARM_16K_TT_L0_INDEX_MASK 0x0000800000000000ULL /* mask for getting index in L0 table from virtual address */ 1871 1872 /* 4K L0 */ 1873 #define ARM_4K_TT_L0_SIZE 0x0000008000000000ULL /* size of area covered by a tte */ 1874 #define ARM_4K_TT_L0_OFFMASK 0x0000007fffffffffULL /* offset within an L0 entry */ 1875 #define ARM_4K_TT_L0_SHIFT 39 /* page descriptor shift */ 1876 #define ARM_4K_TT_L0_INDEX_MASK 0x0000ff8000000000ULL /* mask for getting index in L0 table from virtual address */ 1877 1878 /* 1879 * L1 Translation table 1880 * 1881 * 4KB granule size: 1882 * Each translation table is 4KB 1883 * 512 64-bit entries of 1GB (2^30) of address space. 1884 * Covers 512GB (2^39) of address space. 1885 * 1886 * 16KB granule size: 1887 * Each translation table is 16KB 1888 * 2048 64-bit entries of 64GB (2^36) of address space. 1889 * Covers 128TB (2^47) of address space. 1890 */ 1891 1892 /* 16K L1 */ 1893 #define ARM_16K_TT_L1_SIZE 0x0000001000000000ULL /* size of area covered by a tte */ 1894 #define ARM_16K_TT_L1_OFFMASK 0x0000000fffffffffULL /* offset within an L1 entry */ 1895 #define ARM_16K_TT_L1_SHIFT 36 /* page descriptor shift */ 1896 #define ARM_16K_TT_L1_INDEX_MASK 0x00007ff000000000ULL 1897 1898 /* 4K L1 */ 1899 #define ARM_4K_TT_L1_SIZE 0x0000000040000000ULL /* size of area covered by a tte */ 1900 #define ARM_4K_TT_L1_OFFMASK 0x000000003fffffffULL /* offset within an L1 entry */ 1901 #define ARM_4K_TT_L1_SHIFT 30 /* page descriptor shift */ 1902 1903 #define ARM_4K_TT_L1_INDEX_MASK 0x0000007fc0000000ULL 1904 /* 1905 * Enable concatenated tables if: 1906 * 1. We have a 42-bit PA, and 1907 * 2. Either we're using 4k pages or mixed mode is supported. 1908 */ 1909 #if __ARM_42BIT_PA_SPACE__ 1910 #if !__ARM_16K_PG__ || __ARM_MIXED_PAGE_SIZE__ 1911 /* IPA[39:30] mask for getting index into L1 concatenated table from virtual address */ 1912 #define ARM_4K_TT_L1_40_BIT_CONCATENATED_INDEX_MASK 0x000000ffc0000000ULL 1913 #endif /* !__ARM_16K_PG__ || __ARM_MIXED_PAGE_SIZE__ */ 1914 #endif /* __ARM_42BIT_PA_SPACE__ */ 1915 1916 /* some sugar for getting pointers to page tables and entries */ 1917 #define L1_TABLE_T1_INDEX(va, tcr) (((va) & ARM_PTE_T1_REGION_MASK(tcr)) >> ARM_TT_L1_SHIFT) 1918 #define L2_TABLE_INDEX(va) (((va) & ARM_TT_L2_INDEX_MASK) >> ARM_TT_L2_SHIFT) 1919 #define L3_TABLE_INDEX(va) (((va) & ARM_TT_L3_INDEX_MASK) >> ARM_TT_L3_SHIFT) 1920 1921 #define L2_TABLE_VA(tte) ((tt_entry_t*) phystokv((*(tte)) & ARM_TTE_TABLE_MASK)) 1922 #define L3_TABLE_VA(tte2) ((pt_entry_t*) phystokv((*(tte2)) & ARM_TTE_TABLE_MASK)) 1923 1924 /* 1925 * L2 Translation table 1926 * 1927 * 4KB granule size: 1928 * Each translation table is 4KB 1929 * 512 64-bit entries of 2MB (2^21) of address space. 1930 * Covers 1GB (2^30) of address space. 1931 * 1932 * 16KB granule size: 1933 * Each translation table is 16KB 1934 * 2048 64-bit entries of 32MB (2^25) of address space. 1935 * Covers 64GB (2^36) of address space. 1936 */ 1937 1938 /* 16K L2 */ 1939 #define ARM_16K_TT_L2_SIZE 0x0000000002000000ULL /* size of area covered by a tte */ 1940 #define ARM_16K_TT_L2_OFFMASK 0x0000000001ffffffULL /* offset within an L2 entry */ 1941 #define ARM_16K_TT_L2_SHIFT 25 /* page descriptor shift */ 1942 #define ARM_16K_TT_L2_INDEX_MASK 0x0000000ffe000000ULL /* mask for getting index in L2 table from virtual address */ 1943 1944 /* 4K L2 */ 1945 #define ARM_4K_TT_L2_SIZE 0x0000000000200000ULL /* size of area covered by a tte */ 1946 #define ARM_4K_TT_L2_OFFMASK 0x00000000001fffffULL /* offset within an L2 entry */ 1947 #define ARM_4K_TT_L2_SHIFT 21 /* page descriptor shift */ 1948 #define ARM_4K_TT_L2_INDEX_MASK 0x000000003fe00000ULL /* mask for getting index in L2 table from virtual address */ 1949 1950 /* 1951 * L3 Translation table 1952 * 1953 * 4KB granule size: 1954 * Each translation table is 4KB 1955 * 512 64-bit entries of 4KB (2^12) of address space. 1956 * Covers 2MB (2^21) of address space. 1957 * 1958 * 16KB granule size: 1959 * Each translation table is 16KB 1960 * 2048 64-bit entries of 16KB (2^14) of address space. 1961 * Covers 32MB (2^25) of address space. 1962 */ 1963 1964 /* 16K L3 */ 1965 #define ARM_16K_TT_L3_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ 1966 #define ARM_16K_TT_L3_OFFMASK 0x0000000000003fffULL /* offset within L3 PTE */ 1967 #define ARM_16K_TT_L3_SHIFT 14 /* page descriptor shift */ 1968 #define ARM_16K_TT_L3_INDEX_MASK 0x0000000001ffc000ULL /* mask for page descriptor index */ 1969 1970 /* 4K L3 */ 1971 #define ARM_4K_TT_L3_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ 1972 #define ARM_4K_TT_L3_OFFMASK 0x0000000000000fffULL /* offset within L3 PTE */ 1973 #define ARM_4K_TT_L3_SHIFT 12 /* page descriptor shift */ 1974 #define ARM_4K_TT_L3_INDEX_MASK 0x00000000001ff000ULL /* mask for page descriptor index */ 1975 1976 #ifdef __ARM_16K_PG__ 1977 1978 /* Native L0 defines */ 1979 #define ARM_TT_L0_SIZE ARM_16K_TT_L0_SIZE 1980 #define ARM_TT_L0_OFFMASK ARM_16K_TT_L0_OFFMASK 1981 #define ARM_TT_L0_SHIFT ARM_16K_TT_L0_SHIFT 1982 #define ARM_TT_L0_INDEX_MASK ARM_16K_TT_L0_INDEX_MASK 1983 1984 /* Native L1 defines */ 1985 #define ARM_TT_L1_SIZE ARM_16K_TT_L1_SIZE 1986 #define ARM_TT_L1_OFFMASK ARM_16K_TT_L1_OFFMASK 1987 #define ARM_TT_L1_SHIFT ARM_16K_TT_L1_SHIFT 1988 #define ARM_TT_L1_INDEX_MASK ARM_16K_TT_L1_INDEX_MASK 1989 1990 /* Native L2 defines */ 1991 #define ARM_TT_L2_SIZE ARM_16K_TT_L2_SIZE 1992 #define ARM_TT_L2_OFFMASK ARM_16K_TT_L2_OFFMASK 1993 #define ARM_TT_L2_SHIFT ARM_16K_TT_L2_SHIFT 1994 #define ARM_TT_L2_INDEX_MASK ARM_16K_TT_L2_INDEX_MASK 1995 1996 /* Native L3 defines */ 1997 #define ARM_TT_L3_SIZE ARM_16K_TT_L3_SIZE 1998 #define ARM_TT_L3_OFFMASK ARM_16K_TT_L3_OFFMASK 1999 #define ARM_TT_L3_SHIFT ARM_16K_TT_L3_SHIFT 2000 #define ARM_TT_L3_INDEX_MASK ARM_16K_TT_L3_INDEX_MASK 2001 2002 #else /* !__ARM_16K_PG__ */ 2003 2004 /* Native L0 defines */ 2005 #define ARM_TT_L0_SIZE ARM_4K_TT_L0_SIZE 2006 #define ARM_TT_L0_OFFMASK ARM_4K_TT_L0_OFFMASK 2007 #define ARM_TT_L0_SHIFT ARM_4K_TT_L0_SHIFT 2008 #define ARM_TT_L0_INDEX_MASK ARM_4K_TT_L0_INDEX_MASK 2009 2010 /* Native L1 defines */ 2011 #define ARM_TT_L1_SIZE ARM_4K_TT_L1_SIZE 2012 #define ARM_TT_L1_OFFMASK ARM_4K_TT_L1_OFFMASK 2013 #define ARM_TT_L1_SHIFT ARM_4K_TT_L1_SHIFT 2014 #define ARM_TT_L1_INDEX_MASK ARM_4K_TT_L1_INDEX_MASK 2015 2016 /* Native L2 defines */ 2017 #define ARM_TT_L2_SIZE ARM_4K_TT_L2_SIZE 2018 #define ARM_TT_L2_OFFMASK ARM_4K_TT_L2_OFFMASK 2019 #define ARM_TT_L2_SHIFT ARM_4K_TT_L2_SHIFT 2020 #define ARM_TT_L2_INDEX_MASK ARM_4K_TT_L2_INDEX_MASK 2021 2022 /* Native L3 defines */ 2023 #define ARM_TT_L3_SIZE ARM_4K_TT_L3_SIZE 2024 #define ARM_TT_L3_OFFMASK ARM_4K_TT_L3_OFFMASK 2025 #define ARM_TT_L3_SHIFT ARM_4K_TT_L3_SHIFT 2026 #define ARM_TT_L3_INDEX_MASK ARM_4K_TT_L3_INDEX_MASK 2027 2028 #endif /* !__ARM_16K_PG__ */ 2029 2030 /* 2031 * Convenience definitions for: 2032 * ARM_TT_LEAF: The last level of the configured page table format. 2033 * ARM_TT_TWIG: The second to last level of the configured page table format. 2034 * ARM_TT_ROOT: The first level of the configured page table format. 2035 * 2036 * My apologies to any botanists who may be reading this. 2037 */ 2038 #define ARM_TT_LEAF_SIZE ARM_TT_L3_SIZE 2039 #define ARM_TT_LEAF_OFFMASK ARM_TT_L3_OFFMASK 2040 #define ARM_TT_LEAF_SHIFT ARM_TT_L3_SHIFT 2041 #define ARM_TT_LEAF_INDEX_MASK ARM_TT_L3_INDEX_MASK 2042 2043 #define ARM_TT_TWIG_SIZE ARM_TT_L2_SIZE 2044 #define ARM_TT_TWIG_OFFMASK ARM_TT_L2_OFFMASK 2045 #define ARM_TT_TWIG_SHIFT ARM_TT_L2_SHIFT 2046 #define ARM_TT_TWIG_INDEX_MASK ARM_TT_L2_INDEX_MASK 2047 2048 #define ARM_TT_ROOT_SIZE ARM_TT_L1_SIZE 2049 #define ARM_TT_ROOT_OFFMASK ARM_TT_L1_OFFMASK 2050 #define ARM_TT_ROOT_SHIFT ARM_TT_L1_SHIFT 2051 #define ARM_TT_ROOT_INDEX_MASK ARM_TT_L1_INDEX_MASK 2052 2053 /* 2054 * 4KB granule size: 2055 * 2056 * Level 0 Translation Table Entry 2057 * 2058 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 2059 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2060 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| 2061 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2062 * 2063 * Level 1 Translation Table Entry 2064 * 2065 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 2066 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2067 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| 2068 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2069 * 2070 * Level 1 Translation Block Entry 2071 * 2072 * 63 59 58 55 54 53 52 51 48 47 30 29 12 11 10 9 8 7 6 5 4 2 1 0 2073 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 2074 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:30] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 2075 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 2076 * 2077 * Level 2 Translation Table Entry 2078 * 2079 * 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0 2080 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2081 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| 2082 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2083 * 2084 * Level 2 Translation Block Entry 2085 * 2086 * 63 59 58 55 54 53 52 51 48 47 21 20 12 11 10 9 8 7 6 5 4 2 1 0 2087 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 2088 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:21] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 2089 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 2090 * 2091 * 16KB granule size: 2092 * 2093 * Level 0 Translation Table Entry 2094 * 2095 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 2096 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2097 * |NS| AP |XN|PXN|ignored| zero | L1TableOutputAddress |ignored|1|V| 2098 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2099 * 2100 * Level 1 Translation Table Entry 2101 * 2102 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 2103 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2104 * |NS| AP |XN|PXN|ignored| zero | L2TableOutputAddress |ignored|1|V| 2105 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2106 * 2107 * Level 2 Translation Table Entry 2108 * 2109 * 63 62 61 60 59 58 52 51 48 47 14 13 2 1 0 2110 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2111 * |NS| AP |XN|PXN|ignored| zero | L3TableOutputAddress |ignored|1|V| 2112 * +--+-----+--+---+-------+------+----------------------+-------+-+-+ 2113 * 2114 * Level 2 Translation Block Entry 2115 * 2116 * 63 59 58 55 54 53 52 51 48 47 25 24 12 11 10 9 8 7 6 5 4 2 1 0 2117 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 2118 * | ign |sw use|XN|PXN|HINT| zero | OutputAddress[47:25] | zero |nG|AF| SH | AP |NS|AttrIdx|0|V| 2119 * +-----+------+--+---+----+------+----------------------+------+--+--+----+----+--+-------+-+-+ 2120 * 2121 * where: 2122 * nG: notGlobal bit 2123 * SH: Shareability field 2124 * AP: access protection 2125 * XN: eXecute Never bit 2126 * PXN: Privilege eXecute Never bit 2127 * NS: Non-Secure bit 2128 * HINT: 16 entry continuguous output hint 2129 * AttrIdx: Memory Attribute Index 2130 */ 2131 2132 #define TTE_SHIFT 3 /* shift width of a tte (sizeof(tte) == (1 << TTE_SHIFT)) */ 2133 #ifdef __ARM_16K_PG__ 2134 #define TTE_PGENTRIES (16384 >> TTE_SHIFT) /* number of ttes per page */ 2135 #else 2136 #define TTE_PGENTRIES (4096 >> TTE_SHIFT) /* number of ttes per page */ 2137 #endif 2138 2139 #define ARM_TTE_MAX (TTE_PGENTRIES) 2140 2141 #define ARM_TTE_EMPTY 0x0000000000000000ULL /* unasigned - invalid entry */ 2142 #define ARM_TTE_TYPE_FAULT 0x0000000000000000ULL /* unasigned - invalid entry */ 2143 2144 #define ARM_TTE_VALID 0x0000000000000001ULL /* valid entry */ 2145 2146 #define ARM_TTE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */ 2147 #define ARM_TTE_TYPE_TABLE 0x0000000000000002ULL /* page table type */ 2148 #define ARM_TTE_TYPE_BLOCK 0x0000000000000000ULL /* block entry type */ 2149 #define ARM_TTE_TYPE_L3BLOCK 0x0000000000000002ULL 2150 2151 /* Base AttrIndx transforms */ 2152 #define ARM_TTE_ATTRINDXSHIFT (2) 2153 #define ARM_TTE_ATTRINDXBITS (0x7ULL) 2154 #define ARM_TTE_ATTRINDX(x) (((x) & ARM_TTE_ATTRINDXBITS) << ARM_TTE_ATTRINDXSHIFT) /* memory attributes index */ 2155 #define ARM_TTE_EXTRACT_ATTRINDX(x) (((x) >> ARM_TTE_ATTRINDXSHIFT) & ARM_TTE_ATTRINDXBITS) /* extract memory attributes index */ 2156 #define ARM_TTE_ATTRINDXMASK ARM_TTE_ATTRINDX(ARM_TTE_ATTRINDXBITS) /* mask memory attributes index */ 2157 #define ARM_TTE_ATTRINDX_AIE(x) 0ULL 2158 #define ARM_TTE_ATTRINDXMASK_AIE 0ULL 2159 #define ARM_TTE_EXTRACT_ATTRINDX_AIE(x) 0ULL 2160 2161 #ifdef __ARM_16K_PG__ 2162 /* 2163 * Note that L0/L1 block entries are disallowed for the 16KB granule size; what 2164 * are we doing with these? 2165 */ 2166 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 16KB L3 TTE entry */ 2167 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 128TB section */ 2168 #define ARM_TTE_BLOCK_L1_MASK 0x0000fff000000000ULL /* mask to extract phys address from L1 block entry */ 2169 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 64GB section */ 2170 #define ARM_TTE_BLOCK_L2_MASK 0x0000fffffe000000ULL /* mask to extract phys address from Level 2 Translation Block entry */ 2171 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 32MB section */ 2172 #else 2173 #define ARM_TTE_BLOCK_SHIFT 12 /* entry shift for a 4KB L3 TTE entry */ 2174 #define ARM_TTE_BLOCK_L0_SHIFT ARM_TT_L0_SHIFT /* block shift for 2048GB section */ 2175 #define ARM_TTE_BLOCK_L1_MASK 0x0000ffffc0000000ULL /* mask to extract phys address from L1 block entry */ 2176 #define ARM_TTE_BLOCK_L1_SHIFT ARM_TT_L1_SHIFT /* block shift for 1GB section */ 2177 #define ARM_TTE_BLOCK_L2_MASK 0x0000ffffffe00000ULL /* mask to extract phys address from Level 2 Translation Block entry */ 2178 #define ARM_TTE_BLOCK_L2_SHIFT ARM_TT_L2_SHIFT /* block shift for 2MB section */ 2179 #endif 2180 2181 #define ARM_TTE_BLOCK_APSHIFT 6 2182 #define ARM_TTE_BLOCK_AP(x) ((x)<<ARM_TTE_BLOCK_APSHIFT) /* access protection */ 2183 #define ARM_TTE_BLOCK_APMASK (0x3 << ARM_TTE_BLOCK_APSHIFT) 2184 2185 #define ARM_TTE_BLOCK_ATTRINDX(x) (ARM_TTE_ATTRINDX_AIE(x) | ARM_TTE_ATTRINDX(x)) /* memory attributes index */ 2186 #define ARM_TTE_BLOCK_ATTRINDXMASK (ARM_TTE_ATTRINDXMASK_AIE | ARM_TTE_ATTRINDXMASK) /* mask memory attributes index */ 2187 2188 #define ARM_TTE_BLOCK_SH(x) ((x) << 8) /* access shared */ 2189 #define ARM_TTE_BLOCK_SHMASK (0x3ULL << 8) /* mask access shared */ 2190 2191 #define ARM_TTE_BLOCK_AF 0x0000000000000400ULL /* value for access */ 2192 #define ARM_TTE_BLOCK_AFMASK 0x0000000000000400ULL /* access mask */ 2193 2194 #define ARM_TTE_BLOCK_NG 0x0000000000000800ULL /* value for a global mapping */ 2195 #define ARM_TTE_BLOCK_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ 2196 2197 #define ARM_TTE_BLOCK_NS 0x0000000000000020ULL /* value for a secure mapping */ 2198 #define ARM_TTE_BLOCK_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ 2199 2200 #define ARM_TTE_BLOCK_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ 2201 #define ARM_TTE_BLOCK_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ 2202 2203 #define ARM_TTE_BLOCK_NX 0x0040000000000000ULL /* value for no execute */ 2204 #define ARM_TTE_BLOCK_NXMASK 0x0040000000000000ULL /* no execute mask */ 2205 2206 #define ARM_TTE_BLOCK_WIRED 0x0400000000000000ULL /* value for software wired bit */ 2207 #define ARM_TTE_BLOCK_WIREDMASK 0x0400000000000000ULL /* software wired mask */ 2208 2209 #define ARM_TTE_BLOCK_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ 2210 #define ARM_TTE_BLOCK_WRITEABLEMASK 0x0800000000000000ULL /* software writeable mask */ 2211 2212 #define ARM_TTE_TABLE_MASK 0x0000fffffffff000ULL /* mask for extracting pointer to next table (works at any level) */ 2213 2214 #define ARM_TTE_TABLE_APSHIFT 61 2215 #define ARM_TTE_TABLE_AP_MASK (0x3ULL << ARM_TTE_TABLE_APSHIFT) 2216 #define ARM_TTE_TABLE_AP_NO_EFFECT 0x0ULL 2217 #define ARM_TTE_TABLE_AP_USER_NA 0x1ULL 2218 #define ARM_TTE_TABLE_AP_RO 0x2ULL 2219 #define ARM_TTE_TABLE_AP_KERN_RO 0x3ULL 2220 #define ARM_TTE_TABLE_AP(x) ((x) << ARM_TTE_TABLE_APSHIFT) /* access protection */ 2221 2222 #define ARM_TTE_TABLE_NS 0x8000000000000020ULL /* value for a secure mapping */ 2223 #define ARM_TTE_TABLE_NS_MASK 0x8000000000000020ULL /* notSecure mapping mask */ 2224 2225 #define ARM_TTE_TABLE_XN 0x1000000000000000ULL /* value for no execute */ 2226 #define ARM_TTE_TABLE_XNMASK 0x1000000000000000ULL /* no execute mask */ 2227 2228 #define ARM_TTE_TABLE_PXN 0x0800000000000000ULL /* value for privilege no execute bit */ 2229 #define ARM_TTE_TABLE_PXNMASK 0x0800000000000000ULL /* privilege execute mask */ 2230 2231 /** Software use TTE bits which the kernel actually uses. */ 2232 #define ARM_TTE_TABLE_SW_RESERVED_MASK (0x0000000000000000ULL) 2233 2234 /** 2235 * Table TTE bits which must be set to zero by software when the TTE is valid. 2236 */ 2237 #define ARM_TTE_TABLE_RESERVED_MASK \ 2238 (~(ARM_TTE_VALID | \ 2239 ARM_TTE_TYPE_MASK | \ 2240 ARM_TTE_TABLE_MASK | \ 2241 ARM_TTE_TABLE_SW_RESERVED_MASK | \ 2242 ARM_TTE_TABLE_PXNMASK | \ 2243 ARM_TTE_TABLE_XNMASK | \ 2244 ARM_TTE_TABLE_AP_MASK | \ 2245 ARM_TTE_TABLE_NS_MASK)) 2246 2247 #if __ARM_KERNEL_PROTECT__ 2248 #define ARM_TTE_BOOT_BLOCK_LOWER \ 2249 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ 2250 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF | ARM_TTE_BLOCK_NG) 2251 #else /* __ARM_KERNEL_PROTECT__ */ 2252 #define ARM_TTE_BOOT_BLOCK_LOWER \ 2253 (ARM_TTE_TYPE_BLOCK | ARM_TTE_VALID | ARM_TTE_BLOCK_SH(SH_OUTER_MEMORY) | \ 2254 ARM_TTE_BLOCK_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_TTE_BLOCK_AF) 2255 #endif /* __ARM_KERNEL_PROTECT__ */ 2256 #define ARM_TTE_BOOT_BLOCK_UPPER ARM_TTE_BLOCK_NX 2257 2258 #define ARM_TTE_BOOT_TABLE (ARM_TTE_TYPE_TABLE | ARM_TTE_VALID ) 2259 /* 2260 * L3 Translation table 2261 * 2262 * 4KB granule size: 2263 * Each translation table is 4KB 2264 * 512 64-bit entries of 4KB (2^12) of address space. 2265 * Covers 2MB (2^21) of address space. 2266 * 2267 * 16KB granule size: 2268 * Each translation table is 16KB 2269 * 2048 64-bit entries of 16KB (2^14) of address space. 2270 * Covers 32MB (2^25) of address space. 2271 */ 2272 2273 #ifdef __ARM_16K_PG__ 2274 #define ARM_PTE_SIZE 0x0000000000004000ULL /* size of area covered by a tte */ 2275 #define ARM_PTE_OFFMASK 0x0000000000003fffULL /* offset within pte area */ 2276 #define ARM_PTE_SHIFT 14 /* page descriptor shift */ 2277 #define ARM_PTE_MASK 0x0000ffffffffc000ULL /* mask for output address in PTE */ 2278 #else 2279 #define ARM_PTE_SIZE 0x0000000000001000ULL /* size of area covered by a tte */ 2280 #define ARM_PTE_OFFMASK 0x0000000000000fffULL /* offset within pte area */ 2281 #define ARM_PTE_SHIFT 12 /* page descriptor shift */ 2282 #define ARM_PTE_MASK 0x0000fffffffff000ULL /* mask for output address in PTE */ 2283 #endif 2284 2285 #define ARM_PTE_T0SZ(TCR) (((TCR) >> TCR_T0SZ_SHIFT) & TCR_T0SZ_MASK) 2286 #define ARM_PTE_T1SZ(TCR) (((TCR) >> TCR_T1SZ_SHIFT) & TCR_T1SZ_MASK) 2287 #define ARM_PTE_REGION_MASK(SZ) ((1ULL << (64 - (SZ))) - 1) 2288 #define ARM_TTE_PA_MASK 0x0000fffffffff000ULL 2289 2290 /* Handle Page table address bits in a TCR-aware way. */ 2291 #define ARM_PTE_T0_REGION_MASK(TCR) (ARM_PTE_REGION_MASK(ARM_PTE_T0SZ(TCR))) 2292 #define ARM_PTE_T1_REGION_MASK(TCR) (ARM_PTE_REGION_MASK(ARM_PTE_T1SZ(TCR))) 2293 2294 /* 2295 * L3 Page table entries 2296 * 2297 * The following page table entry types are possible: 2298 * 2299 * fault page entry 2300 * 63 2 0 2301 * +------------------------------+--+ 2302 * | ignored |00| 2303 * +------------------------------+--+ 2304 * 2305 * 2306 * 63 59 58 55 54 53 52 51 50 47 48 12 11 10 9 8 7 6 5 4 2 1 0 2307 * +-----+------+--+---+----+---+--+----+----------------------+--+--+----+----+--+-------+-+-+ 2308 * | ign |sw use|XN|PXN|HINT|DBM|GP|zero| OutputAddress[47:12] |nG|AF| SH | AP |NS|AttrIdx|1|V| 2309 * +-----+------+--+---+----+---+--+----+----------------------+--+--+----+----+--+-------+-+-+ 2310 * 2311 * where: 2312 * nG: notGlobal bit 2313 * SH: Shareability field 2314 * AP: access protection 2315 * XN: eXecute Never bit 2316 * PXN: Privilege eXecute Never bit 2317 * NS: Non-Secure bit 2318 * HINT: 16 entry continuguous output hint 2319 * DBM: Dirty Bit Modifier 2320 * GP: Guraded Page 2321 * AttrIdx: Memory Attribute Index 2322 */ 2323 2324 #define PTE_SHIFT 3 /* shift width of a pte (sizeof(pte) == (1 << PTE_SHIFT)) */ 2325 #ifdef __ARM_16K_PG__ 2326 #define PTE_PGENTRIES (16384 >> PTE_SHIFT) /* number of ptes per page */ 2327 #else 2328 #define PTE_PGENTRIES (4096 >> PTE_SHIFT) /* number of ptes per page */ 2329 #endif 2330 2331 #define ARM_PTE_EMPTY 0x0000000000000000ULL /* unassigned - invalid entry */ 2332 2333 /* markers for (invalid) PTE for a page sent to compressor */ 2334 #define ARM_PTE_COMPRESSED 0x8000000000000000ULL /* compressed... */ 2335 #define ARM_PTE_COMPRESSED_ALT 0x4000000000000000ULL /* ... and was "alt_acct" */ 2336 #define ARM_PTE_COMPRESSED_MASK 0xC000000000000000ULL 2337 2338 #define ARM_PTE_TYPE_VALID 0x0000000000000003ULL /* valid L3 entry: includes bit #1 (counterintuitively) */ 2339 #define ARM_PTE_TYPE_FAULT 0x0000000000000000ULL /* invalid L3 entry */ 2340 #define ARM_PTE_TYPE_MASK 0x0000000000000003ULL /* mask to get pte type */ 2341 2342 /* This mask works for both 16K and 4K pages because bits 12-13 will be zero in 16K pages */ 2343 #define ARM_PTE_PAGE_MASK 0x0000FFFFFFFFF000ULL /* output address mask for page */ 2344 #define ARM_PTE_PAGE_SHIFT 12 /* page shift for the output address in the entry */ 2345 2346 #define ARM_PTE_AP(x) ((x) << 6) /* access protections */ 2347 #define ARM_PTE_APMASK (0x3ULL << 6) /* mask access protections */ 2348 #define ARM_PTE_EXTRACT_AP(x) (((x) >> 6) & 0x3ULL) /* extract access protections from PTE */ 2349 2350 #define ARM_PTE_ATTRINDX(x) (uint64_t)(ARM_TTE_ATTRINDX_AIE(x) | ARM_TTE_ATTRINDX(x)) /* memory attributes index */ 2351 #define ARM_PTE_ATTRINDXMASK (ARM_TTE_ATTRINDXMASK_AIE | ARM_TTE_ATTRINDXMASK) /* mask memory attributes index */ 2352 #define ARM_PTE_EXTRACT_ATTRINDX(x) (ARM_TTE_EXTRACT_ATTRINDX_AIE(x) | ARM_TTE_EXTRACT_ATTRINDX(x)) /* extract memory attributes index */ 2353 2354 #define ARM_PTE_SH(x) ((x) << 8) /* access shared */ 2355 #define ARM_PTE_SHMASK (0x3ULL << 8) /* mask access shared */ 2356 2357 #define ARM_PTE_AF 0x0000000000000400ULL /* value for access */ 2358 #define ARM_PTE_AFMASK 0x0000000000000400ULL /* access mask */ 2359 2360 #define ARM_PTE_NG 0x0000000000000800ULL /* value for a global mapping */ 2361 #define ARM_PTE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */ 2362 2363 #define ARM_PTE_NS 0x0000000000000020ULL /* value for a secure mapping */ 2364 #define ARM_PTE_NS_MASK 0x0000000000000020ULL /* notSecure mapping mask */ 2365 2366 #define ARM_PTE_HINT 0x0010000000000000ULL /* value for contiguous entries hint */ 2367 #define ARM_PTE_HINT_MASK 0x0010000000000000ULL /* mask for contiguous entries hint */ 2368 2369 #define ARM_PTE_GP 0x0004000000000000ULL /* value marking a guarded page */ 2370 #define ARM_PTE_GP_MASK 0x0004000000000000ULL /* mask for a guarded page mark */ 2371 2372 #if __ARM_16K_PG__ 2373 #define ARM_PTE_HINT_ENTRIES 128ULL /* number of entries the hint covers */ 2374 #define ARM_PTE_HINT_ENTRIES_SHIFT 7ULL /* shift to construct the number of entries */ 2375 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFE00000ULL /* mask to extract the starting hint address */ 2376 #define ARM_PTE_HINT_ADDR_SHIFT 21 /* shift for the hint address */ 2377 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFE00000ULL /* mask to extract the starting hint address */ 2378 #else 2379 #define ARM_PTE_HINT_ENTRIES 16ULL /* number of entries the hint covers */ 2380 #define ARM_PTE_HINT_ENTRIES_SHIFT 4ULL /* shift to construct the number of entries */ 2381 #define ARM_PTE_HINT_ADDR_MASK 0x0000FFFFFFFF0000ULL /* mask to extract the starting hint address */ 2382 #define ARM_PTE_HINT_ADDR_SHIFT 16 /* shift for the hint address */ 2383 #define ARM_KVA_HINT_ADDR_MASK 0xFFFFFFFFFFFF0000ULL /* mask to extract the starting hint address */ 2384 #endif 2385 2386 #define ARM_PTE_PNX 0x0020000000000000ULL /* value for privilege no execute bit */ 2387 #define ARM_PTE_PXN ARM_PTE_PNX 2388 #define ARM_PTE_PNXMASK 0x0020000000000000ULL /* privilege no execute mask */ 2389 2390 #define ARM_PTE_NX 0x0040000000000000ULL /* value for no execute bit */ 2391 #define ARM_PTE_XN ARM_PTE_NX 2392 #define ARM_PTE_NXMASK 0x0040000000000000ULL /* no execute mask */ 2393 2394 #define ARM_PTE_XMASK (ARM_PTE_PNXMASK | ARM_PTE_NXMASK) 2395 2396 #define ARM_PTE_GUARDED 0x0004000000000000ULL /* value for "guarded"/BTI enforcing code page */ 2397 #define ARM_PTE_GUARDED_MASK (PTE_GUARDED) 2398 2399 #define ARM_PTE_WIRED 0x0400000000000000ULL /* value for software wired bit */ 2400 #define ARM_PTE_WIRED_MASK 0x0400000000000000ULL /* software wired mask */ 2401 2402 #define ARM_PTE_WRITEABLE 0x0800000000000000ULL /* value for software writeable bit */ 2403 #define ARM_PTE_WRITEABLE_MASK 0x0800000000000000ULL /* software writeable mask */ 2404 #define ARM_PTE_WRITABLE ARM_PTE_WRITEABLE 2405 2406 /** Software use PTE bits which the kernel actually uses. */ 2407 #define ARM_PTE_SW_RESERVED_MASK (ARM_PTE_WIRED_MASK | ARM_PTE_WRITEABLE_MASK) 2408 2409 /** 2410 * PTE bits which must be set to zero by software when the PTE is valid. 2411 */ 2412 #define ARM_PTE_RESERVED_MASK \ 2413 (~(ARM_PTE_TYPE_MASK | \ 2414 ARM_PTE_ATTRINDXMASK | \ 2415 ARM_PTE_NS_MASK | \ 2416 ARM_PTE_APMASK | \ 2417 ARM_PTE_SHMASK | \ 2418 ARM_PTE_AFMASK | \ 2419 ARM_PTE_NG_MASK | \ 2420 ARM_PTE_PAGE_MASK | \ 2421 ARM_PTE_GP_MASK | \ 2422 ARM_PTE_HINT_MASK | \ 2423 ARM_PTE_PNXMASK | \ 2424 ARM_PTE_NXMASK | \ 2425 ARM_PTE_SW_RESERVED_MASK)) 2426 2427 #define ARM_PTE_BOOT_PAGE_BASE \ 2428 (ARM_PTE_TYPE_VALID | ARM_PTE_SH(SH_OUTER_MEMORY) | \ 2429 ARM_PTE_ATTRINDX(CACHE_ATTRINDX_WRITEBACK) | ARM_PTE_AF) 2430 2431 #if __ARM_KERNEL_PROTECT__ 2432 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE | ARM_PTE_NG) 2433 #else /* __ARM_KERNEL_PROTECT__ */ 2434 #define ARM_PTE_BOOT_PAGE (ARM_PTE_BOOT_PAGE_BASE) 2435 #endif /* __ARM_KERNEL_PROTECT__ */ 2436 2437 /* 2438 * TLBI appers to only deal in 4KB page addresses, so give 2439 * it an explicit shift of 12. 2440 */ 2441 #define TLBI_ADDR_SHIFT (0) 2442 #define TLBI_ADDR_SIZE (44) 2443 #define TLBI_ADDR_MASK ((1ULL << TLBI_ADDR_SIZE) - 1) 2444 #define TLBI_IPA_SHIFT (0) 2445 #define TLBI_IPA_SIZE (36) 2446 #define TLBI_IPA_MASK ((1ULL << TLBI_IPA_SIZE) - 1) 2447 #define TLBI_ASID_SHIFT (48) 2448 #define TLBI_ASID_SIZE (16) 2449 #define TLBI_ASID_MASK (((1ULL << TLBI_ASID_SIZE) - 1)) 2450 2451 #define RTLBI_ADDR_SIZE (37) 2452 #define RTLBI_ADDR_MASK ((1ULL << RTLBI_ADDR_SIZE) - 1) 2453 #define RTLBI_ADDR_SHIFT ARM_TT_L3_SHIFT 2454 #define RTLBI_TG(_page_shift_) ((uint64_t)((((_page_shift_) - 12) >> 1) + 1) << 46) 2455 #define RTLBI_SCALE_SHIFT (44) 2456 #define RTLBI_NUM_SHIFT (39) 2457 2458 /* 2459 * RCTX instruction operand fields. 2460 */ 2461 #define RCTX_EL_SHIFT (24) 2462 #define RCTX_EL_SIZE (2) 2463 #define RCTX_EL_MASK (((1ULL << RCTX_EL_SIZE) - 1) << RCTX_EL_SHIFT) 2464 #define RCTX_EL(x) ((x << RCTX_EL_SHIFT) & RCTX_EL_MASK) 2465 #define RCTX_ASID_SHIFT (0) 2466 #define RCTX_ASID_SIZE (16) 2467 #define RCTX_ASID_MASK (((1ULL << RCTX_ASID_SIZE) - 1) << RCTX_ASID_SHIFT) 2468 #define RCTX_ASID(x) ((x << RCTX_ASID_SHIFT) & RCTX_ASID_MASK) 2469 2470 /* 2471 * Exception Syndrome Register 2472 * 2473 * 63 56 55 32 31 26 25 24 0 2474 * +------+------+------+--+------------------+ 2475 * | RES0 | ISS2 | EC |IL| ISS | 2476 * +------+------+------+--+------------------+ 2477 * 2478 * RES0 - Reserved bits. 2479 * ISS2 - Instruction Specific Syndrome 2. 2480 * EC - Exception Class 2481 * IL - Instruction Length 2482 * ISS - Instruction Specific Syndrome 2483 * 2484 * Note: The ISS can have many forms. These are defined separately below. 2485 */ 2486 2487 #define ESR_EC_SHIFT 26 2488 #define ESR_EC_WIDTH 6 2489 #define ESR_EC_MASK (0x3FULL << ESR_EC_SHIFT) 2490 #define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT) 2491 2492 #define ESR_IL_SHIFT 25 2493 #define ESR_IL (1 << ESR_IL_SHIFT) 2494 2495 #define ESR_INSTR_IS_2BYTES(x) (!(x & ESR_IL)) 2496 2497 #define ESR_ISS_MASK 0x01FFFFFF 2498 #define ESR_ISS(x) (x & ESR_ISS_MASK) 2499 2500 #define ESR_ISS2_SHIFT 32 2501 #define ESR_ISS2_MASK 0xFFFFFF00000000 2502 #define ESR_ISS2(x) ((x & ESR_ISS2_MASK) >> ESR_ISS2_SHIFT) 2503 2504 #ifdef __ASSEMBLER__ 2505 /* Define only the classes we need to test in the exception vectors. */ 2506 #define ESR_EC_UNCATEGORIZED 0x00 2507 #define ESR_EC_BTI_FAIL 0x0D 2508 #define ESR_EC_SVC_64 0x15 2509 #define ESR_EC_HVC_64 0x16 2510 #define ESR_EC_PAC_FAIL 0x1C 2511 #define ESR_EC_IABORT_EL1 0x21 2512 #define ESR_EC_DABORT_EL1 0x25 2513 #define ESR_EC_SP_ALIGN 0x26 2514 #define ESR_EC_BRK_AARCH64 0x3C 2515 #else 2516 typedef enum { 2517 ESR_EC_UNCATEGORIZED = 0x00, 2518 ESR_EC_WFI_WFE = 0x01, 2519 ESR_EC_MCR_MRC_CP15_TRAP = 0x03, 2520 ESR_EC_MCRR_MRRC_CP15_TRAP = 0x04, 2521 ESR_EC_MCR_MRC_CP14_TRAP = 0x05, 2522 ESR_EC_LDC_STC_CP14_TRAP = 0x06, 2523 ESR_EC_TRAP_SIMD_FP = 0x07, 2524 ESR_EC_PTRAUTH_INSTR_TRAP = 0x09, 2525 ESR_EC_MCRR_MRRC_CP14_TRAP = 0x0c, 2526 ESR_EC_BTI_FAIL = 0x0d, 2527 ESR_EC_ILLEGAL_INSTR_SET = 0x0e, 2528 ESR_EC_SVC_32 = 0x11, 2529 ESR_EC_HVC_32 = 0x12, 2530 ESR_EC_SVC_64 = 0x15, 2531 ESR_EC_HVC_64 = 0x16, 2532 ESR_EC_MSR_TRAP = 0x18, 2533 #if __has_feature(ptrauth_calls) 2534 ESR_EC_PAC_FAIL = 0x1C, 2535 #endif /* __has_feature(ptrauth_calls) */ 2536 #if HAS_ARM_FEAT_SME 2537 ESR_EC_SME = 0x1D, 2538 #endif 2539 ESR_EC_IABORT_EL0 = 0x20, 2540 ESR_EC_IABORT_EL1 = 0x21, 2541 ESR_EC_PC_ALIGN = 0x22, 2542 ESR_EC_DABORT_EL0 = 0x24, 2543 ESR_EC_DABORT_EL1 = 0x25, 2544 ESR_EC_SP_ALIGN = 0x26, 2545 ESR_EC_FLOATING_POINT_32 = 0x28, 2546 ESR_EC_FLOATING_POINT_64 = 0x2C, 2547 ESR_EC_SERROR_INTERRUPT = 0x2F, 2548 ESR_EC_BKPT_REG_MATCH_EL0 = 0x30, // Breakpoint Debug event taken to the EL from a lower EL. 2549 ESR_EC_BKPT_REG_MATCH_EL1 = 0x31, // Breakpoint Debug event taken to the EL from the EL. 2550 ESR_EC_SW_STEP_DEBUG_EL0 = 0x32, // Software Step Debug event taken to the EL from a lower EL. 2551 ESR_EC_SW_STEP_DEBUG_EL1 = 0x33, // Software Step Debug event taken to the EL from the EL. 2552 ESR_EC_WATCHPT_MATCH_EL0 = 0x34, // Watchpoint Debug event taken to the EL from a lower EL. 2553 ESR_EC_WATCHPT_MATCH_EL1 = 0x35, // Watchpoint Debug event taken to the EL from the EL. 2554 ESR_EC_BKPT_AARCH32 = 0x38, 2555 ESR_EC_BRK_AARCH64 = 0x3C, 2556 } esr_exception_class_t; 2557 2558 typedef enum { 2559 FSC_TRANSLATION_FAULT_L0 = 0x04, 2560 FSC_TRANSLATION_FAULT_L1 = 0x05, 2561 FSC_TRANSLATION_FAULT_L2 = 0x06, 2562 FSC_TRANSLATION_FAULT_L3 = 0x07, 2563 FSC_ACCESS_FLAG_FAULT_L1 = 0x09, 2564 FSC_ACCESS_FLAG_FAULT_L2 = 0x0A, 2565 FSC_ACCESS_FLAG_FAULT_L3 = 0x0B, 2566 FSC_PERMISSION_FAULT_L1 = 0x0D, 2567 FSC_PERMISSION_FAULT_L2 = 0x0E, 2568 FSC_PERMISSION_FAULT_L3 = 0x0F, 2569 FSC_SYNC_EXT_ABORT = 0x10, 2570 FSC_SYNC_TAG_CHECK_FAULT = 0x11, 2571 FSC_SYNC_EXT_ABORT_TT_L1 = 0x15, 2572 FSC_SYNC_EXT_ABORT_TT_L2 = 0x16, 2573 FSC_SYNC_EXT_ABORT_TT_L3 = 0x17, 2574 FSC_SYNC_PARITY = 0x18, 2575 FSC_ASYNC_PARITY = 0x19, 2576 FSC_SYNC_PARITY_TT_L1 = 0x1D, 2577 FSC_SYNC_PARITY_TT_L2 = 0x1E, 2578 FSC_SYNC_PARITY_TT_L3 = 0x1F, 2579 FSC_ALIGNMENT_FAULT = 0x21, 2580 FSC_DEBUG_FAULT = 0x22, 2581 } fault_status_t; 2582 #endif /* ASSEMBLER */ 2583 2584 /* 2585 * SVC event 2586 * 24 16 15 0 2587 * +---------+-----+ 2588 * |000000000| IMM | 2589 * +---------+-----+ 2590 * 2591 * where: 2592 * IMM: Immediate value 2593 */ 2594 2595 #define ISS_SVC_IMM_MASK 0xffff 2596 #define ISS_SVC_IMM(x) ((x) & ISS_SVC_IMM_MASK) 2597 2598 /* 2599 * HVC event 2600 * 24 16 15 0 2601 * +---------+-----+ 2602 * |000000000| IMM | 2603 * +---------+-----+ 2604 * 2605 * where: 2606 * IMM: Immediate value 2607 */ 2608 2609 #define ISS_HVC_IMM_MASK 0xffff 2610 #define ISS_HVC_IMM(x) ((x) & ISS_HVC_IMM_MASK) 2611 2612 2613 /* 2614 * Software step debug event ISS (EL1) 2615 * 24 23 6 5 0 2616 * +---+-----------------+--+------+ 2617 * |ISV|00000000000000000|EX| IFSC | 2618 * +---+-----------------+--+------+ 2619 * 2620 * where: 2621 * ISV: Instruction syndrome valid 2622 * EX: Exclusive access 2623 * IFSC: Instruction Fault Status Code 2624 */ 2625 2626 #define ISS_SSDE_ISV_SHIFT 24 2627 #define ISS_SSDE_ISV (0x1 << ISS_SSDE_ISV_SHIFT) 2628 2629 #define ISS_SSDE_EX_SHIFT 6 2630 #define ISS_SSDE_EX (0x1 << ISS_SSDE_EX_SHIFT) 2631 2632 #define ISS_SSDE_FSC_MASK 0x3F 2633 #define ISS_SSDE_FSC(x) (x & ISS_SSDE_FSC_MASK) 2634 2635 /* 2636 * Instruction Abort ISS (EL1) 2637 * 24 10 9 5 0 2638 * +--------------+---+--+---+------+ 2639 * |00000000000000|FnV|EA|000| IFSC | 2640 * +--------------+---+--+---+------+ 2641 * 2642 * where: 2643 * FnV: FAR not Valid 2644 * EA: External Abort type 2645 * IFSC: Instruction Fault Status Code 2646 */ 2647 2648 #define ISS_IA_FNV_SHIFT 10 2649 #define ISS_IA_FNV (0x1 << ISS_IA_FNV_SHIFT) 2650 2651 #define ISS_IA_EA_SHIFT 9 2652 #define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT) 2653 2654 #define ISS_IA_FSC_MASK 0x3F 2655 #define ISS_IA_FSC(x) (x & ISS_IA_FSC_MASK) 2656 2657 2658 /* 2659 * Data Abort ISS (EL1) 2660 * 2661 * 24 10 9 8 7 6 5 0 2662 * +--------------+---+--+--+-----+---+----+ 2663 * |00000000000000|FnV|EA|CM|S1PTW|WnR|DFSC| 2664 * +--------------+---+--+--+-----+---+----+ 2665 * 2666 * where: 2667 * FnV: FAR not Valid 2668 * EA: External Abort type 2669 * CM: Cache Maintenance operation 2670 * WnR: Write not Read 2671 * S1PTW: Stage 2 exception on Stage 1 page table walk 2672 * DFSC: Data Fault Status Code 2673 */ 2674 #define ISS_DA_FNV_SHIFT 10 2675 #define ISS_DA_FNV (0x1 << ISS_DA_FNV_SHIFT) 2676 2677 #define ISS_DA_ISV_SHIFT 24 2678 #define ISS_DA_ISV (0x1 << ISS_DA_ISV_SHIFT) 2679 2680 #define ISS_DA_SAS_MASK 0x3 2681 #define ISS_DA_SAS_SHIFT 22 2682 #define ISS_DA_SAS(x) (((x) >> ISS_DA_SAS_SHIFT) & ISS_DA_SAS_MASK) 2683 2684 #define ISS_DA_SRT_MASK 0x1f 2685 #define ISS_DA_SRT_SHIFT 16 2686 #define ISS_DA_SRT(x) (((x) >> ISS_DA_SRT_SHIFT) & ISS_DA_SRT_MASK) 2687 2688 #define ISS_DA_EA_SHIFT 9 2689 #define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT) 2690 2691 #define ISS_DA_CM_SHIFT 8 2692 #define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT) 2693 2694 #define ISS_DA_WNR_SHIFT 6 2695 #define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT) 2696 2697 #define ISS_DA_S1PTW_SHIFT 7 2698 #define ISS_DA_S1PTW (0x1 << ISS_DA_S1PTW_SHIFT) 2699 2700 #define ISS_DA_FSC_MASK 0x3F 2701 #define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK) 2702 2703 /* 2704 * Floating Point Exception ISS (EL1) 2705 * 2706 * 24 23 22 8 7 4 3 2 1 0 2707 * +-+---+---------------+---+--+---+---+---+---+---+ 2708 * |0|TFV|000000000000000|IDF|00|IXF|UFF|OFF|DZF|IOF| 2709 * +-+---+---------------+---+--+---+---+---+---+---+ 2710 * 2711 * where: 2712 * TFV: Trapped Fault Valid 2713 * IDF: Input Denormal Exception 2714 * IXF: Input Inexact Exception 2715 * UFF: Underflow Exception 2716 * OFF: Overflow Exception 2717 * DZF: Divide by Zero Exception 2718 * IOF: Invalid Operation Exception 2719 */ 2720 #define ISS_FP_TFV_SHIFT 23 2721 #define ISS_FP_TFV (0x1 << ISS_FP_TFV_SHIFT) 2722 2723 #define ISS_FP_IDF_SHIFT 7 2724 #define ISS_FP_IDF (0x1 << ISS_FP_IDF_SHIFT) 2725 2726 #define ISS_FP_IXF_SHIFT 4 2727 #define ISS_FP_IXF (0x1 << ISS_FP_IXF_SHIFT) 2728 2729 #define ISS_FP_UFF_SHIFT 3 2730 #define ISS_FP_UFF (0x1 << ISS_FP_UFF_SHIFT) 2731 2732 #define ISS_FP_OFF_SHIFT 2 2733 #define ISS_FP_OFF (0x1 << ISS_FP_OFF_SHIFT) 2734 2735 #define ISS_FP_DZF_SHIFT 1 2736 #define ISS_FP_DZF (0x1 << ISS_FP_DZF_SHIFT) 2737 2738 #define ISS_FP_IOF_SHIFT 0 2739 #define ISS_FP_IOF (0x1 << ISS_FP_IOF_SHIFT) 2740 2741 /* 2742 * Breakpoint Exception ISS (EL1) 2743 * 24 16 0 2744 * +---------+---------+ 2745 * |000000000| Comment | 2746 * +---------+---------+ 2747 * 2748 * where: 2749 * Comment: Instruction Comment Field Value 2750 */ 2751 #define ISS_BRK_COMMENT_MASK 0xFFFF 2752 #define ISS_BRK_COMMENT(x) (x & ISS_BRK_COMMENT_MASK) 2753 2754 /* 2755 * Data Abort ISS2 (EL1) 2756 * 2757 * 23 12 11 10 9 8 7 6 5 4 0 2758 * +--------------+--------+-----+-----------+-----+-------------+---------+----------+----+ 2759 * | 000000000000 | HDBSSF | TnD | TagAccess | GCS | AssuredOnly | Overlay | DirtyBit | Xs | 2760 * +--------------+--------+-----+-----------+-----+-------------+---------+----------+----+ 2761 */ 2762 #define ISS2_DA_TND_SHIFT 10 2763 #define ISS2_DA_TND (0x1 << ISS2_DA_TND_SHIFT) 2764 2765 2766 /* 2767 * SError Interrupt, IDS=1 2768 * 24 23 0 2769 * +---+------------------------+ 2770 * |IDS| IMPLEMENTATION DEFINED | 2771 * +---+------------------------+ 2772 * 2773 * where: 2774 * IDS: Implementation-defined syndrome (1) 2775 */ 2776 2777 #define ISS_SEI_IDS_SHIFT 24 2778 #define ISS_SEI_IDS (0x1 << ISS_SEI_IDS_SHIFT) 2779 2780 2781 #if HAS_UCNORMAL_MEM 2782 #define ISS_UC 0x11 2783 #endif /* HAS_UCNORMAL_MEM */ 2784 2785 2786 2787 #if HAS_ARM_FEAT_SME 2788 2789 /* 2790 * SME ISS (EL1) 2791 * 2792 * 24 3 2 0 2793 * +----------------------+----+ 2794 * |0000000000000000000000|SMTC| 2795 * +----------------------+----+ 2796 * 2797 * where: 2798 * SMTC: SME Trap Code 2799 */ 2800 #define ISS_SME_SMTC_CAPCR 0x0 2801 #define ISS_SME_SMTC_MASK 0x7 2802 #define ISS_SME_SMTC(x) ((x) & ISS_SME_SMTC_MASK) 2803 2804 2805 /* 2806 * SME Control Register (EL1) 2807 * 31 30 29 4 3 0 2808 * +----+----+--------------------------+---+ 2809 * |FA64|EZT0|00000000000000000000000000|LEN| 2810 * +----+----+--------------------------+---+ 2811 * 2812 * where: 2813 * FA64: Enable FEAT_SME_FA64 2814 * EZT0: Enable ZT0 2815 * LEN: Effective SVL = (LEN + 1) * 128 2816 */ 2817 2818 #define SMCR_EL1_LEN_MASK 0xf 2819 #if HAS_ARM_FEAT_SME2 2820 #define SMCR_EL1_EZT0 (1ULL << 30) 2821 #endif 2822 #define SMCR_EL1_LEN(x) ((x) & SMCR_EL1_LEN_MASK) 2823 2824 #define SMPRI_EL1_PRIORITY_MASK 0xf 2825 #define SMPRI_EL1_PRIORITY(x) ((x) & SMPRI_EL1_PRIORITY_MASK) 2826 2827 /* 2828 * Streaming Vector Control Register (SVCR) 2829 */ 2830 #define SVCR_ZA_SHIFT (1) 2831 #define SVCR_ZA (1ULL << SVCR_ZA_SHIFT) 2832 #define SVCR_SM_SHIFT (0) 2833 #define SVCR_SM (1ULL << SVCR_SM_SHIFT) 2834 2835 #endif /* HAS_ARM_FEAT_SME */ 2836 2837 /* 2838 * Branch Target Indication Exception ISS 2839 * 24 3 2 0 2840 * +----+-----+ 2841 * |res0|BTYPE| 2842 * +----+-----+ 2843 */ 2844 #define ISS_BTI_BTYPE_SHIFT (0) 2845 #define ISS_BTI_BTYPE_MASK (0x3 << ISS_BTI_BTYPE_SHIFT) 2846 2847 /* 2848 * Physical Address Register (EL1) 2849 */ 2850 #define PAR_F_SHIFT 0 2851 #define PAR_F (0x1 << PAR_F_SHIFT) 2852 2853 #define PLATFORM_SYSCALL_TRAP_NO 0x80000000 2854 2855 #define ARM64_SYSCALL_CODE_REG_NUM (16) 2856 2857 #define ARM64_CLINE_SHIFT 6 2858 2859 #if defined(APPLE_ARM64_ARCH_FAMILY) 2860 #define L2CERRSTS_DATSBEESV (1ULL << 2) /* L2C data single bit ECC error */ 2861 #define L2CERRSTS_DATDBEESV (1ULL << 4) /* L2C data double bit ECC error */ 2862 #endif 2863 2864 /* 2865 * Timer definitions. 2866 */ 2867 #define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */ 2868 #define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */ 2869 #define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */ 2870 #define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */ 2871 #define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */ 2872 #define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */ 2873 #define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to virtual timebase + frequency reg enabled */ 2874 #define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to physical timebase + frequency reg enabled */ 2875 2876 #define CNTV_CTL_EL0_ISTATUS (0x1 << 2) /* (read only): whether interrupt asserted */ 2877 #define CNTV_CTL_EL0_IMASKED (0x1 << 1) /* 1: interrupt masked */ 2878 #define CNTV_CTL_EL0_ENABLE (0x1 << 0) /* 1: virtual timer enabled */ 2879 2880 #define CNTP_CTL_EL0_ISTATUS CNTV_CTL_EL0_ISTATUS 2881 #define CNTP_CTL_EL0_IMASKED CNTV_CTL_EL0_IMASKED 2882 #define CNTP_CTL_EL0_ENABLE CNTV_CTL_EL0_ENABLE 2883 2884 #define MIDR_EL1_REV_SHIFT 0 2885 #define MIDR_EL1_REV_MASK (0xf << MIDR_EL1_REV_SHIFT) 2886 #define MIDR_EL1_PNUM_SHIFT 4 2887 #define MIDR_EL1_PNUM_MASK (0xfff << MIDR_EL1_PNUM_SHIFT) 2888 #define MIDR_EL1_ARCH_SHIFT 16 2889 #define MIDR_EL1_ARCH_MASK (0xf << MIDR_EL1_ARCH_SHIFT) 2890 #define MIDR_EL1_VAR_SHIFT 20 2891 #define MIDR_EL1_VAR_MASK (0xf << MIDR_EL1_VAR_SHIFT) 2892 #define MIDR_EL1_IMP_SHIFT 24 2893 #define MIDR_EL1_IMP_MASK (0xff << MIDR_EL1_IMP_SHIFT) 2894 2895 #define MIDR_FIJI (0x002 << MIDR_EL1_PNUM_SHIFT) 2896 #define MIDR_CAPRI (0x003 << MIDR_EL1_PNUM_SHIFT) 2897 #define MIDR_MAUI (0x004 << MIDR_EL1_PNUM_SHIFT) 2898 #define MIDR_ELBA (0x005 << MIDR_EL1_PNUM_SHIFT) 2899 #define MIDR_CAYMAN (0x006 << MIDR_EL1_PNUM_SHIFT) 2900 #define MIDR_MYST (0x007 << MIDR_EL1_PNUM_SHIFT) 2901 #define MIDR_SKYE_MONSOON (0x008 << MIDR_EL1_PNUM_SHIFT) 2902 #define MIDR_SKYE_MISTRAL (0x009 << MIDR_EL1_PNUM_SHIFT) 2903 #define MIDR_CYPRUS_VORTEX (0x00B << MIDR_EL1_PNUM_SHIFT) 2904 #define MIDR_CYPRUS_TEMPEST (0x00C << MIDR_EL1_PNUM_SHIFT) 2905 #define MIDR_M9 (0x00F << MIDR_EL1_PNUM_SHIFT) 2906 #define MIDR_ARUBA_VORTEX (0x010 << MIDR_EL1_PNUM_SHIFT) 2907 #define MIDR_ARUBA_TEMPEST (0x011 << MIDR_EL1_PNUM_SHIFT) 2908 2909 #ifdef APPLELIGHTNING 2910 #define MIDR_CEBU_LIGHTNING (0x012 << MIDR_EL1_PNUM_SHIFT) 2911 #define MIDR_CEBU_THUNDER (0x013 << MIDR_EL1_PNUM_SHIFT) 2912 #define MIDR_TURKS (0x026 << MIDR_EL1_PNUM_SHIFT) 2913 #endif 2914 2915 #ifdef APPLEFIRESTORM 2916 #define MIDR_SICILY_ICESTORM (0x020 << MIDR_EL1_PNUM_SHIFT) 2917 #define MIDR_SICILY_FIRESTORM (0x021 << MIDR_EL1_PNUM_SHIFT) 2918 #define MIDR_TONGA_ICESTORM (0x022 << MIDR_EL1_PNUM_SHIFT) 2919 #define MIDR_TONGA_FIRESTORM (0x023 << MIDR_EL1_PNUM_SHIFT) 2920 #define MIDR_JADE_CHOP_ICESTORM (0x024 << MIDR_EL1_PNUM_SHIFT) 2921 #define MIDR_JADE_CHOP_FIRESTORM (0x025 << MIDR_EL1_PNUM_SHIFT) 2922 #define MIDR_JADE_DIE_ICESTORM (0x028 << MIDR_EL1_PNUM_SHIFT) 2923 #define MIDR_JADE_DIE_FIRESTORM (0x029 << MIDR_EL1_PNUM_SHIFT) 2924 #endif 2925 2926 #ifdef APPLEAVALANCHE 2927 #define MIDR_ELLIS_BLIZZARD (0x030 << MIDR_EL1_PNUM_SHIFT) 2928 #define MIDR_ELLIS_AVALANCHE (0x031 << MIDR_EL1_PNUM_SHIFT) 2929 #endif 2930 #define MIDR_STATEN_BLIZZARD (0x032 << MIDR_EL1_PNUM_SHIFT) 2931 #define MIDR_STATEN_AVALANCHE (0x033 << MIDR_EL1_PNUM_SHIFT) 2932 #define MIDR_RHODES_CHOP_BLIZZARD (0x034 << MIDR_EL1_PNUM_SHIFT) 2933 #define MIDR_RHODES_CHOP_AVALANCHE (0x035 << MIDR_EL1_PNUM_SHIFT) 2934 #define MIDR_RHODES_DIE_BLIZZARD (0x038 << MIDR_EL1_PNUM_SHIFT) 2935 #define MIDR_RHODES_DIE_AVALANCHE (0x039 << MIDR_EL1_PNUM_SHIFT) 2936 2937 #if defined(APPLEEVEREST) 2938 #define MIDR_CRETE_SAWTOOTH (0x040 << MIDR_EL1_PNUM_SHIFT) 2939 #define MIDR_CRETE_EVEREST (0x041 << MIDR_EL1_PNUM_SHIFT) 2940 #define MIDR_IBIZA_ACCE (0x042 << MIDR_EL1_PNUM_SHIFT) 2941 #define MIDR_IBIZA_ACCP (0x043 << MIDR_EL1_PNUM_SHIFT) 2942 #define MIDR_LOBOS_ACCE (0x044 << MIDR_EL1_PNUM_SHIFT) 2943 #define MIDR_LOBOS_ACCP (0x045 << MIDR_EL1_PNUM_SHIFT) 2944 #define MIDR_CAICOS_ACCE (0x046 << MIDR_EL1_PNUM_SHIFT) 2945 #define MIDR_PALMA_ACCE (0x048 << MIDR_EL1_PNUM_SHIFT) 2946 #define MIDR_PALMA_ACCP (0x049 << MIDR_EL1_PNUM_SHIFT) 2947 #define MIDR_COLL_ACCE (0x050 << MIDR_EL1_PNUM_SHIFT) 2948 #define MIDR_COLL_ACCP (0x051 << MIDR_EL1_PNUM_SHIFT) 2949 #endif /* defined(APPLEEVEREST) */ 2950 2951 /*Donan*/ 2952 #define MIDR_DONAN_ACCE (0x052 << MIDR_EL1_PNUM_SHIFT) 2953 #define MIDR_DONAN_ACCP (0x053 << MIDR_EL1_PNUM_SHIFT) 2954 /*Brava*/ 2955 #define MIDR_BRAVA_ACCE (0x054 << MIDR_EL1_PNUM_SHIFT) 2956 #define MIDR_BRAVA_ACCP (0x055 << MIDR_EL1_PNUM_SHIFT) 2957 2958 #if defined(APPLEACC8) 2959 /*Hidra*/ 2960 #define MIDR_HIDRA_ACCE (0x062 << MIDR_EL1_PNUM_SHIFT) 2961 #define MIDR_HIDRA_ACCP (0x063 << MIDR_EL1_PNUM_SHIFT) 2962 #endif /* defined(APPLEACC8) */ 2963 2964 2965 2966 /* 2967 * Apple-ISA-Extensions ID Register. 2968 */ 2969 #define AIDR_MUL53 (1ULL << 0) 2970 #define AIDR_WKDM (1ULL << 1) 2971 #define AIDR_ARCHRETENTION (1ULL << 2) 2972 2973 2974 #if HAS_MTE 2975 #define AIDR_MTEVER_SHIFT 41 2976 #define AIDR_MTEVER_MASK (0b11ULL << AIDR_MTEVER_SHIFT) 2977 #define AIDR_MTEVER_V1 (0b01ULL << AIDR_MTEVER_SHIFT) 2978 #endif 2979 2980 2981 /* 2982 * CoreSight debug registers 2983 */ 2984 #define CORESIGHT_ED 0 2985 #define CORESIGHT_CTI 1 2986 #define CORESIGHT_PMU 2 2987 #define CORESIGHT_UTT 3 /* Not truly a coresight thing, but at a fixed convenient location right after the coresight region */ 2988 2989 #define CORESIGHT_OFFSET(x) ((x) * 0x10000) 2990 #define CORESIGHT_REGIONS 4 2991 #define CORESIGHT_SIZE 0x1000 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 /* 3004 * ID_AA64ISAR0_EL1 - AArch64 Instruction Set Attribute Register 0 3005 * 3006 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 3007 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ 3008 * | rndr | tlb | ts | fhm | dp | sm4 | sm3 | sha3 | rdm | res0 | atomic | crc32 | sha2 | sha1 | aes | res0 | 3009 * +--------+-------+------+-------+------+-------+-------+------+-------+--------+--------+-------+------+------+-----+------+ 3010 */ 3011 3012 #define ID_AA64ISAR0_EL1_TS_OFFSET 52 3013 #define ID_AA64ISAR0_EL1_TS_MASK (0xfull << ID_AA64ISAR0_EL1_TS_OFFSET) 3014 #define ID_AA64ISAR0_EL1_TS_FLAGM_EN (1ull << ID_AA64ISAR0_EL1_TS_OFFSET) 3015 #define ID_AA64ISAR0_EL1_TS_FLAGM2_EN (2ull << ID_AA64ISAR0_EL1_TS_OFFSET) 3016 3017 #define ID_AA64ISAR0_EL1_FHM_OFFSET 48 3018 #define ID_AA64ISAR0_EL1_FHM_MASK (0xfull << ID_AA64ISAR0_EL1_FHM_OFFSET) 3019 #define ID_AA64ISAR0_EL1_FHM_8_2 (1ull << ID_AA64ISAR0_EL1_FHM_OFFSET) 3020 3021 #define ID_AA64ISAR0_EL1_DP_OFFSET 44 3022 #define ID_AA64ISAR0_EL1_DP_MASK (0xfull << ID_AA64ISAR0_EL1_DP_OFFSET) 3023 #define ID_AA64ISAR0_EL1_DP_EN (1ull << ID_AA64ISAR0_EL1_DP_OFFSET) 3024 3025 #define ID_AA64ISAR0_EL1_SHA3_OFFSET 32 3026 #define ID_AA64ISAR0_EL1_SHA3_MASK (0xfull << ID_AA64ISAR0_EL1_SHA3_OFFSET) 3027 #define ID_AA64ISAR0_EL1_SHA3_EN (1ull << ID_AA64ISAR0_EL1_SHA3_OFFSET) 3028 3029 #define ID_AA64ISAR0_EL1_RDM_OFFSET 28 3030 #define ID_AA64ISAR0_EL1_RDM_MASK (0xfull << ID_AA64ISAR0_EL1_RDM_OFFSET) 3031 #define ID_AA64ISAR0_EL1_RDM_EN (1ull << ID_AA64ISAR0_EL1_RDM_OFFSET) 3032 3033 #define ID_AA64ISAR0_EL1_ATOMIC_OFFSET 20 3034 #define ID_AA64ISAR0_EL1_ATOMIC_MASK (0xfull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) 3035 #define ID_AA64ISAR0_EL1_ATOMIC_8_1 (2ull << ID_AA64ISAR0_EL1_ATOMIC_OFFSET) 3036 3037 #define ID_AA64ISAR0_EL1_CRC32_OFFSET 16 3038 #define ID_AA64ISAR0_EL1_CRC32_MASK (0xfull << ID_AA64ISAR0_EL1_CRC32_OFFSET) 3039 #define ID_AA64ISAR0_EL1_CRC32_EN (1ull << ID_AA64ISAR0_EL1_CRC32_OFFSET) 3040 3041 #define ID_AA64ISAR0_EL1_SHA2_OFFSET 12 3042 #define ID_AA64ISAR0_EL1_SHA2_MASK (0xfull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 3043 #define ID_AA64ISAR0_EL1_SHA2_EN (1ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 3044 #define ID_AA64ISAR0_EL1_SHA2_512_EN (2ull << ID_AA64ISAR0_EL1_SHA2_OFFSET) 3045 3046 #define ID_AA64ISAR0_EL1_SHA1_OFFSET 8 3047 #define ID_AA64ISAR0_EL1_SHA1_MASK (0xfull << ID_AA64ISAR0_EL1_SHA1_OFFSET) 3048 #define ID_AA64ISAR0_EL1_SHA1_EN (1ull << ID_AA64ISAR0_EL1_SHA1_OFFSET) 3049 3050 #define ID_AA64ISAR0_EL1_AES_OFFSET 4 3051 #define ID_AA64ISAR0_EL1_AES_MASK (0xfull << ID_AA64ISAR0_EL1_AES_OFFSET) 3052 #define ID_AA64ISAR0_EL1_AES_EN (1ull << ID_AA64ISAR0_EL1_AES_OFFSET) 3053 #define ID_AA64ISAR0_EL1_AES_PMULL_EN (2ull << ID_AA64ISAR0_EL1_AES_OFFSET) 3054 3055 /* 3056 * ID_AA64ISAR1_EL1 - AArch64 Instruction Set Attribute Register 1 3057 * 3058 * 63 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 3059 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ 3060 * | res0 | i8mm | dgh | bf16 | specres | sb | frintts | gpi | gpa | lrcpc | fcma | jscvt | api | apa | dpb | 3061 * +------+------+-----+------+---------+------+---------+-----+-----+-------+------+-------+-----+-----+-----+ 3062 */ 3063 3064 #define ID_AA64ISAR1_EL1_I8MM_OFFSET 52 3065 #define ID_AA64ISAR1_EL1_I8MM_MASK (0xfull << ID_AA64ISAR1_EL1_I8MM_OFFSET) 3066 #define ID_AA64ISAR1_EL1_I8MM_EN (1ull << ID_AA64ISAR1_EL1_I8MM_OFFSET) 3067 3068 #define ID_AA64ISAR1_EL1_DGH_OFFSET 48 3069 #define ID_AA64ISAR1_EL1_DGH_MASK (0xfull << ID_AA64ISAR1_EL1_DGH_OFFSET) 3070 3071 #define ID_AA64ISAR1_EL1_BF16_OFFSET 44 3072 #define ID_AA64ISAR1_EL1_BF16_MASK (0xfull << ID_AA64ISAR1_EL1_BF16_OFFSET) 3073 #define ID_AA64ISAR1_EL1_BF16_EN (1ull << ID_AA64ISAR1_EL1_BF16_OFFSET) 3074 #define ID_AA64ISAR1_EL1_EBF16_EN (2ull << ID_AA64ISAR1_EL1_BF16_OFFSET) 3075 3076 #define ID_AA64ISAR1_EL1_SPECRES_OFFSET 40 3077 #define ID_AA64ISAR1_EL1_SPECRES_MASK (0xfull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 3078 #define ID_AA64ISAR1_EL1_SPECRES_EN (1ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 3079 #define ID_AA64ISAR1_EL1_SPECRES2_EN (2ull << ID_AA64ISAR1_EL1_SPECRES_OFFSET) 3080 3081 #define ID_AA64ISAR1_EL1_SB_OFFSET 36 3082 #define ID_AA64ISAR1_EL1_SB_MASK (0xfull << ID_AA64ISAR1_EL1_SB_OFFSET) 3083 #define ID_AA64ISAR1_EL1_SB_EN (1ull << ID_AA64ISAR1_EL1_SB_OFFSET) 3084 3085 #define ID_AA64ISAR1_EL1_FRINTTS_OFFSET 32 3086 #define ID_AA64ISAR1_EL1_FRINTTS_MASK (0xfull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) 3087 #define ID_AA64ISAR1_EL1_FRINTTS_EN (1ull << ID_AA64ISAR1_EL1_FRINTTS_OFFSET) 3088 3089 #define ID_AA64ISAR1_EL1_GPI_OFFSET 28 3090 #define ID_AA64ISAR1_EL1_GPI_MASK (0xfull << ID_AA64ISAR1_EL1_GPI_OFFSET) 3091 #define ID_AA64ISAR1_EL1_GPI_EN (1ull << ID_AA64ISAR1_EL1_GPI_OFFSET) 3092 3093 #define ID_AA64ISAR1_EL1_GPA_OFFSET 24 3094 #define ID_AA64ISAR1_EL1_GPA_MASK (0xfull << ID_AA64ISAR1_EL1_GPA_OFFSET) 3095 3096 #define ID_AA64ISAR1_EL1_LRCPC_OFFSET 20 3097 #define ID_AA64ISAR1_EL1_LRCPC_MASK (0xfull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 3098 #define ID_AA64ISAR1_EL1_LRCPC_EN (1ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 3099 #define ID_AA64ISAR1_EL1_LRCP2C_EN (2ull << ID_AA64ISAR1_EL1_LRCPC_OFFSET) 3100 3101 #define ID_AA64ISAR1_EL1_FCMA_OFFSET 16 3102 #define ID_AA64ISAR1_EL1_FCMA_MASK (0xfull << ID_AA64ISAR1_EL1_FCMA_OFFSET) 3103 #define ID_AA64ISAR1_EL1_FCMA_EN (1ull << ID_AA64ISAR1_EL1_FCMA_OFFSET) 3104 3105 #define ID_AA64ISAR1_EL1_JSCVT_OFFSET 12 3106 #define ID_AA64ISAR1_EL1_JSCVT_MASK (0xfull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) 3107 #define ID_AA64ISAR1_EL1_JSCVT_EN (1ull << ID_AA64ISAR1_EL1_JSCVT_OFFSET) 3108 3109 #define ID_AA64ISAR1_EL1_API_OFFSET 8 3110 #define ID_AA64ISAR1_EL1_API_MASK (0xfull << ID_AA64ISAR1_EL1_API_OFFSET) 3111 #define ID_AA64ISAR1_EL1_API_PAuth_EN (1ull << ID_AA64ISAR1_EL1_API_OFFSET) 3112 #define ID_AA64ISAR1_EL1_API_PAuth2_EN (3ull << ID_AA64ISAR1_EL1_API_OFFSET) 3113 #define ID_AA64ISAR1_EL1_API_FPAC_EN (4ull << ID_AA64ISAR1_EL1_API_OFFSET) 3114 #define ID_AA64ISAR1_EL1_API_FPACCOMBINE (5ull << ID_AA64ISAR1_EL1_API_OFFSET) 3115 3116 #define ID_AA64ISAR1_EL1_APA_OFFSET 4 3117 #define ID_AA64ISAR1_EL1_APA_MASK (0xfull << ID_AA64ISAR1_EL1_APA_OFFSET) 3118 3119 #define ID_AA64ISAR1_EL1_DPB_OFFSET 0 3120 #define ID_AA64ISAR1_EL1_DPB_MASK (0xfull << ID_AA64ISAR1_EL1_DPB_OFFSET) 3121 #define ID_AA64ISAR1_EL1_DPB_EN (1ull << ID_AA64ISAR1_EL1_DPB_OFFSET) 3122 #define ID_AA64ISAR1_EL1_DPB2_EN (2ull << ID_AA64ISAR1_EL1_DPB_OFFSET) 3123 3124 /* 3125 * ID_AA64ISAR2_EL1 - AArch64 Instruction Set Attribute Register 2 3126 * 3127 * 63 56 55 52 51 24 23 20 19 8 7 4 3 0 3128 * +------+------+------+------+-------+-------+------+ 3129 * | res2 | CSSC | res1 | BC | res0 | RPRES | WFxT | 3130 * +------+------+------+------+-------+-------+------+ 3131 */ 3132 3133 3134 #define ID_AA64ISAR2_EL1_CSSC_OFFSET 52 3135 #define ID_AA64ISAR2_EL1_CSSC_MASK (0xfull << ID_AA64ISAR2_EL1_CSSC_OFFSET) 3136 #define ID_AA64ISAR2_EL1_CSSC_EN (1ull << ID_AA64ISAR2_EL1_CSSC_OFFSET) 3137 3138 #define ID_AA64ISAR2_EL1_BC_OFFSET 20 3139 #define ID_AA64ISAR2_EL1_BC_MASK (0xfull << ID_AA64ISAR2_EL1_BC_OFFSET) 3140 #define ID_AA64ISAR2_EL1_BC_EN (1ull << ID_AA64ISAR2_EL1_BC_OFFSET) 3141 3142 #define ID_AA64ISAR2_EL1_RPRES_OFFSET 4 3143 #define ID_AA64ISAR2_EL1_RPRES_MASK (0xfull << ID_AA64ISAR2_EL1_RPRES_OFFSET) 3144 #define ID_AA64ISAR2_EL1_RPRES_EN (1ull << ID_AA64ISAR2_EL1_RPRES_OFFSET) 3145 3146 #define ID_AA64ISAR2_EL1_WFxT_OFFSET 0 3147 #define ID_AA64ISAR2_EL1_WFxT_MASK (0xfull << ID_AA64ISAR2_EL1_WFxT_OFFSET) 3148 #define ID_AA64ISAR2_EL1_WFxT_EN (1ull << ID_AA64ISAR2_EL1_WFxT_OFFSET) 3149 3150 3151 /* 3152 * ID_AA64MMFR0_EL1 - AArch64 Memory Model Feature Register 0 3153 * 63 60 59 56 55 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 3154 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ 3155 * | ECV | FGT | RES0 | ExS | TGran4_2 | TGran64_2 | TGran16_2 | TGran4 | TGran64 | TGran16 | BigEndEL0 | SNSMem | BigEnd | ASIDBits | PARange | 3156 * +-------+-------+------------+-------+----------+-----------+-----------+--------+---------+---------+-----------+--------+--------+----------+---------+ 3157 */ 3158 3159 #define ID_AA64MMFR0_EL1_ECV_OFFSET 60 3160 #define ID_AA64MMFR0_EL1_ECV_MASK (0xfull << ID_AA64MMFR0_EL1_ECV_OFFSET) 3161 #define ID_AA64MMFR0_EL1_ECV_EN (1ull << ID_AA64MMFR0_EL1_ECV_OFFSET) 3162 3163 /* 3164 * ID_AA64MMFR2_EL1 - AArch64 Memory Model Feature Register 2 3165 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 14 8 7 4 3 0 3166 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ 3167 * | E0PD | EVT | BBM | TTL | RES0 | FWB | IDS | AT | ST | NV | CCIDX | VARANGE | IESB | LSM | UAO | CnP | 3168 * +------+-------+-------+-------+--------+-------+-------+------+------+------+-------+---------+------+-------+-------+-------+ 3169 */ 3170 3171 #define ID_AA64MMFR2_EL1_AT_OFFSET 32 3172 #define ID_AA64MMFR2_EL1_AT_MASK (0xfull << ID_AA64MMFR2_EL1_AT_OFFSET) 3173 #define ID_AA64MMFR2_EL1_AT_LSE2_EN (1ull << ID_AA64MMFR2_EL1_AT_OFFSET) 3174 #define ID_AA64MMFR2_EL1_VARANGE_OFFSET 16 3175 #define ID_AA64MMFR2_EL1_VARANGE_MASK (0xfull << ID_AA64MMFR2_EL1_VARANGE_OFFSET) 3176 3177 #define ID_AA64MMFR2_EL1_CNP_OFFSET 0 3178 #define ID_AA64MMFR2_EL1_CNP_MASK (0xfull << ID_AA64MMFR2_EL1_CNP_OFFSET) 3179 #define ID_AA64MMFR2_EL1_CNP_EN (1ull << ID_AA64MMFR2_EL1_CNP_OFFSET) 3180 3181 /* 3182 * ID_AA64PFR0_EL1 - AArch64 Processor Feature Register 0 3183 * 63 60 59 56 55 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 3184 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ 3185 * | CSV3 | CSV2 | RES0 | DIT | AMU | MPAM | SEL2 | SVE | RAS | GIC | AdvSIMD | FP | EL3 | EL2 | EL1 | EL0 | 3186 * +--------+--------+--------+-------+-------+--------+--------+-------+-------+-----+---------+------+-----+-----+-----+-----+ 3187 */ 3188 3189 #define ID_AA64PFR0_EL1_CSV3_OFFSET 60 3190 #define ID_AA64PFR0_EL1_CSV3_MASK (0xfull << ID_AA64PFR0_EL1_CSV3_OFFSET) 3191 #define ID_AA64PFR0_EL1_CSV3_EN (1ull << ID_AA64PFR0_EL1_CSV3_OFFSET) 3192 3193 #define ID_AA64PFR0_EL1_CSV2_OFFSET 56 3194 #define ID_AA64PFR0_EL1_CSV2_MASK (0xfull << ID_AA64PFR0_EL1_CSV2_OFFSET) 3195 #define ID_AA64PFR0_EL1_CSV2_EN (1ull << ID_AA64PFR0_EL1_CSV2_OFFSET) 3196 #define ID_AA64PFR0_EL1_CSV2_2 (2ull << ID_AA64PFR0_EL1_CSV2_OFFSET) 3197 3198 #define ID_AA64PFR0_EL1_DIT_OFFSET 48 3199 #define ID_AA64PFR0_EL1_DIT_MASK (0xfull << ID_AA64PFR0_EL1_DIT_OFFSET) 3200 #define ID_AA64PFR0_EL1_DIT_EN (1ull << ID_AA64PFR0_EL1_DIT_OFFSET) 3201 3202 #define ID_AA64PFR0_EL1_AdvSIMD_OFFSET 20 3203 #define ID_AA64PFR0_EL1_AdvSIMD_MASK (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 3204 #define ID_AA64PFR0_EL1_AdvSIMD_HPFPCVT (0x0ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 3205 #define ID_AA64PFR0_EL1_AdvSIMD_FP16 (0x1ull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 3206 #define ID_AA64PFR0_EL1_AdvSIMD_DIS (0xfull << ID_AA64PFR0_EL1_AdvSIMD_OFFSET) 3207 3208 /* 3209 * ID_AA64PFR1_EL1 - AArch64 Processor Feature Register 1 3210 * 63 20 19 16 15 12 11 8 7 4 3 0 3211 * +----------------------------------+-----------+----------+-------+------+------+ 3212 * | RES0 | MPAM_frac | RAS_frac | MTE | SSBS | BT | 3213 * +----------------------------------+-----------+----------+-------+------+------+ 3214 */ 3215 3216 3217 #define ID_AA64PFR1_EL1_MTEX_OFFSET 52 3218 #define ID_AA64PFR1_EL1_MTEX_MASK (0xfull << ID_AA64PFR1_EL1_MTEX_OFFSET) 3219 #define ID_AA64PFR1_EL1_MTEX_EN (1ull << ID_AA64PFR1_EL1_MTEX_OFFSET) 3220 3221 #define ID_AA64PFR1_EL1_MTE_FRAC_OFFSET 40 3222 #define ID_AA64PFR1_EL1_MTE_FRAC_MASK (0xfull << ID_AA64PFR1_EL1_MTE_FRAC_OFFSET) 3223 #define ID_AA64PFR1_EL1_MTE_FRAC_EN (1ull << ID_AA64PFR1_EL1_MTE_FRAC_OFFSET) 3224 3225 #define ID_AA64PFR1_EL1_SME_OFFSET 24 3226 #define ID_AA64PFR1_EL1_SME_MASK (0xfull << ID_AA64PFR1_EL1_SME_OFFSET) 3227 #define ID_AA64PFR1_EL1_SME_EN (1ull << ID_AA64PFR1_EL1_SME_OFFSET) 3228 #define ID_AA64PFR1_EL1_CSV2_frac_OFFSET 32 3229 #define ID_AA64PFR1_EL1_CSV2_frac_MASK (0xfull << ID_AA64PFR1_EL1_CSV2_frac_OFFSET) 3230 #define ID_AA64PFR1_EL1_CSV2_frac_1p1 (1ull << ID_AA64PFR1_EL1_CSV2_frac_OFFSET) 3231 #define ID_AA64PFR1_EL1_CSV2_frac_1p2 (2ull << ID_AA64PFR1_EL1_CSV2_frac_OFFSET) 3232 3233 #define ID_AA64PFR1_EL1_MTE_OFFSET 8 3234 #define ID_AA64PFR1_EL1_MTE_MASK (0xfull << ID_AA64PFR1_EL1_MTE_OFFSET) 3235 3236 #define ID_AA64PFR1_EL1_SSBS_OFFSET 4 3237 #define ID_AA64PFR1_EL1_SSBS_MASK (0xfull << ID_AA64PFR1_EL1_SSBS_OFFSET) 3238 #define ID_AA64PFR1_EL1_SSBS_EN (1ull << ID_AA64PFR1_EL1_SSBS_OFFSET) 3239 3240 #define ID_AA64PFR1_EL1_BT_OFFSET 0 3241 #define ID_AA64PFR1_EL1_BT_MASK (0xfull << ID_AA64PFR1_EL1_BT_OFFSET) 3242 #define ID_AA64PFR1_EL1_BT_EN (1ull << ID_AA64PFR1_EL1_BT_OFFSET) 3243 3244 /* 3245 * ID_AA64PFR2_EL1 - AArch64 Processor Feature Register 2 3246 */ 3247 3248 3249 #define ID_AA64PFR2_EL1_MTE_FAR_OFFSET 8 3250 #define ID_AA64PFR2_EL1_MTE_FAR_MASK (0xfull << ID_AA64PFR2_EL1_MTE_FAR_OFFSET) 3251 #define ID_AA64PFR2_EL1_MTE_FAR_EN (1ull << ID_AA64PFR2_EL1_MTE_FAR_OFFSET) 3252 3253 #define ID_AA64PFR2_EL1_MTE_STORE_ONLY_OFFSET 4 3254 #define ID_AA64PFR2_EL1_MTE_STORE_ONLY_MASK (0xfull << ID_AA64PFR2_EL1_MTE_STORE_ONLY_OFFSET) 3255 #define ID_AA64PFR2_EL1_MTE_STORE_ONLY_EN (1ull << ID_AA64PFR2_EL1_MTE_STORE_ONLY_OFFSET) 3256 3257 #define ID_AA64PFR2_EL1_MTE_PERM_OFFSET 0 3258 #define ID_AA64PFR2_EL1_MTE_PERM_MASK (0xfull << ID_AA64PFR2_EL1_MTE_PERM_OFFSET) 3259 #define ID_AA64PFR2_EL1_MTE_PERM_EN (1ull << ID_AA64PFR2_EL1_MTE_PERM_OFFSET) 3260 3261 /* 3262 * ID_AA64MMFR1_EL1 - AArch64 Memory Model Feature Register 1 3263 * 3264 * 63 52 51 48 47 44 43 40 39 36 35 32 31 28 27 24 23 20 19 16 15 12 11 8 7 4 3 0 3265 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ 3266 * | res0 | nTLBPA | AFP | HCX | ETS | TWED | XNX | SpecSEI | PAN | LO | HPDS | VH | VMIDBits | HAFDBS | 3267 * +------+--------+-----+-----+-----+-----+------+---------+-------+------+------+------+----------+--------+ 3268 */ 3269 3270 #define ID_AA64MMFR1_EL1_AFP_OFFSET 44 3271 #define ID_AA64MMFR1_EL1_AFP_MASK (0xfull << ID_AA64MMFR1_EL1_AFP_OFFSET) 3272 #define ID_AA64MMFR1_EL1_AFP_EN (1ull << ID_AA64MMFR1_EL1_AFP_OFFSET) 3273 3274 #define ID_AA64MMFR1_EL1_HCX_OFFSET 40 3275 #define ID_AA64MMFR1_EL1_HCX_MASK (0xfull << ID_AA64MMFR1_EL1_HCX_OFFSET) 3276 #define ID_AA64MMFR1_EL1_HCX_EN (1ull << ID_AA64MMFR1_EL1_HCX_OFFSET) 3277 3278 /* 3279 * ID_AA64SMFR0_EL1 - SME Feature ID Register 0 3280 * 3281 * 63 62 60 59 56 55 52 51 49 48 47 44 43 40 39 36 35 34 33 32 31 0 3282 * +------+------+--------+--------+------+--------+--------+------+-------+--------+--------+---------+--------+------+ 3283 * | FA64 | res0 | SMEver | I16I64 | res0 | F64F64 | I16I32 | res0 | I8I32 | F16F32 | B16F32 | BI32I32 | F32F32 | res0 | 3284 * +------+------+--------+--------+------+--------+--------+------+-------+--------+--------+---------+--------+------+ 3285 */ 3286 3287 3288 #define ID_AA64SMFR0_EL1_SMEver_OFFSET 56 3289 #define ID_AA64SMFR0_EL1_SMEver_MASK (0xfull << ID_AA64SMFR0_EL1_SMEver_OFFSET) 3290 #define ID_AA64SMFR0_EL1_SMEver_SME (0ull << ID_AA64SMFR0_EL1_SMEver_OFFSET) 3291 #define ID_AA64SMFR0_EL1_SMEver_SME2 (1ull << ID_AA64SMFR0_EL1_SMEver_OFFSET) 3292 #define ID_AA64SMFR0_EL1_SMEver_SME2p1 (2ull << ID_AA64SMFR0_EL1_SMEver_OFFSET) 3293 3294 #define ID_AA64SMFR0_EL1_I16I64_OFFSET 52 3295 #define ID_AA64SMFR0_EL1_I16I64_MASK (0xfull << ID_AA64SMFR0_EL1_I16I64_OFFSET) 3296 #define ID_AA64SMFR0_EL1_I16I64_EN (0xfull << ID_AA64SMFR0_EL1_I16I64_OFFSET) 3297 3298 #define ID_AA64SMFR0_EL1_F64F64_OFFSET 48 3299 #define ID_AA64SMFR0_EL1_F64F64_MASK (1ull << ID_AA64SMFR0_EL1_F64F64_OFFSET) 3300 #define ID_AA64SMFR0_EL1_F64F64_EN (1ull << ID_AA64SMFR0_EL1_F64F64_OFFSET) 3301 3302 #define ID_AA64SMFR0_EL1_I16I32_OFFSET 44 3303 #define ID_AA64SMFR0_EL1_I16I32_MASK (0xfull << ID_AA64SMFR0_EL1_I16I32_OFFSET) 3304 #define ID_AA64SMFR0_EL1_I16I32_EN (0x5ull << ID_AA64SMFR0_EL1_I16I32_OFFSET) 3305 3306 #define ID_AA64SMFR0_EL1_B16B16_OFFSET 43 3307 #define ID_AA64SMFR0_EL1_B16B16_MASK (1ull << ID_AA64SMFR0_EL1_B16B16_OFFSET) 3308 #define ID_AA64SMFR0_EL1_B16B16_EN (1ull << ID_AA64SMFR0_EL1_B16B16_OFFSET) 3309 3310 #define ID_AA64SMFR0_EL1_F16F16_OFFSET 42 3311 #define ID_AA64SMFR0_EL1_F16F16_MASK (1ull << ID_AA64SMFR0_EL1_F16F16_OFFSET) 3312 #define ID_AA64SMFR0_EL1_F16F16_EN (1ull << ID_AA64SMFR0_EL1_F16F16_OFFSET) 3313 3314 3315 #define ID_AA64SMFR0_EL1_I8I32_OFFSET 36 3316 #define ID_AA64SMFR0_EL1_I8I32_MASK (0xfull << ID_AA64SMFR0_EL1_I8I32_OFFSET) 3317 #define ID_AA64SMFR0_EL1_I8I32_EN (0xfull << ID_AA64SMFR0_EL1_I8I32_OFFSET) 3318 3319 #define ID_AA64SMFR0_EL1_F16F32_OFFSET 35 3320 #define ID_AA64SMFR0_EL1_F16F32_MASK (1ull << ID_AA64SMFR0_EL1_F16F32_OFFSET) 3321 #define ID_AA64SMFR0_EL1_F16F32_EN (1ull << ID_AA64SMFR0_EL1_F16F32_OFFSET) 3322 3323 #define ID_AA64SMFR0_EL1_B16F32_OFFSET 34 3324 #define ID_AA64SMFR0_EL1_B16F32_MASK (1ull << ID_AA64SMFR0_EL1_B16F32_OFFSET) 3325 #define ID_AA64SMFR0_EL1_B16F32_EN (1ull << ID_AA64SMFR0_EL1_B16F32_OFFSET) 3326 3327 #define ID_AA64SMFR0_EL1_BI32I32_OFFSET 33 3328 #define ID_AA64SMFR0_EL1_BI32I32_MASK (1ull << ID_AA64SMFR0_EL1_BI32I32_OFFSET) 3329 #define ID_AA64SMFR0_EL1_BI32I32_EN (1ull << ID_AA64SMFR0_EL1_BI32I32_OFFSET) 3330 3331 #define ID_AA64SMFR0_EL1_F32F32_OFFSET 32 3332 #define ID_AA64SMFR0_EL1_F32F32_MASK (1ull << ID_AA64SMFR0_EL1_F32F32_OFFSET) 3333 #define ID_AA64SMFR0_EL1_F32F32_EN (1ull << ID_AA64SMFR0_EL1_F32F32_OFFSET) 3334 3335 3336 #if HAS_MTE 3337 3338 /* 3339 * MTE tag metadata is kept in DRAM. The metadata is 4-bit per 16-bytes of memory. 3340 * This translates to (potentially) using 3% of DRAM to represent the whole 3341 * memory space and also to one single TAG page to contain metadata covering 3342 * 32 regular pages. 3343 */ 3344 #define MTE_PAGES_PER_TAG_PAGE 32 3345 3346 /* 3347 * TagOffset_EL2 is the register that contains the start of the stolen DRAM 3348 * memory that contains MTE tags metadata. This definition is currently here, 3349 * but should probably be moved to a more suitable place as part of the 3350 * final support for H17g. 3351 * 3352 * The MTE tag array is physically contiguous and aligned to 1MB. 3353 */ 3354 #define MTE_TAG_OFFSET_EL2 "S3_0_C11_C9_0" 3355 #define MTE_TAG_OFFSET_EL2_PA_SHIFT (20) 3356 #define MTE_TAG_OFFSET_PA_ALIGN (1ULL << MTE_TAG_OFFSET_EL2_PA_SHIFT) 3357 #define MTE_TAG_OFFSET_EL2_PA_MASK (0x3FFFFFULL << MTE_TAG_OFFSET_EL2_PA_SHIFT) 3358 #define MTE_TAG_OFFSET_EL2_LOCK_BIT (63) 3359 3360 /* 3361 * GCR_EL1 - Tag Control Register 3362 * 3363 * 63 17 16 15 0 3364 * +------+------+---------+ 3365 * | res0 | RRND | Exclude | 3366 * +------+------+---------+ 3367 */ 3368 3369 #define GCR_EL1_RRND_OFFSET 16 3370 #define GCR_EL1_RRND (1ULL << GCR_EL1_RRND_OFFSET) 3371 #define GCR_EL1_RRND_ASM 0x10000 3372 3373 #define GCR_EL1_EXCLUDE_OFFSET 0 3374 #define GCR_EL1_EXCLUDE_MASK (0xFFFFULL << GCR_EL1_EXCLUDE_OFFSET) 3375 3376 /* Default exclude masks to skip canonical tags in user and kernel. */ 3377 3378 #define GCR_EL1_EXCLUDE_TAGS_KERNEL (0x8000) 3379 #define GCR_EL1_EXCLUDE_TAGS_USER (0x0001) 3380 3381 /* 3382 * RGSR_EL1 - Random Allocation Tag Seed Register 3383 * 3384 * 63 24 23 8 7 4 3 0 3385 * +------+------+------+-----+ 3386 * | res0 | SEED | res0 | TAG | (GCR_EL1.RRND == 0) 3387 * +------+------+------+-----+ 3388 * 3389 * 63 56 55 8 7 4 3 0 3390 * +------+------+------+-----+ 3391 * | res0 | SEED | res0 | TAG | (GCR_EL1.RRND == 1) 3392 * +------+------+------+-----+ 3393 */ 3394 3395 #define RGSR_EL1_SEED_OFFSET 8 3396 #define RGSR_EL1_SEED_RRND_0_MASK (0xFFFFULL << RGSR_EL1_SEED_OFFSET) 3397 #define RGSR_EL1_SEED_RRND_1_MASK (0xFFFFFFFFFFFFULL << RGSR_EL1_SEED_OFFSET) 3398 3399 #define RGSR_EL1_TAG_OFFSET 0 3400 #define RGSR_EL1_TAG (0xFULL << RGSR_EL1_TAG_OFFSET) 3401 3402 #endif /* HAS_MTE */ 3403 3404 3405 #define APSTATE_G_SHIFT (0) 3406 #define APSTATE_P_SHIFT (1) 3407 #define APSTATE_A_SHIFT (2) 3408 #define APSTATE_AP_MASK ((1ULL << APSTATE_A_SHIFT) | (1ULL << APSTATE_P_SHIFT)) 3409 3410 3411 #define ACTLR_EL1_EnTSO (1ULL << 1) 3412 #define ACTLR_EL1_EnAPFLG (1ULL << 4) 3413 #define ACTLR_EL1_EnAFP (1ULL << 5) 3414 #define ACTLR_EL1_EnPRSV (1ULL << 6) 3415 3416 3417 #if HAS_USAT_BIT 3418 #define ACTLR_EL1_USAT_OFFSET 0 3419 #define ACTLR_EL1_USAT_MASK (1ULL << ACTLR_EL1_USAT_OFFSET) 3420 #define ACTLR_EL1_USAT ACTLR_EL1_USAT_MASK 3421 #endif 3422 3423 3424 3425 3426 3427 3428 #ifdef HAS_DISDDHWP0 3429 #define ACTLR_EL1_DisDDHWP0_OFFSET 17 3430 #define ACTLR_EL1_DisDDHWP0_MASK (1ULL << ACTLR_EL1_DisDDHWP0_OFFSET) 3431 #define ACTLR_EL1_DisDDHWP0 ACTLR_EL1_DisDDHWP0_MASK 3432 #endif /* HAS_DISDDDHWP0 */ 3433 3434 3435 #if defined(HAS_APPLE_PAC) 3436 // The value of ptrauth_string_discriminator("recover"), hardcoded so it can be used from assembly code 3437 #define PAC_DISCRIMINATOR_RECOVER 0x1e02 3438 #endif 3439 3440 3441 #define CTR_EL0_L1Ip_OFFSET 14 3442 #define CTR_EL0_L1Ip_VIPT (2ULL << CTR_EL0_L1Ip_OFFSET) 3443 #define CTR_EL0_L1Ip_PIPT (3ULL << CTR_EL0_L1Ip_OFFSET) 3444 #define CTR_EL0_L1Ip_MASK (3ULL << CTR_EL0_L1Ip_OFFSET) 3445 3446 3447 #define ACNTHV_CTL_EL2 S3_1_C15_C7_4 3448 #define ACNTHV_CTL_EL2_EN_OFFSET 0 3449 #define ACNTHV_CTL_EL2_EN_MASK (1ULL << ACNTHV_CTL_EL2_EN_OFFSET) 3450 3451 #ifdef __ASSEMBLER__ 3452 3453 /* 3454 * Conditionally write to system/special-purpose register. 3455 * The register is written to only when the first two arguments 3456 * do not match. If they do match, the macro jumps to a 3457 * caller-provided label. 3458 * The _ISB variant also conditionally issues an ISB after the MSR. 3459 * 3460 * $0 - System/special-purpose register to modify 3461 * $1 - Register containing current FPCR value 3462 * $2 - Register containing expected value 3463 * $3 - Label to jump to when register is already set to expected value 3464 */ 3465 .macro CMSR 3466 cmp $1, $2 3467 3468 /* Skip expensive MSR if not required */ 3469 b.eq $3f 3470 msr $0, $2 3471 .endmacro 3472 3473 .macro CMSR_ISB 3474 CMSR $0, $1, $2, $3 3475 isb sy 3476 .endmacro 3477 3478 /* 3479 * Modify FPCR only if it does not contain the XNU default value. 3480 * $0 - Register containing current FPCR value 3481 * $1 - Scratch register 3482 * $2 - Label to jump to when FPCR is already set to default value 3483 */ 3484 .macro SANITIZE_FPCR 3485 mov $1, #FPCR_DEFAULT 3486 CMSR FPCR, $0, $1, $2 3487 .endmacro 3488 3489 /* 3490 * Family of macros that can be used to protect code sections such that they 3491 * are only executed on a particular SoC/Revision/CPU, and skipped otherwise. 3492 * All macros will forward-jump to 1f when the condition is not matched. 3493 * This label may be defined manually, or implicitly through the use of 3494 * the EXEC_END macro. 3495 * For cores, XX can be: EQ (equal), ALL (don't care). 3496 * For revisions, XX can be: EQ (equal), LO (lower than), HS (higher or same), ALL (don't care). 3497 */ 3498 3499 /* 3500 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX 3501 * $1 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 3502 * $2 - GPR containing MIDR_EL1 value 3503 * $3 - Scratch register 3504 */ 3505 .macro EXEC_COREEQ_REVEQ 3506 and $3, $2, #MIDR_EL1_PNUM_MASK 3507 cmp $3, $0 3508 b.ne 1f 3509 3510 mov $3, $2 3511 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 3512 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 3513 cmp $3, $1 3514 b.ne 1f 3515 .endmacro 3516 3517 .macro EXEC_COREEQ_REVLO 3518 and $3, $2, #MIDR_EL1_PNUM_MASK 3519 cmp $3, $0 3520 b.ne 1f 3521 3522 mov $3, $2 3523 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 3524 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 3525 cmp $3, $1 3526 b.pl 1f 3527 .endmacro 3528 3529 .macro EXEC_COREEQ_REVHS 3530 and $3, $2, #MIDR_EL1_PNUM_MASK 3531 cmp $3, $0 3532 b.ne 1f 3533 3534 mov $3, $2 3535 bfi $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #4 3536 ubfx $3, $3, #(MIDR_EL1_VAR_SHIFT - 4), #8 3537 cmp $3, $1 3538 b.mi 1f 3539 .endmacro 3540 3541 /* 3542 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 3543 * $1 - GPR containing MIDR_EL1 value 3544 * $2 - Scratch register 3545 */ 3546 .macro EXEC_COREALL_REVEQ 3547 mov $2, $1 3548 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3549 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3550 cmp $2, $0 3551 b.ne 1f 3552 .endmacro 3553 3554 .macro EXEC_COREALL_REVLO 3555 mov $2, $1 3556 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3557 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3558 cmp $2, $0 3559 b.pl 1f 3560 .endmacro 3561 3562 .macro EXEC_COREALL_REVHS 3563 mov $2, $1 3564 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3565 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3566 cmp $2, $0 3567 b.mi 1f 3568 .endmacro 3569 3570 .macro CMP_FOREACH reg, cc, label, car, cdr:vararg 3571 cmp \reg, \car 3572 b.\cc \label 3573 .ifnb \cdr 3574 CMP_FOREACH \reg, \cc, \label, \cdr 3575 .endif 3576 .endm 3577 3578 .macro EXEC_COREIN_REVALL midr_el1, scratch, midr_list:vararg 3579 and \scratch, \midr_el1, #MIDR_EL1_PNUM_MASK 3580 CMP_FOREACH \scratch, eq, Lmatch\@, \midr_list 3581 b 1f 3582 Lmatch\@: 3583 .endm 3584 3585 /* 3586 * $0 - MIDR_SOC[_CORE], e.g. MIDR_ARUBA_VORTEX 3587 * $1 - GPR containing MIDR_EL1 value 3588 * $2 - Scratch register 3589 */ 3590 .macro EXEC_COREEQ_REVALL 3591 and $2, $1, #MIDR_EL1_PNUM_MASK 3592 cmp $2, $0 3593 b.ne 1f 3594 .endmacro 3595 3596 /* 3597 * $0 - CPU_VERSION_XX, e.g. CPU_VERSION_B1 3598 * $1 - GPR containing MIDR_EL1 value 3599 * $2 - Scratch register 3600 */ 3601 .macro EXEC_PCORE_REVEQ 3602 ARM64_IS_PCORE $2 3603 cbz $2, 1f 3604 3605 mov $2, $1 3606 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3607 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3608 cmp $2, $0 3609 b.ne 1f 3610 .endmacro 3611 3612 .macro EXEC_PCORE_REVLO 3613 ARM64_IS_PCORE $2 3614 cbz $2, 1f 3615 3616 mov $2, $1 3617 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3618 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3619 cmp $2, $0 3620 b.pl 1f 3621 .endmacro 3622 3623 .macro EXEC_PCORE_REVHS 3624 ARM64_IS_PCORE $2 3625 cbz $2, 1f 3626 3627 mov $2, $1 3628 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3629 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3630 cmp $2, $0 3631 b.mi 1f 3632 .endmacro 3633 3634 .macro EXEC_ECORE_REVEQ 3635 ARM64_IS_ECORE $2 3636 cbz $2, 1f 3637 3638 mov $2, $1 3639 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3640 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3641 cmp $2, $0 3642 b.ne 1f 3643 .endmacro 3644 3645 .macro EXEC_ECORE_REVLO 3646 ARM64_IS_ECORE $2 3647 cbz $2, 1f 3648 3649 mov $2, $1 3650 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3651 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3652 cmp $2, $0 3653 b.pl 1f 3654 .endmacro 3655 3656 .macro EXEC_ECORE_REVHS 3657 ARM64_IS_ECORE $2 3658 cbz $2, 1f 3659 3660 mov $2, $1 3661 bfi $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #4 3662 ubfx $2, $2, #(MIDR_EL1_VAR_SHIFT - 4), #8 3663 cmp $2, $0 3664 b.mi 1f 3665 .endmacro 3666 3667 /* 3668 * $0 - GPR containing MIDR_EL1 value 3669 * $1 - Scratch register 3670 */ 3671 .macro EXEC_PCORE_REVALL 3672 ARM64_IS_PCORE $1 3673 cbz $1, 1f 3674 .endmacro 3675 3676 .macro EXEC_ECORE_REVALL 3677 ARM64_IS_ECORE $1 3678 cbz $1, 1f 3679 .endmacro 3680 3681 /* 3682 * Macro that defines the label that all EXEC_COREXX_REVXX macros jump to. 3683 */ 3684 .macro EXEC_END 3685 1: 3686 .endmacro 3687 3688 /* 3689 * Wedges CPUs with a specified core that are below a specified revision. This 3690 * macro is intended for CPUs that have been deprecated in iBoot and may have 3691 * incorrect behavior if they continue running xnu. 3692 */ 3693 .macro DEPRECATE_COREEQ_REVLO core, rev, midr_el1, scratch 3694 EXEC_COREEQ_REVLO \core, \rev, \midr_el1, \scratch 3695 /* BEGIN IGNORE CODESTYLE */ 3696 b . 3697 /* END IGNORE CODESTYLE */ 3698 EXEC_END 3699 .endmacro 3700 3701 /* 3702 * Sets bits in an SPR register. 3703 * arg0: Name of the register to be accessed. 3704 * arg1: Mask of bits to be set. 3705 * arg2: Scratch register 3706 */ 3707 .macro HID_SET_BITS 3708 mrs $2, $0 3709 orr $2, $2, $1 3710 msr $0, $2 3711 .endmacro 3712 3713 /* 3714 * Clears bits in an SPR register. 3715 * arg0: Name of the register to be accessed. 3716 * arg1: Mask of bits to be cleared. 3717 * arg2: Scratch register 3718 */ 3719 .macro HID_CLEAR_BITS 3720 mrs $2, $0 3721 bic $2, $2, $1 3722 msr $0, $2 3723 .endmacro 3724 3725 /* 3726 * Combines the functionality of HID_CLEAR_BITS followed by HID_SET_BITS into 3727 * a single read-modify-write sequence. 3728 * arg0: Name of the register to be accessed. 3729 * arg1: Mask of bits to be cleared. 3730 * arg2: Value to insert 3731 * arg3: Scratch register 3732 */ 3733 .macro HID_INSERT_BITS 3734 mrs $3, $0 3735 bic $3, $3, $1 3736 orr $3, $3, $2 3737 msr $0, $3 3738 .endmacro 3739 3740 /* 3741 * Replaces the value of a field in an implementation-defined system register. 3742 * sreg: system register name 3743 * field: field name within the sysreg, where the assembler symbols 3744 * ARM64_REG_<field>_{shift,width} specify the bounds of the field 3745 * (note that preprocessor macros will not work here) 3746 * value: the value to insert 3747 * scr{1,2}: scratch regs 3748 */ 3749 .macro HID_WRITE_FIELD sreg, field, val, scr1, scr2 3750 mrs \scr1, \sreg 3751 mov \scr2, \val 3752 bfi \scr1, \scr2, ARM64_REG_\sreg\()_\field\()_shift, ARM64_REG_\sreg\()_\field\()_width 3753 msr \sreg, \scr1 3754 .endmacro 3755 3756 /* 3757 * This macro is a replacement for ERET with better security properties. 3758 * 3759 * It prevents "straight-line speculation" (an Arm term) past the ERET. 3760 */ 3761 .macro ERET_NO_STRAIGHT_LINE_SPECULATION 3762 eret 3763 #if __ARM_SB_AVAILABLE__ 3764 sb // Technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures 3765 #else /* __ARM_SB_AVAILABLE__ */ 3766 isb // ISB technically unnecessary on Apple micro-architectures, may restrict mis-speculation on other architectures 3767 nop // Sequence of six NOPs to pad out and terminate instruction decode group */ 3768 nop 3769 nop 3770 nop 3771 nop 3772 nop 3773 #endif /* !__ARM_SB_AVAILABLE__ */ 3774 .endmacro 3775 3776 3777 #endif /* __ASSEMBLER__ */ 3778 3779 #define MSR(reg, src) __asm__ volatile ("msr " reg ", %0" :: "r" (src)) 3780 #define MRS(dest, reg) __asm__ volatile ("mrs %0, " reg : "=r" (dest)) 3781 3782 #if XNU_MONITOR 3783 #define __ARM_PTE_PHYSMAP__ 1 3784 #define PPL_STATE_KERNEL 0 3785 #define PPL_STATE_DISPATCH 1 3786 #define PPL_STATE_PANIC 2 3787 #define PPL_STATE_EXCEPTION 3 3788 #endif 3789 3790 3791 #if HAS_ESB 3792 #define DISR_A_SHIFT 31 3793 #define DISR_A (1ULL << DISR_A_SHIFT) 3794 #endif 3795 #endif /* _ARM64_PROC_REG_H_ */ 3796