1 /* 2 * Copyright (c) 2024 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _VM_VM_MTEINFO_INTERNAL_H_ 30 #define _VM_VM_MTEINFO_INTERNAL_H_ 31 32 #include <stdint.h> 33 #include <kern/kcdata.h> 34 #include <mach/vm_param.h> 35 #ifndef VM_MTE_FF_VERIFY 36 #include <vm/vm_page.h> 37 #if MACH_KERNEL_PRIVATE 38 #include <vm/vm_page_internal.h> 39 #endif 40 #endif /* VM_MTE_FF_VERIFY */ 41 42 __BEGIN_DECLS 43 #if HAS_MTE 44 45 #pragma mark Types 46 47 struct vm_page; 48 49 /*! 50 * @typedef mte_cell_state_t 51 * 52 * @abstract 53 * This type denotes the state of a cell, which influences which queue it 54 * belongs to. 55 * 56 * @discussion 57 * For any given state untagged covered pages associated with a tag storage page 58 * (or its cell) can be allocated. However, tagged covered pages can only be 59 * allocated if the associated tag storage cell is in the MTE_STATE_ACTIVE 60 * state. 61 * 62 * @const MTE_STATE_DISABLED 63 * This cell is disabled from being selected as a tag storage page. 64 * 65 * This can happen for: 66 * - recursive tag storage, 67 * - tag storage for iBoot carveouts, 68 * - tag storage for unmanaged memory not using MTE, 69 * - pages with ECC errors that have been retired. 70 * 71 * In the first two cases, the page is usable for regular untaggable usage, 72 * and is on the global free queue, in the latter case the page is retired 73 * and unusable. 74 * 75 * @const MTE_STATE_PINNED 76 * The tag storage page is currently used as non tag storage, and a reclaim was 77 * attempted and failed due to the page being pinned (most likely wired). 78 * 79 * This state is discovered lazily by the refill thread as it would be expensive 80 * to maintain explicitly. It serves as a way to not attempt reclaiming the same 81 * pages over and over again when they are in a state that doesn't permit it. 82 * 83 * This page shall have no covered pages with MTE enabled (the SPTM will 84 * enforce this). 85 * 86 * @const MTE_STATE_DEACTIVATING 87 * List of pages in the process of being deactivated (from MTE_STATE_ACTIVE). 88 * 89 * This page might transiently have pages with MTE enabled, 90 * however none should be in use. 91 * 92 * @const MTE_STATE_CLAIMED 93 * The tag storage page is currently used as non tag storage, and is typically 94 * typed XNU_DEFAULT (though nothing prevents other uses, provided the usage is 95 * relocatable). 96 * 97 * This page shall have no covered pages with MTE enabled (the SPTM will 98 * enforce this). 99 * 100 * @const MTE_STATE_INACTIVE 101 * The tag storage page is currently completely free and unused, and is typed 102 * XNU_DEFAULT. 103 * 104 * This page shall have no covered pages with MTE enabled (the SPTM will 105 * enforce this). 106 * 107 * @const MTE_STATE_RECLAIMING 108 * List of pages which used to be in MTE_STATE_CLAIMED state, that the fill 109 * thread is attempting to relocate. 110 * 111 * This page shall have no covered pages with MTE enabled (the SPTM must 112 * enforce this). 113 * 114 * @const MTE_STATE_ACTIVATING 115 * List of pages in the process of being activated (from MTE_STATE_INACTIVE). 116 * 117 * This page shall have no covered pages with MTE enabled (the SPTM must 118 * enforce this). 119 * 120 * @const MTE_STATE_ACTIVE 121 * The tag storage page is currently typed XNU_TAG_STORAGE and might have 122 * covered pages with MTE enabled. 123 */ 124 __enum_closed_decl(mte_cell_state_t, uint8_t, { 125 MTE_STATE_DISABLED, 126 MTE_STATE_PINNED, 127 MTE_STATE_DEACTIVATING, 128 MTE_STATE_CLAIMED, 129 MTE_STATE_INACTIVE, 130 MTE_STATE_RECLAIMING, 131 MTE_STATE_ACTIVATING, 132 MTE_STATE_ACTIVE, 133 }); 134 135 /*! 136 * @const MTE_BUCKETS_COUNT_MAX 137 * The maximum number of buckets in a cell list. 138 * 139 * Cell list buckets are a function of the number of free covered pages 140 * associated with the tag storage pages being considered: 141 * - bucket 0: no free covered page. 142 * - bucket 1: 1 to 8 free covered pages. 143 * - bucket 2: 9 to 16 free covered pages. 144 * - bucket 3: 17 to 24 free covered pages. 145 * - bucket 4: 25 to 32 free covered pages. 146 */ 147 #define MTE_BUCKETS_COUNT_MAX 5 148 149 /*! 150 * @typedef mte_cell_list_idx_t 151 * 152 * @abstract 153 * Represents the index of a cell list inside the mteinfo data structure. 154 * 155 * @discussion 156 * The order of these values matter: 157 * - Lists with single buckets must be first 158 * - Active lists must be last 159 */ 160 __enum_closed_decl(mte_cell_list_idx_t, uint32_t, { 161 MTE_LIST_DISABLED_IDX = MTE_STATE_DISABLED, 162 MTE_LIST_PINNED_IDX = MTE_STATE_PINNED, 163 MTE_LIST_DEACTIVATING_IDX = MTE_STATE_DEACTIVATING, 164 MTE_LIST_CLAIMED_IDX = MTE_STATE_CLAIMED, 165 MTE_LIST_INACTIVE_IDX = MTE_STATE_INACTIVE, 166 MTE_LIST_RECLAIMING_IDX = MTE_STATE_RECLAIMING, 167 MTE_LIST_ACTIVATING_IDX = MTE_STATE_ACTIVATING, 168 MTE_LIST_ACTIVE_0_IDX = MTE_STATE_ACTIVE, 169 MTE_LIST_ACTIVE_IDX = MTE_STATE_ACTIVE + 1, 170 171 MTE_LISTS_COUNT, 172 }); 173 174 /*! 175 * @typedef mte_cell_list_t 176 * 177 * @abstract 178 * This data structure represents a segregated list of page queues. 179 * 180 * @discussion 181 * The list is segregated per number of free pages. Each segregation queue is 182 * called a "bucket". 183 * 184 * @field mask 185 * Mask of all the non empty buckets in this list. 186 * 187 * @field count 188 * Total number of cells on the list 189 * 190 * @field buckets 191 * A vector of queues this list covers. The number of buckets depends on the 192 * list and can range from 1 to MTE_BUCKETS_COUNT_MAX 193 */ 194 typedef struct mte_cell_list { 195 uint32_t mask; 196 uint32_t count; 197 struct mte_cell_queue_head *buckets; 198 } *mte_cell_list_t; 199 200 /*! 201 * @abstract 202 * Indices for the mte free queue buckets. 203 * 204 * @discussion 205 * This bucketing is designed to order allocations: 206 * 207 * - untagged allocations will consider buckets in ascending order from 208 * @c MTE_FREE_UNTAGGABLE through @c MTE_FREE_UNTAGGABLE_ACTIVATING. 209 * 210 * - tagged allocations will consider buckets in descending order from 211 * @c MTE_FREE_ACTIVE_3 through MTE_FREE_ACTIVE_0. 212 * 213 * Said another way: lower indices denote buckets where untagged allocations are 214 * more desirable and higher indices buckets where tagged allocations are more 215 * desirable. 216 * 217 * 218 * @const MTE_FREE_UNTAGGABLE_0 219 * The bucket for pages with disabled, pinned, deactivating tag storage 220 * pages, or claimed with 16 or less associated free pages. 221 * 222 * @const MTE_FREE_UNTAGGABLE_1 223 * The bucket for claimed pages with 17 or more associated free pages or 224 * inactive pages with 16 or less associated free pages. 225 * 226 * @const MTE_FREE_UNTAGGABLE_2 227 * The bucket for pages with inactive tag storage pages which have 17 associated 228 * covered free pages or more. 229 * 230 * 231 * @const MTE_FREE_UNTAGGABLE_ACTIVATING 232 * The bucket for pages with activating or reclaiming tag storage pages. 233 * 234 * This bucket is kept "last" because the system has selected these pages 235 * for upgrading into the active pools either from inactive or claimed 236 * as being the best current candidates. In other words, once the activation 237 * is finished, we expect these pages to fall into a high @c MTE_FREE_ACTIVE_* 238 * bucket. 239 * 240 * These transitions are unfortunately not atomic and the untagged workloads 241 * can tap into these during the transitions defeating the purpose of the 242 * upgrades, so by making them last, we protect them from untagged allocations. 243 * 244 * 245 * @const MTE_FREE_NOT_QUEUED 246 * This a pseudo bucket for tag storage pages with no free pages. 247 */ 248 __enum_closed_decl(mte_free_queue_idx_t, uint32_t, { 249 MTE_FREE_UNTAGGABLE_0, 250 MTE_FREE_UNTAGGABLE_1, 251 MTE_FREE_UNTAGGABLE_2, 252 MTE_FREE_ACTIVE_0, 253 MTE_FREE_ACTIVE_1, 254 MTE_FREE_ACTIVE_2, 255 MTE_FREE_ACTIVE_3, 256 MTE_FREE_UNTAGGABLE_ACTIVATING, 257 MTE_FREE_NOT_QUEUED, 258 }); 259 260 261 #pragma mark Counters and Globals 262 263 /*! 264 * The cell lists, in mte_cell_list_idx_t order. 265 * 266 * Each list contains this many buckets: 267 * - MTE_LIST_DISABLED_IDX: 1 268 * - MTE_LIST_PINNED_IDX: 1 269 * - MTE_LIST_CLAIMED_IDX: MTE_BUCKETS_COUNT_MAX 270 * - MTE_LIST_INACTIVE_IDX: MTE_BUCKETS_COUNT_MAX 271 * - MTE_LIST_RECLAIMING_IDX: 1 272 * - MTE_LIST_ACTIVATING_IDX: 1 273 * - MTE_LIST_ACTIVE_0_IDX: MTE_BUCKETS_COUNT_MAX 274 * - MTE_LIST_ACTIVE_IDX: 1 275 */ 276 extern struct mte_cell_list mte_info_lists[MTE_LISTS_COUNT]; 277 278 /*! 279 * The MTE free queues. 280 */ 281 extern struct vm_page_free_queue mte_free_queues[MTE_FREE_NOT_QUEUED]; 282 283 /*! 284 * The queue of claimable tag storage pages. 285 * (pages with <= 8 associated free pages and inactive). 286 */ 287 extern struct vm_page_free_queue mte_claimable_queue; 288 289 #ifndef VM_MTE_FF_VERIFY 290 291 /*! 292 * @abstract 293 * The type for MTE related per-cpu free queues. 294 * 295 * @field free_tagged_pages 296 * The per-cpu free list of tagged pages. 297 * Pages on this list have the @c VM_PAGE_ON_FREE_LOCAL_Q state. 298 * Their associated cell will have state @c MTE_STATE_ACTIVE. 299 * This list is only accessed by the current CPU with preemption disabled. 300 * 301 * @field free_claimed_pages 302 * The per-cpu free queue of claimed pages. 303 * Pages on this list have the @c VM_PAGE_ON_FREE_LOCAL_Q state. 304 * Their associated cell will have state @c MTE_STATE_CLAIMED. 305 * Access to this queue is protected by the @c free_claimed_lock. 306 * 307 * @field free_claimed_lock 308 * The lock protecting the per-cpu free queue of claimed pages. 309 * This allows for the refill thread to steal claimed pages from this queue. 310 * 311 * @field deactivate_suspend 312 * Per-cpu marker that suspends page deactivations until the current CPU 313 * has finished some untagging (see mteinfo_tag_storage_deactivate_barrier()). 314 */ 315 typedef struct mte_pcpu { 316 vm_page_queue_head_t free_claimed_pages VM_PAGE_PACKED_ALIGNED; 317 vm_page_t free_tagged_pages; 318 lck_ticket_t free_claimed_lock; 319 uint32_t deactivate_suspend; 320 } *mte_pcpu_t; 321 PERCPU_DECL(struct mte_pcpu, mte_pcpu); 322 323 /*! 324 * @var vm_cpu_free_count 325 * Scalable counter of the number of free pages CPU free list. 326 * (not an MTE concept but here for the sake of vm_unix.c) 327 * 328 * @var vm_cpu_free_tagged_count 329 * Scalable counter of the number of free tagged pages on CPU free lists. 330 * 331 * @var vm_cpu_free_tagged_count 332 * Scalable counter of the number of free claimed pages on CPU free lists. 333 */ 334 SCALABLE_COUNTER_DECLARE(vm_cpu_free_count); 335 SCALABLE_COUNTER_DECLARE(vm_cpu_free_tagged_count); 336 SCALABLE_COUNTER_DECLARE(vm_cpu_free_claimed_count); 337 338 #endif /* VM_MTE_FF_VERIFY */ 339 340 /*! 341 * @var vm_page_free_taggable_count 342 * Number of free pages in the MTE_FREE_ACTIVE_* MTE free queue buckets. 343 * 344 * @var vm_page_free_unmanaged_tag_storage_count 345 * Number of free unmanaged tag storage pages. These do not participate in the 346 * global free count. 347 * 348 * @var vm_page_recursive_tag_storage_count 349 * Number of recursive tag storage pages. 350 * These should be VM_MEMORY_CLASS_DEAD_TAG_STORAGE. 351 * 352 * @var vm_page_retired_tag_storage_count 353 * Number of retired tag storage pages. 354 * These should be unusable pages due to ECC errors. 355 * 356 * @var vm_page_unmanaged_tag_storage_count 357 * Number of unmanaged tag storage pages. 358 * These should be VM_MEMORY_CLASS_DEAD_TAG_STORAGE. 359 * 360 * @var vm_page_wired_tag_storage_count 361 * Number of tag storage range pages that are wired (note: this is not 362 * current the number of VM_MEMORY_CLASS_TAG_STORAGE pages that are wired). 363 * 364 * @var vm_page_tagged_count 365 * Number of tagged pages in use. 366 * 367 * @var vm_mte_refill_thread_wakeups 368 * The number of times the refill thread was woken up. 369 * 370 * @var vm_page_tag_storage_activation_count 371 * Number of activation (inactive/claimed -> active) transitions ever done. 372 * 373 * @var vm_page_tag_storage_deactivation_count 374 * Number of deactivation (active -> inactive) transitions ever done. 375 * 376 * @var vm_page_tag_storage_reclaim_from_cpu_count 377 * Number of times a claimed tag storage page was successfully reclaimed from 378 * a cpu free list. 379 * 380 * @var vm_page_tag_storage_reclaim_success_count 381 * Number of times a claimed tag storage page was successfully reclaimed. 382 * 383 * @var vm_page_tag_storage_reclaim_failure_count 384 * Number of times a claimed tag storage page failed to be reclaimed. 385 * 386 * @var vm_page_tag_storage_reclaim_wired_failure_count 387 * Number of times a claimed tag storage page failed to be reclaimed because it 388 * was wired. 389 * 390 * @var vm_page_tag_storage_wire_relocation_count 391 * Number of relocations of tag storage pages due to wiring. 392 * 393 * @var vm_page_tag_storage_reclaim_compressor_failure_count 394 * Number of times a claimed tag storage page failed to be reclaimed because it 395 * was used in the compressor pool and getting swapped out. 396 * 397 * @var vm_page_tag_storage_compressor_relocation_count 398 * Number of relocations of tag storage pages due to the compressor. 399 * 400 * @var vm_page_free_wanted_tagged 401 * Number of threads that are waiting for covered tagged pages. Also the event 402 * those threads wait on. 403 * 404 * @var vm_page_free_wanted_tagged_privileged 405 * Number privileged threads that are waiting for covered tagged pages. Also 406 * the event those threads wait on. 407 */ 408 extern uint32_t vm_page_free_taggable_count; 409 extern uint32_t vm_page_free_unmanaged_tag_storage_count; 410 extern uint32_t vm_page_recursive_tag_storage_count; 411 extern uint32_t vm_page_retired_tag_storage_count; 412 extern uint32_t vm_page_unmanaged_tag_storage_count; 413 extern uint32_t vm_page_wired_tag_storage_count; 414 extern uint32_t vm_page_tagged_count; 415 extern uint64_t vm_mte_refill_thread_wakeups; 416 extern uint64_t vm_page_tag_storage_activation_count; 417 extern uint64_t vm_page_tag_storage_deactivation_count; 418 extern uint64_t vm_page_tag_storage_reclaim_from_cpu_count; 419 extern uint64_t vm_page_tag_storage_reclaim_success_count; 420 extern uint64_t vm_page_tag_storage_reclaim_failure_count; 421 extern uint64_t vm_page_tag_storage_reclaim_wired_failure_count; 422 extern uint64_t vm_page_tag_storage_wire_relocation_count; 423 extern uint64_t vm_page_tag_storage_reclaim_compressor_failure_count; 424 extern uint64_t vm_page_tag_storage_compressor_relocation_count; 425 extern uint32_t vm_page_free_wanted_tagged; 426 extern uint32_t vm_page_free_wanted_tagged_privileged; 427 428 #ifndef VM_MTE_FF_VERIFY 429 /*! 430 * @var vm_cpu_claimed_count 431 * Scalable counter of the number of claimed tag storage pages allocated. 432 */ 433 SCALABLE_COUNTER_DECLARE(vm_cpu_claimed_count); 434 #endif /* VM_MTE_FF_VERIFY */ 435 436 /*! 437 * @var vm_page_tag_storage_reserved 438 * Number of free tag storage pages reserved for the fill thread. 439 */ 440 extern uint32_t vm_page_tag_storage_reserved; 441 442 /*! 443 * @var vm_mte_tag_storage_for_compressor 444 * Whether we use tag storage pages for the compressor pool. 445 */ 446 extern bool vm_mte_tag_storage_for_compressor; 447 448 /*! 449 * @var vm_mte_tag_storage_for_vm_tags_mask 450 * Which VM tags can use tag storage. 451 */ 452 extern bitmap_t vm_mte_tag_storage_for_vm_tags_mask[BITMAP_LEN(VM_MEMORY_COUNT)]; 453 454 #if MACH_KERNEL_PRIVATE 455 #pragma mark Tag storage space state machine 456 457 /*! 458 * @function mteinfo_tag_storage_disabled() 459 * 460 * @abstract 461 * Returns whether a tag storage page is disabled. 462 * 463 * @discussion 464 * Unlike other mteinfo_* functions, this can be called without the free queue 465 * lock held because disabling pages is a one way transition after lockdown, 466 * and before lockdown the kernel is single threaded. 467 * 468 * @param page The pointer to a page inside the tag storage space. 469 */ 470 extern bool mteinfo_tag_storage_disabled(const struct vm_page *page); 471 472 473 /*! 474 * @function mteinfo_tag_storage_is_active() 475 * 476 * @abstract 477 * Returns whether a tag storage page is active. 478 * 479 * @param page The pointer to a page inside the tag storage space. 480 */ 481 extern bool mteinfo_tag_storage_is_active(const struct vm_page *page); 482 483 484 /*! 485 * @function mteinfo_tag_storage_set_retired() 486 * 487 * @abstract 488 * Mark a tag storage page as retired due to ECC errors. 489 * 490 * @param page The pointer to a page inside the tag storage space. 491 */ 492 extern void mteinfo_tag_storage_set_retired(struct vm_page *page); 493 494 /*! 495 * @function mteinfo_tag_storage_set_inactive() 496 * 497 * @abstract 498 * Mark a tag storage page as inactive. 499 * 500 * @param page The pointer to a page inside the tag storage space. 501 * @param init This is the initial "inactive" transition. 502 */ 503 extern void mteinfo_tag_storage_set_inactive(struct vm_page *page, bool init); 504 505 /*! 506 * @function mteinfo_tag_storage_set_claimed() 507 * 508 * @abstract 509 * Mark a tag storage page as claimed for regular memory usage. 510 * 511 * @discussion 512 * The tag storage page must be either inactive or reclaiming. 513 * 514 * @param page The pointer to a page inside the tag storage space. 515 */ 516 extern void mteinfo_tag_storage_set_claimed(struct vm_page *page); 517 518 /*! 519 * @function mteinfo_tag_storage_wakeup() 520 * 521 * @abstract 522 * Mark a tag storage page as no longer pinned. 523 * 524 * @discussion 525 * The tag storage page must be in the pinned state. 526 * The page queues lock must be held. 527 * 528 * @param page The pointer to a page inside the tag storage space. 529 * @param fq_locked Whether the page free queue lock is held. 530 */ 531 extern void mteinfo_tag_storage_wakeup(struct vm_page *page, bool fq_locked); 532 533 534 #pragma mark Covered pages state machine 535 536 /*! 537 * @function mteinfo_covered_page_taggable() 538 * 539 * @abstract 540 * Returns whether a specified covered page has an active tag storage page 541 * associated. 542 * 543 * @param pnum A page number outside of the tag storage space. 544 */ 545 extern bool mteinfo_covered_page_taggable(ppnum_t pnum); 546 547 /*! 548 * @function mteinfo_covered_page_set_free() 549 * 550 * @abstract 551 * Mark the specified untagged page as free in its tag storage tracking metadata. 552 * 553 * @param pnum A page number outside of the tag storage space. 554 * @param tagged Whether the page will was used as tagged. 555 */ 556 extern void mteinfo_covered_page_set_free(ppnum_t pnum, bool tagged); 557 558 /*! 559 * @function mteinfo_covered_page_set_used() 560 * 561 * @abstract 562 * Mark the specified untagged page as used in its tag storage tracking metadata. 563 * 564 * @param pnum A page number outside of the tag storage space. 565 * @param tagged Whether the page will be used as tagged. 566 */ 567 extern void mteinfo_covered_page_set_used(ppnum_t pnum, bool tagged); 568 569 /*! 570 * @function mteinfo_covered_page_set_stolen_tagged() 571 * 572 * @abstract 573 * Mark the specified page as using MTE in its tag storage tracking metadata. 574 * 575 * @discussion 576 * These pages are expected to be "stolen" in that bootstrap has allocated them 577 * through bootstrap allocation strategies (see: bump allocation) before MachVM 578 * is properly initialized and able to call into this module properly. Because 579 * of this, bootstrap will call into this method to directly tell the module 580 * that the page is used as tagged now. 581 * 582 * @param pnum A page number outside of the tag storage space, 583 * the page must be used. 584 */ 585 __startup_func 586 extern void mteinfo_covered_page_set_stolen_tagged(ppnum_t pnum); 587 588 /*! 589 * @function mteinfo_covered_page_clear_tagged() 590 * 591 * @abstract 592 * Mark the specified page as no longer using MTE in its tag storage tracking 593 * metadata, while remaining "in use". 594 * 595 * @param pnum A page number outside of the tag storage space, 596 * the page must be used. 597 */ 598 extern void mteinfo_covered_page_clear_tagged(ppnum_t pnum); 599 600 601 #pragma mark Activate 602 603 /*! 604 * @function mteinfo_tag_storage_try_activate() 605 * 606 * @abstract 607 * Try to activate tag storage pages in order to make a certain amount of 608 * covered taggable pages available. 609 * 610 * @discussion 611 * This must be called with the page free queue lock held. 612 * This function will have dropped the free queue lock if it returns true. 613 * 614 * @param target how many covered taggable free pages to try to generate 615 * as a result of this activation. 616 * @param spin_mode whether to take the free page queue lock in spin mode. 617 * 618 * @returns whether the page free queue lock was dropped 619 * (in which case it means pages have been activated, 620 * either by this thread or another we synchronized with). 621 */ 622 extern bool mteinfo_tag_storage_try_activate(uint32_t target, bool spin_mode); 623 624 625 #pragma mark Refill 626 627 /*! 628 * @function mteinfo_wake_fill_thread() 629 * 630 * @abstract 631 * Wake the fill thread if it has not already been woken. 632 */ 633 extern void mteinfo_wake_fill_thread(void); 634 635 636 #pragma mark Alloc 637 638 /*! 639 * @function mteinfo_free_queue_grab() 640 * 641 * @abstract 642 * Gets pages from the MTE free queue. 643 * 644 * @discussion 645 * Clients cannot get more pages than the free queue has; attempting to do so 646 * will cause a panic. 647 * 648 * @param options The grab options. 649 * @param mem_class The memory class to allocate from. 650 * @param num_pages The number of pages to grab. 651 * @param q_state The vmp_q_state to set on the page. 652 * 653 * @returns 654 * A list of pages; the list will be at least num_pages long. 655 */ 656 extern vm_page_list_t mteinfo_free_queue_grab( 657 vm_grab_options_t options, 658 vm_memory_class_t mem_class, 659 unsigned int num_pages, 660 vm_page_q_state_t q_state); 661 662 663 /*! 664 * @function mteinfo_page_list_fix_tagging() 665 * 666 * @abstract 667 * Fix the tagging for a list returned by @c mteinfo_free_queue_grab(). 668 * 669 * @discussion 670 * Preemption must be disabled (under the same preemption disabled 671 * hold as the call to @c mteinfo_free_queue_grab() that preceded). 672 * 673 * @param mem_class The memory class being allocated. 674 * @param list The list returned by mteinfo_free_queue_grab(). 675 */ 676 extern void mteinfo_page_list_fix_tagging( 677 vm_memory_class_t mem_class, 678 vm_page_list_t *list); 679 680 681 #pragma mark Bootstrap API 682 683 extern void mteinfo_init(uint32_t num_tag_pages); 684 685 #if HIBERNATION 686 687 /*! 688 * @abstract 689 * Iterate all free pages from the MTE free queue (covered or tag storage). 690 */ 691 extern void mteinfo_free_queue_foreach(void (^block)(vm_page_t)); 692 693 #endif /* HIBERNATION */ 694 695 /*! 696 * @function mteinfo_tag_storage_release_startup() 697 * 698 * @abstract 699 * Marks a tag storage page active or inactive, as appropriate. 700 * 701 * @discussion 702 * The tag storage page does not have to be active, but none of its covered 703 * pages may have been made tagged. If the tag storage page was active, then 704 * it will be put on a list to be added to the mte_tags_object by @see 705 * mteinfo_tag_storage_startup_list_flush. 706 * 707 * @param page 708 * The tag storage page to be marked. 709 */ 710 __startup_func 711 extern void mteinfo_tag_storage_release_startup(vm_page_t page); 712 713 #endif /* MACH_KERNEL_PRIVATE */ 714 #pragma mark Counter methods 715 716 /*! 717 * @function mteinfo_tag_storage_fragmentation() 718 * 719 * @abstract 720 * Computed value returning the tag storage fragmentation 721 * in parts per thousand. 722 * 723 * @param actual Whether to show the "actual" fragmentation 724 * or what is achievable assuming enough memory 725 * pressure. 726 */ 727 extern uint32_t mteinfo_tag_storage_fragmentation(bool actual); 728 729 /*! 730 * @function mteinfo_tag_storage_active() 731 * 732 * @abstract 733 * Computed value returning the number of active tag storage pages. 734 */ 735 extern uint32_t mteinfo_tag_storage_active(bool fq_locked); 736 737 /*! 738 * @function mteinfo_tag_storage_free_pages_for_covered() 739 * 740 * @abstract 741 * Returns the number of covered pages that are free for the tag storage page 742 * associated with this page. 743 * 744 * @param page The pointer to a page outside of the tag storage space. 745 */ 746 extern uint32_t mteinfo_tag_storage_free_pages_for_covered(const struct vm_page *page); 747 748 /*! 749 * @function mteinfo_increment_wire_count() 750 * 751 * @abstract 752 * Increment the wired tag storage page counter if the given page is tag 753 * storage. 754 * 755 * @discussion 756 * This currently considers all pages in the tag storage range to be tag 757 * storage, whether or not they are VM_MEMORY_CLASS_TAG_STORAGE. This is due 758 * to how the other page counters are initialized; currently they account for 759 * all tag storage range pages. 760 * 761 * Note that the callers should make sure the reason for wiring isn't because 762 * the page is used for tag storage by checking against the VM_KERN_MEMORY_MTAG 763 * tag which is used in that case. 764 * 765 * @param page 766 * The page being wired. 767 */ 768 extern void mteinfo_increment_wire_count(vm_page_t page); 769 770 /*! 771 * @function mteinfo_decrement_wire_count() 772 * 773 * @abstract 774 * Decrement the wired tag storage page counter if the given page is tag 775 * storage. 776 * 777 * @discussion 778 * This currently considers all pages in the tag storage range to be tag 779 * storage, whether or not they are VM_MEMORY_CLASS_TAG_STORAGE. This is due 780 * to how the other page counters are initialized; currently they account for 781 * all tag storage range pages. 782 * 783 * Note that this function is a no-op if the page was associated with the 784 * mte_tags_object as it means its wiring was because it's used for tag storage. 785 * 786 * @param page The page being wired. 787 * @param pqs_locked Whether the page queues locked is held 788 * (possibly in spin mode). 789 */ 790 extern void mteinfo_decrement_wire_count(vm_page_t page, bool pqs_locked); 791 792 #ifndef VM_MTE_FF_VERIFY 793 /*! 794 * @function mteinfo_vm_tag_can_use_tag_storage() 795 * 796 * @abstract 797 * Determine if a given VM tag is eligible to dip into tag storage. 798 * 799 * @param vm_tag The VM tag in question. 800 */ 801 extern bool mteinfo_vm_tag_can_use_tag_storage(vm_tag_t vm_tag); 802 /*! 803 * @function kdp_mteinfo_snapshot() 804 * 805 * @abstract 806 * Snapshot the current state of all tag storage pages. 807 * 808 * @discussion 809 * Can only be called from debugger context. 810 * 811 * @param cells Array of struct mte_cell_info (from kcdata.h) 812 * @param count Size of the array, must match mte_tag_storage_count 813 */ 814 extern void kdp_mteinfo_snapshot(struct mte_info_cell __counted_by(count) *cells, size_t count); 815 816 817 #endif /* VM_MTE_FF_VERIFY */ 818 819 #if DEBUG || DEVELOPMENT 820 /*! 821 * @function mteinfo_covered_page_is_kernel_tagged() 822 * 823 * @abstract 824 * Get the MTE tag data page of a covered page, as well as the internal offset 825 * to the tag data within the tag page. 826 * 827 * @param pnum The covered page in question. 828 * @param offset_to_tag_data Output parameter to return the offset to the 829 * covered page's data within the tag data page. 830 * 831 * @return The MTE tag data page of the covered page, or NULL if the covered 832 * page is not taggable. 833 * 834 */ 835 extern vm_page_t mteinfo_tag_page_from_covered_page( 836 ppnum_t pnum, 837 vm_offset_t * offset_to_tag_data); 838 #endif /* DEBUG || DEVELOPMENT */ 839 #endif /* HAS_MTE */ 840 __END_DECLS 841 842 #endif /* _VM_VM_MTEINFO_INTERNAL_H_ */ 843