1 /* 2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 32 #ifdef XNU_KERNEL_PRIVATE 33 34 #ifndef _KERN_STARTUP_H_ 35 #define _KERN_STARTUP_H_ 36 37 #include <stdbool.h> 38 #include <stddef.h> 39 #include <stdint.h> 40 41 #include <libkern/section_keywords.h> 42 43 __BEGIN_DECLS 44 45 __exported_push_hidden 46 47 /*! 48 * @enum startup_subsystem_id_t 49 * 50 * @abstract 51 * Represents a stage of kernel intialization, ubnd allows for subsystems 52 * to register initializers for a specific stage. 53 * 54 * @discussion 55 * Documentation of each subsystem initialization sequence exists in 56 * @file doc/lifecycle/startup.md. 57 */ 58 __enum_decl(startup_subsystem_id_t, uint32_t, { 59 STARTUP_SUB_NONE = 0, /**< reserved for the startup subsystem */ 60 61 STARTUP_SUB_TUNABLES, /**< support for the tunables subsystem */ 62 STARTUP_SUB_TIMEOUTS, /**< configurable machine timeouts */ 63 STARTUP_SUB_LOCKS, /**< various subsystem locks */ 64 STARTUP_SUB_KPRINTF, /**< kprintf initialization */ 65 66 STARTUP_SUB_PMAP_STEAL, /**< to perform various pmap carveouts */ 67 STARTUP_SUB_KMEM, /**< once kmem_alloc is ready */ 68 STARTUP_SUB_ZALLOC, /**< initialize zalloc and kalloc */ 69 STARTUP_SUB_KTRACE, /**< initialize kernel trace */ 70 STARTUP_SUB_PERCPU, /**< initialize the percpu subsystem */ 71 STARTUP_SUB_EVENT, /**< initiailze the event subsystem */ 72 73 STARTUP_SUB_CODESIGNING, /**< codesigning subsystem */ 74 STARTUP_SUB_OSLOG, /**< oslog and kernel logging */ 75 STARTUP_SUB_MACH_IPC, /**< Mach IPC */ 76 STARTUP_SUB_THREAD_CALL, /**< Thread calls */ 77 STARTUP_SUB_SYSCTL, /**< registers sysctls */ 78 STARTUP_SUB_EXCLAVES, /**< initialize exclaves */ 79 STARTUP_SUB_EARLY_BOOT, /**< interrupts/preemption are turned on */ 80 81 STARTUP_SUB_LOCKDOWN = ~0u, /**< reserved for the startup subsystem */ 82 }); 83 84 /*! 85 * Stores the last subsystem to have been fully initialized; 86 */ 87 extern startup_subsystem_id_t startup_phase; 88 89 /*! 90 * @enum startup_debug_t 91 * 92 * @abstract 93 * Flags set in the @c startup_debug global to configure startup debugging. 94 */ 95 __options_decl(startup_debug_t, uint32_t, { 96 STARTUP_DEBUG_NONE = 0x00000000, 97 STARTUP_DEBUG_VERBOSE = 0x00000001, 98 }); 99 100 /*! 101 * @enum startup_source_t 102 * 103 * @abstract 104 * The source from which a tunable was resolved. 105 */ 106 __options_decl(startup_source_t, uint32_t, { 107 STARTUP_SOURCE_DEFAULT, 108 STARTUP_SOURCE_DEVICETREE, 109 STARTUP_SOURCE_BOOTPARAM, 110 }); 111 112 extern startup_debug_t startup_debug; 113 114 /*! 115 * @enum startup_rank 116 * 117 * @abstract 118 * Specifies in which rank a given initializer runs within a given section 119 * to register initializers for a specific rank within the subsystem. 120 * 121 * @description 122 * A startup function, declared with @c STARTUP or @c STARTUP_ARG, can specify 123 * a rank within the subsystem they initialize. 124 * 125 * @c STARTUP_RANK_NTH(n) will let callbacks be run at stage @c n (0-based). 126 * 127 * @c STARTUP_RANK_FIRST, @c STARTUP_RANK_SECOND, @c STARTUP_RANK_THIRD and 128 * @c STARTUP_RANK_FOURTH are given as convenient names for these. 129 * 130 * @c STARTUP_RANK_MIDDLE is a reserved value that will let startup functions 131 * run after all the @c STARTUP_RANK_NTH(n) ones have. 132 * 133 * @c STARTUP_RANK_NTH_LATE_NTH(n) will let callbacks be run then in @c n rank 134 * after the @c STARTUP_RANK_MIDDLE ones (0-based). 135 * 136 * @c STARTUP_RANK_LAST callbacks will run absolutely last after everything 137 * else did for this subsystem. 138 */ 139 __enum_decl(startup_rank_t, uint32_t, { 140 #define STARTUP_RANK_NTH(n) ((startup_rank_t)(n - 1)) 141 STARTUP_RANK_FIRST = 0, 142 STARTUP_RANK_SECOND = 1, 143 STARTUP_RANK_THIRD = 2, 144 STARTUP_RANK_FOURTH = 3, 145 146 STARTUP_RANK_MIDDLE = 0x7fffffff, 147 148 #define STARTUP_RANK_LATE_NTH(n) \ 149 ((startup_rank_t)(STARTUP_RANK_MIDDLE + 1 + (n))) 150 151 STARTUP_RANK_LAST = 0xffffffff, 152 }); 153 154 #if KASAN 155 /* 156 * The use of weird sections that get unmapped confuse the hell out of kasan, 157 * so for KASAN leave things in regular __TEXT/__DATA segments 158 */ 159 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 160 #define STARTUP_CONST_SEGSECT "__DATA_CONST,__init" 161 #define STARTUP_DATA_SEGSECT "__DATA,__init" 162 #define STARTUP_HOOK_SEGMENT "__DATA" 163 #define STARTUP_HOOK_SECTION "__init_entry_set" 164 #elif defined(__x86_64__) 165 /* Intel doesn't have a __BOOTDATA but doesn't protect __KLD */ 166 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 167 #define STARTUP_CONST_SEGSECT "__KLDDATA,__const" 168 #define STARTUP_DATA_SEGSECT "__KLDDATA,__init" 169 #define STARTUP_HOOK_SEGMENT "__KLDDATA" 170 #define STARTUP_HOOK_SECTION "__init_entry_set" 171 #else 172 /* arm protects __KLD early, so use __BOOTDATA for data */ 173 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 174 #define STARTUP_CONST_SEGSECT "__KLDDATA,__const" 175 #define STARTUP_DATA_SEGSECT "__BOOTDATA,__init" 176 #define STARTUP_HOOK_SEGMENT "__BOOTDATA" 177 #define STARTUP_HOOK_SECTION "__init_entry_set" 178 #endif 179 180 /*! 181 * @macro __startup_func 182 * 183 * @abstract 184 * Attribute to place on functions used only during the kernel startup phase. 185 * 186 * @description 187 * Code marked with this attribute will be unmapped after kernel lockdown. 188 */ 189 #ifndef __BUILDING_XNU_LIBRARY__ 190 #define __startup_func \ 191 __PLACE_IN_SECTION(STARTUP_CODE_SEGSECT) \ 192 __attribute__((cold, visibility("hidden"))) 193 #else 194 /* tester needs some startup function to be visible from outside the XNU library */ 195 #define __startup_func __unused 196 #endif 197 /*! 198 * @macro __startup_data 199 * 200 * @abstract 201 * Attribute to place on globals used during the kernel startup phase. 202 * 203 * @description 204 * Data marked with this attribute will be unmapped after kernel lockdown. 205 */ 206 #ifndef __BUILDING_XNU_LIBRARY__ 207 #define __startup_data \ 208 __PLACE_IN_SECTION(STARTUP_DATA_SEGSECT) 209 #else 210 #define __startup_data 211 #endif 212 213 /*! 214 * @macro __startup_const 215 * 216 * @abstract 217 * Attribute to place on global constants used during the kernel startup phase. 218 * 219 * @description 220 * Data marked with this attribute will be unmapped after kernel lockdown. 221 * 222 * __startup_const implies const. Be mindful that for pointers, the `const` 223 * will end up on the wrong side of the star if you put __startup_const at the 224 * start of the declaration. 225 */ 226 #define __startup_const \ 227 __PLACE_IN_SECTION(STARTUP_CONST_SEGSECT) const 228 229 /*! 230 * @macro STARTUP 231 * 232 * @abstract 233 * Declares a kernel startup callback. 234 */ 235 #define STARTUP(subsystem, rank, func) \ 236 __STARTUP(func, __LINE__, subsystem, rank, func) 237 238 /*! 239 * @macro STARTUP_ARG 240 * 241 * @abstract 242 * Declares a kernel startup callback that takes an argument. 243 */ 244 #define STARTUP_ARG(subsystem, rank, func, arg) \ 245 __STARTUP_ARG(func, __LINE__, subsystem, rank, func, arg) 246 247 /*! 248 * @macro TUNABLE 249 * 250 * @abstract 251 * Declares a read-only kernel tunable that is read from a boot-arg with 252 * a default value, without further processing. 253 * 254 * @param type_t 255 * Should be an integer type or bool. 256 * 257 * @param var 258 * The name of the C variable to use for storage. 259 * 260 * @param boot_arg 261 * The name of the boot-arg to parse for initialization 262 * 263 * @param default_value 264 * The default value for the tunable if the boot-arg is absent. 265 */ 266 #define TUNABLE(type_t, var, boot_arg, default_value) \ 267 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \ 268 __TUNABLE(type_t, var, boot_arg) 269 270 /*! 271 * @macro TUNABLE_WRITEABLE 272 * 273 * @abstract 274 * Declares a writeable kernel tunable that is read from a boot-arg with 275 * a default value, without further processing. 276 * 277 * @param type_t 278 * Should be an integer type or bool. 279 * 280 * @param var 281 * The name of the C variable to use for storage. 282 * 283 * @param boot_arg 284 * The name of the boot-arg to parse for initialization 285 * 286 * @param default_value 287 * The default value for the tunable if the boot-arg is absent. 288 */ 289 #define TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) \ 290 type_t var = default_value; \ 291 __TUNABLE(type_t, var, boot_arg) 292 293 #if DEBUG || DEVELOPMENT 294 #define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \ 295 TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) 296 #else 297 #define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \ 298 TUNABLE(type_t, var, boot_arg, default_value) 299 #endif 300 301 /*! 302 * @macro TUNABLE_STR 303 * 304 * @abstract 305 * Declares a read-only kernel tunable that is read from a boot-arg with 306 * a default value, without further processing. 307 * 308 * @param var 309 * The name of the C variable to use for storage. 310 * 311 * @param count 312 * The number of bytes in the buffer. 313 * 314 * @param boot_arg 315 * The name of the boot-arg to parse for initialization 316 * 317 * @param default_value 318 * The default value for the tunable if the boot-arg is absent. 319 */ 320 #define TUNABLE_STR(var, count, boot_arg, default_value) \ 321 char __security_const_late var[count] = default_value; \ 322 __TUNABLE_STR(var, boot_arg) 323 324 /*! 325 * @enum tunable_dt_flags_t 326 * 327 * @abstract 328 * Flags used with the @c TUNABLE_DT* macros. 329 * 330 * @description 331 * If TUNABLE_DT_CHECK_CHOSEN is set, a value in 332 * /chosen/<dt_base>/<dt_name> takes precedence over any value in 333 * /<dt_base>/<dt_name>. /chosen is by convention the area where 334 * synthesized values not coming from the serialized device tree are 335 * being added, so this provides a way for e.g. the boot-loader to 336 * set/override tunables. Don't override with a boot-arg if 337 * TUNABLE_DT_NO_BOOTARG is set. 338 */ 339 __options_decl(tunable_dt_flags_t, uint32_t, { 340 TUNABLE_DT_NONE = 0x00000000, 341 TUNABLE_DT_CHECK_CHOSEN = 0x00000001, 342 TUNABLE_DT_NO_BOOTARG = 0x00000002, 343 }); 344 345 /*! 346 * @macro TUNABLE_DT 347 * 348 * @abstract 349 * Like TUNABLE, but gets the initial value from both Device Tree and 350 * boot-args. The order in which the initial value is resolved is as 351 * follows, with later steps overriding previous ones (if they are 352 * specified): 353 * 354 * 1. Device Tree Entry "/<dt_base>/<dt_name>", 355 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry 356 * "/chosen/<dt_base>/<dt_name>" (see the description for 357 * @c tunable_dt_flags_t), 358 * 3. boot-args. 359 * 360 * @param type_t 361 * Should be an integer type or bool. 362 * 363 * @param var 364 * The name of the C variable to use for storage. 365 * 366 * @param dt_base 367 * The name of the DT node containing the property. 368 * 369 * @param dt_name 370 * The name of the DT property containing the default value. 371 * 372 * @param boot_arg 373 * The name of the boot-arg overriding the initial value from the DT. 374 * 375 * @param default_value 376 * The default value for the tunable if both DT entry and boot-arg are 377 * absent. 378 * 379 * @param flags 380 * See the description for @c tunable_dt_flags_t. 381 */ 382 #define TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 383 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \ 384 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags) 385 386 /*! 387 * @macro TUNABLE_DT_WRITEABLE 388 * 389 * @abstract 390 * Like TUNABLE_WRITEABLE, but gets the initial value from both Device 391 * Tree and boot-args. The order in which the initial value is 392 * resolved is as follows, with later steps overriding previous ones 393 * (if they are specified): 394 * 395 * 1. Device Tree Entry "/<dt_base>/<dt_name>", 396 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry 397 * "/chosen/<dt_base>/<dt_name>" (see the description for 398 * @c tunable_dt_flags_t), 399 * 3. boot-args. 400 * 401 * @param type_t 402 * Should be an integer type or bool. 403 * 404 * @param var 405 * The name of the C variable to use for storage. 406 * 407 * @param dt_base 408 * The name of the DT node containing the property. 409 * 410 * @param dt_name 411 * The name of the DT property containing the default value. 412 * 413 * @param boot_arg 414 * The name of the boot-arg overriding the initial value from the DT. 415 * 416 * @param default_value 417 * The default value for the tunable if both DT entry and boot-arg are 418 * absent. 419 * 420 * @param flags 421 * See the description for @c tunable_dt_flags_t. 422 */ 423 #define TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 424 type_t var = default_value; \ 425 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags) 426 427 #if DEBUG || DEVELOPMENT 428 #define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 429 TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) 430 #else 431 #define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 432 TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) 433 #endif 434 435 /*! 436 * @macro TUNABLE_DT_WRITEABLE_SOURCE 437 * 438 * @abstract 439 * Like TUNABLE_DT_WRITEABLE, but records the source from which the value was resolved. 440 * 441 * @param type_t 442 * Should be an integer type or bool. 443 * 444 * @param var 445 * The name of the C variable to use for storage. 446 * 447 * @param source_var 448 * The name of the C variable to record the source. It will be of type startup_source_t. 449 * 450 * @param dt_base 451 * The name of the DT node containing the property. 452 * 453 * @param dt_name 454 * The name of the DT property containing the default value. 455 * 456 * @param boot_arg 457 * The name of the boot-arg overriding the initial value from the DT. 458 * 459 * @param default_value 460 * The default value for the tunable if both DT entry and boot-arg are 461 * absent. 462 * 463 * @param flags 464 * See the description for @c tunable_dt_flags_t. 465 */ 466 #define TUNABLE_DT_WRITEABLE_SOURCE(type_t, var, source_var, dt_base, dt_name, boot_arg, default_value, flags) \ 467 type_t var = default_value; \ 468 startup_source_t source_var; \ 469 __TUNABLE_DT_SOURCE(type_t, var, source_var, dt_base, dt_name, boot_arg, flags) 470 471 /* 472 * Machine Timeouts 473 * 474 * Machine Timeouts are timeouts for low level kernel code manifesting 475 * as _Atomic uint64_t variables, whose default value can be 476 * overridden and scaled via the device tree and boot-args. 477 * 478 * Each timeout has a name, looked up directly as the property name in 479 * the device tree in both the "/machine-timeouts" and 480 * "/chosen/machine-timeouts" nodes. The "chosen" property always 481 * overrides the other one. This allows fixed per-device timeouts in 482 * the device tree to be overridden by iBoot in "chosen". 483 * 484 * Additionally, the same name with "-scale" appended is looked up as 485 * properties for optional scale factors. Scale factors are not 486 * overridden by chosen, instead all scale factors (including global 487 * and/or boot-arg scale factors) combine by multiplication. 488 * 489 * The special name "global-scale" provides a scale that applies to 490 * every timeout. 491 * 492 * All property names can be used as boot-args by prefixing 493 * "ml-timeout-", e.g. th global scale is available as the 494 * "ml-timeout-global-scale" boot-arg. 495 * 496 * By convention, if the timeout value resolves to 0, the timeout 497 * should be disabled. 498 */ 499 500 /* 501 * Machine Timeouts types. See the next section for what unit 502 * they are in. 503 * 504 * We use _Atomic, but only with relaxed ordering: This is just to 505 * make sure all devices see consistent values all the time. Since 506 * the actual timeout value will be seen as 0 before initializaton, 507 * relaxed ordering means that code that runs concurrently with 508 * initialization only risks to see a disabled timeout during early 509 * boot. 510 */ 511 typedef _Atomic uint64_t machine_timeout_t; 512 513 /* 514 * Units 515 * 516 * Machine Timeouts are ALWAYS in picoseconds in the device tree or 517 * boot-args, to avoid confusion when changing or comparing timeouts 518 * as a user, but the actual storage value might contain the same 519 * duration in another unit, calculated by the initialization code. 520 * 521 * This is done because otherwise we would likely introduce another 522 * multiplication in potentially hot code paths, given that code that 523 * actually uses the timeout storage variable is unlikely to work with 524 * picosecond values when comparing against the timeout deadline. 525 * 526 * This unit scale is *only* applied during initialization at early 527 * boot, and only if the timeout's default value was overridden 528 * through the device tree or a boot-arg. 529 */ 530 #define MACHINE_TIMEOUT_UNIT_PSEC 1 531 #define MACHINE_TIMEOUT_UNIT_NSEC 1000 532 #define MACHINE_TIMEOUT_UNIT_USEC (1000*1000) 533 #define MACHINE_TIMEOUT_UNIT_MSEC (1000*1000*1000) 534 // Special unit for timebase ticks (usually 1/24MHz) 535 #define MACHINE_TIMEOUT_UNIT_TIMEBASE 0 536 537 // DT property names are limited to 31 chars, minus "-global" suffix 538 #define MACHINE_TIMEOUT_MAX_NAME_LEN 25 539 struct machine_timeout_spec { 540 void *ptr; 541 uint64_t default_value; 542 uint64_t unit_scale; 543 char name[MACHINE_TIMEOUT_MAX_NAME_LEN + 1]; 544 bool (*skip_predicate)(struct machine_timeout_spec const *); 545 }; 546 547 extern void 548 machine_timeout_init_with_suffix(const struct machine_timeout_spec *spec, char const *phase_suffix, bool always_enabled); 549 550 extern void 551 machine_timeout_init(const struct machine_timeout_spec *spec); 552 553 extern void 554 machine_timeout_init_always_enabled(const struct machine_timeout_spec *spec); 555 556 #if DEVELOPMENT || DEBUG 557 // Late timeout (re-)initialization, at the end of bsd_init() 558 extern void 559 machine_timeout_bsd_init(void); 560 #endif /* DEVELOPMENT || DEBUG */ 561 562 /*! 563 * @macro MACHINE_TIMEOUT, MACHINE_TIMEOUT_ALWAYS_ENABLED, and MACHINE_TIMEOUT_DEV_WRITEABLE 564 * 565 * @abstract 566 * Defines a Machine Timeout that can be overridden and 567 * scaled through the device tree and boot-args. 568 * 569 * The variant with the _DEV_WRITEABLE suffix does not mark the timeout as 570 * SECURITY_READ_ONLY_LATE on DEVELOPMENT kernels, so that e.g. 571 * machine_timeout_init_with_suffix or sysctls can change it after lockdown. 572 * 573 * @param var 574 * The name of the C variable to use for storage. If the storage value 575 * contains 0, the timeout is considered disabled by convention. 576 * 577 * @param timeout_name 578 * The name of the timeout, used for property and boot-arg names. See 579 * the general description of Machine Timeouts above for how this name 580 * ends up being used. 581 * 582 * @param timeout_default 583 * The default value for the timeout if not specified through device 584 * tree or boot-arg. Will still be scaled if a scale factor exists. 585 * 586 * @param var_unit 587 * The unit that the storage variable is in. Note that timeout values 588 * must always be specified as picoseconds in the device tree and 589 * boot-args, but timeout initialization will convert the value to the 590 * unit specified here before writing it to the storage variable. 591 * 592 * @param skip_predicate 593 * Optionally, a function to call to decide whether the timeout should 594 * be set or not. If NULL, the timeout will always be set (if 595 * specified anywhere). A predicate has the following signature: 596 * bool skip_predicate (struct machine_timeout_spec const *) 597 */ 598 599 #define _MACHINE_TIMEOUT(var, timeout_name, timeout_default, var_unit, skip_pred, init_fn) \ 600 struct machine_timeout_spec \ 601 __machine_timeout_spec_ ## var = { \ 602 .ptr = &var, \ 603 .default_value = timeout_default, \ 604 .unit_scale = var_unit, \ 605 .name = timeout_name, \ 606 .skip_predicate = skip_pred, \ 607 }; \ 608 __STARTUP_ARG(var, __LINE__, TIMEOUTS, STARTUP_RANK_FIRST, \ 609 init_fn, &__machine_timeout_spec_ ## var) 610 611 #define MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) \ 612 SECURITY_READ_ONLY_LATE(machine_timeout_t) var = 0; \ 613 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate, machine_timeout_init) 614 615 /* 616 * Variant of MACHINE_TIMEOUT that does not get zeroed if wdt == -1 boot arg is set 617 */ 618 #define MACHINE_TIMEOUT_ALWAYS_ENABLED(var, name, default, unit) \ 619 SECURITY_READ_ONLY_LATE(machine_timeout_t) var = 0; \ 620 _MACHINE_TIMEOUT(var, name, default, unit, NULL, machine_timeout_init_always_enabled) 621 622 #if DEVELOPMENT || DEBUG 623 #define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \ 624 machine_timeout_t var = 0; \ 625 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate, machine_timeout_init) 626 #else 627 #define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \ 628 MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 629 #endif /* DEVELOPMENT || DEBUG */ 630 631 /*! 632 * @macro MACHINE_TIMEOUT_SPEC_REF 633 * 634 * @abstract 635 * References a previously defined MACHINE_TIMEOUT. 636 * 637 * This is primarily useful for overriding individual timeouts 638 * at arbitrary times (even after boot), by manually calling 639 * machine_timeout_init_with_suffix() with this macro 640 * as first argument, and a suffix to apply to both device tree and 641 * boot-arg as second argument. 642 * 643 * @param var 644 * The name of the C variable used for storage, as it was specified 645 * in MACHINE_TIMEOUT. 646 */ 647 #define MACHINE_TIMEOUT_SPEC_REF(var) (&__machine_timeout_spec_ ## var) 648 649 /*! 650 * @macro MACHINE_TIMEOUT_SPEC_DECL 651 * 652 * @abstract 653 * Declaration of machine timeout spec, mostly useful to make it known 654 * for MACHINE_TIMEOUT_SPEC_REF. 655 * 656 * @param var 657 * The name of the C variable used for storage, as it was specified 658 * in MACHINE_TIMEOUT. 659 */ 660 #define MACHINE_TIMEOUT_SPEC_DECL(var) extern struct machine_timeout_spec __machine_timeout_spec_ ## var 661 662 /* 663 * Event subsystem 664 * 665 * This allows to define a few system-wide events that allow for loose coupling 666 * between subsystems interested in those events and the emitter. 667 */ 668 669 /*! 670 * @macro EVENT_DECLARE() 671 * 672 * @brief 673 * Declares an event namespace with a given callback type. 674 * 675 * @param name The name for the event (typically in all caps). 676 * @param cb_type_t A function type for the callbacks. 677 */ 678 #define EVENT_DECLARE(name, cb_type_t) \ 679 struct name##_event { \ 680 struct event_hdr evt_link; \ 681 cb_type_t *evt_cb; \ 682 }; \ 683 extern struct event_hdr name##_HEAD 684 685 /*! 686 * @macro EVENT_DEFINE() 687 * 688 * @brief 689 * Defines the head for the event corresponding to an EVENT_DECLARE() 690 */ 691 #define EVENT_DEFINE(name) \ 692 __security_const_late struct event_hdr name##_HEAD 693 694 /*! 695 * @macro EVENT_REGISTER_HANDLER() 696 * 697 * @brief 698 * Registers a handler for a given event. 699 * 700 * @param name The name for the event as declared in EVENT_DECLARE() 701 * @param handler The handler to register for this event. 702 */ 703 #define EVENT_REGISTER_HANDLER(name, handler) \ 704 __EVENT_REGISTER(name, __LINE__, handler) 705 706 707 /*! 708 * @macro EVENT_INVOKE() 709 * 710 * @brief 711 * Call all the events handlers with the specified arguments. 712 * 713 * @param name The name for the event as declared in EVENT_DECLARE() 714 * @param handler The handler to register for this event. 715 */ 716 #define EVENT_INVOKE(name, ...) \ 717 for (struct event_hdr *__e = &name##_HEAD; (__e = __e->next);) { \ 718 __container_of(__e, struct name##_event, \ 719 evt_link)->evt_cb(__VA_ARGS__); \ 720 } 721 722 723 #if DEBUG || DEVELOPMENT 724 725 /*! 726 * @macro SYSCTL_TEST_REGISTER 727 * 728 * @abstract 729 * Declares a test that will appear under @c debug.test.${name}. 730 * 731 * @param name 732 * An identifier that will be stringified to form the sysctl test name. 733 * 734 * @param cb 735 * The callback to run, of type: 736 * <code> 737 * int (callback *)(int64_t value, int64_t *); 738 * </code> 739 */ 740 #define SYSCTL_TEST_REGISTER(name, cb) \ 741 static __startup_data struct sysctl_test_setup_spec \ 742 __startup_SYSCTL_TEST_ ## name = { \ 743 .st_name = #name, \ 744 .st_func = &cb, \ 745 }; \ 746 STARTUP_ARG(SYSCTL, STARTUP_RANK_MIDDLE, \ 747 sysctl_register_test_startup, &__startup_SYSCTL_TEST_ ## name) 748 749 #endif /* DEBUG || DEVELOPMENT */ 750 #pragma mark - internals 751 752 __END_DECLS 753 754 #ifdef __cplusplus 755 template <typename T> 756 struct __startup_tunable { 757 static const bool value = false; 758 }; 759 760 template <> 761 struct __startup_tunable <bool>{ 762 static const bool value = true; 763 }; 764 #define __startup_type_is_bool(type_t) __startup_tunable<type_t>::value 765 #else 766 #define __startup_type_is_bool(type_t) __builtin_types_compatible_p(bool, type_t) 767 #endif 768 769 __BEGIN_DECLS 770 771 #define __TUNABLE(type_t, var, key) \ 772 static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \ 773 static __startup_const struct startup_tunable_spec \ 774 __startup_TUNABLES_spec_ ## var = { \ 775 .name = __startup_TUNABLES_name_ ## var, \ 776 .var_addr = (void *)&var, \ 777 .var_len = sizeof(type_t), \ 778 .var_is_bool = __startup_type_is_bool(type_t), \ 779 }; \ 780 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 781 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var) 782 783 #define __TUNABLE_STR(var, key) \ 784 static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \ 785 static __startup_const struct startup_tunable_spec \ 786 __startup_TUNABLES_spec_ ## var = { \ 787 .name = __startup_TUNABLES_name_ ## var, \ 788 .var_addr = (void *)&var, \ 789 .var_len = sizeof(var), \ 790 .var_is_str = true, \ 791 }; \ 792 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 793 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var) 794 795 #define __TUNABLE_DT(type_t, var, dt_base_key, dt_name_key, boot_arg_key, flags) \ 796 static __startup_const char __startup_TUNABLES_dt_base_ ## var[] = dt_base_key; \ 797 static __startup_const char __startup_TUNABLES_dt_name_ ## var[] = dt_name_key; \ 798 static __startup_const char __startup_TUNABLES_name_ ## var[] = boot_arg_key; \ 799 static __startup_const struct startup_tunable_dt_spec \ 800 __startup_TUNABLES_DT_spec_ ## var = { \ 801 .dt_base = __startup_TUNABLES_dt_base_ ## var, \ 802 .dt_name = __startup_TUNABLES_dt_name_ ## var, \ 803 .dt_chosen_override = (bool)((flags) & TUNABLE_DT_CHECK_CHOSEN), \ 804 .boot_arg_name = (flags & TUNABLE_DT_NO_BOOTARG) ? NULL : __startup_TUNABLES_name_ ## var, \ 805 .var_addr = (void *)&var, \ 806 .var_len = sizeof(type_t), \ 807 .var_is_bool = __startup_type_is_bool(type_t), \ 808 }; \ 809 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 810 kernel_startup_tunable_dt_init, &__startup_TUNABLES_DT_spec_ ## var) 811 812 #define __TUNABLE_DT_SOURCE(type_t, var, source_var, dt_base_key, dt_name_key, boot_arg_key, flags) \ 813 static __startup_const char __startup_TUNABLES_dt_base_ ## var[] = dt_base_key; \ 814 static __startup_const char __startup_TUNABLES_dt_name_ ## var[] = dt_name_key; \ 815 static __startup_const char __startup_TUNABLES_name_ ## var[] = boot_arg_key; \ 816 static __startup_const struct startup_tunable_dt_source_spec \ 817 __startup_TUNABLES_DT_spec_ ## var = { \ 818 .dt_base = __startup_TUNABLES_dt_base_ ## var, \ 819 .dt_name = __startup_TUNABLES_dt_name_ ## var, \ 820 .dt_chosen_override = (bool)((flags) & TUNABLE_DT_CHECK_CHOSEN), \ 821 .boot_arg_name = (flags & TUNABLE_DT_NO_BOOTARG) ? NULL : __startup_TUNABLES_name_ ## var, \ 822 .var_addr = (void *)&var, \ 823 .var_len = sizeof(type_t), \ 824 .var_is_bool = __startup_type_is_bool(type_t), \ 825 .source_addr = &source_var, \ 826 }; \ 827 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 828 kernel_startup_tunable_dt_source_init, &__startup_TUNABLES_DT_spec_ ## var) 829 830 #ifdef __cplusplus 831 #define __STARTUP_FUNC_CAST(func, a) \ 832 (void(*)(const void *))func 833 #else 834 #define __STARTUP_FUNC_CAST(func, a) \ 835 (typeof(func(a))(*)(const void *))func 836 #endif 837 838 839 #define __STARTUP1(name, line, subsystem, rank, func, a, b) \ 840 __PLACE_IN_SECTION(STARTUP_HOOK_SEGMENT "," STARTUP_HOOK_SECTION) \ 841 static const struct startup_entry \ 842 __startup_ ## subsystem ## _entry_ ## name ## _ ## line = { \ 843 STARTUP_SUB_ ## subsystem, \ 844 rank, __STARTUP_FUNC_CAST(func, a), b, \ 845 } 846 847 #define __STARTUP(name, line, subsystem, rank, func) \ 848 __STARTUP1(name, line, subsystem, rank, func, , NULL) 849 850 #define __STARTUP_ARG(name, line, subsystem, rank, func, arg) \ 851 __STARTUP1(name, line, subsystem, rank, func, arg, arg) 852 853 struct startup_entry { 854 startup_subsystem_id_t subsystem; 855 startup_rank_t rank; 856 void (*func)(const void *); 857 const void *arg; 858 }; 859 860 struct startup_tunable_spec { 861 const char *name; 862 void *var_addr; 863 int var_len; 864 bool var_is_bool; 865 bool var_is_str; 866 }; 867 868 struct startup_tunable_dt_spec { 869 const char *dt_base; 870 const char *dt_name; 871 bool dt_chosen_override; 872 const char *boot_arg_name; 873 void *var_addr; 874 int var_len; 875 bool var_is_bool; 876 }; 877 878 struct startup_tunable_dt_source_spec { 879 const char *dt_base; 880 const char *dt_name; 881 bool dt_chosen_override; 882 const char *boot_arg_name; 883 void *var_addr; 884 int var_len; 885 bool var_is_bool; 886 startup_source_t *source_addr; 887 }; 888 889 #if DEBUG || DEVELOPMENT 890 struct sysctl_test_setup_spec { 891 const char *st_name; 892 int (*st_func)(int64_t, int64_t *); 893 }; 894 895 extern void sysctl_register_test_startup( 896 struct sysctl_test_setup_spec *spec); 897 #endif /* DEBUG || DEVELOPMENT */ 898 899 /* 900 * Kernel and machine startup declarations 901 */ 902 903 /* Initialize kernel */ 904 extern void kernel_startup_bootstrap(void); 905 extern void kernel_startup_initialize_upto(startup_subsystem_id_t upto); 906 extern void kernel_startup_tunable_init(const struct startup_tunable_spec *); 907 extern void kernel_startup_tunable_dt_init(const struct startup_tunable_dt_spec *); 908 extern void kernel_startup_tunable_dt_source_init(const struct startup_tunable_dt_source_spec *); 909 extern void kernel_bootstrap(void); 910 911 #ifdef __BUILDING_XNU_LIB_UNITTEST__ 912 void kernel_startup_initialize_only(startup_subsystem_id_t sysid); 913 #endif 914 915 /* Initialize machine dependent stuff */ 916 extern void machine_init(void); 917 918 extern void secondary_cpu_main(void *machine_param); 919 920 /* Secondary cpu initialization */ 921 extern void machine_cpu_reinit(void *machine_param); 922 extern void processor_cpu_reinit(void* machine_param, bool wait_for_cpu_signal, bool is_final_system_sleep); 923 924 /* Device subsystem initialization */ 925 extern void device_service_create(void); 926 927 struct event_hdr { 928 struct event_hdr *next; 929 }; 930 931 extern void event_register_handler(struct event_hdr *event_hdr); 932 933 #define __EVENT_REGISTER(name, lno, handler) \ 934 static __security_const_late struct name##_event name##_event_##lno = { \ 935 .evt_link.next = &name##_HEAD, \ 936 .evt_cb = (handler), \ 937 }; \ 938 __STARTUP_ARG(name, lno, EVENT, STARTUP_RANK_FIRST, \ 939 event_register_handler, &name##_event_##lno.evt_link) 940 941 #ifdef MACH_BSD 942 943 /* BSD subsystem initialization */ 944 extern void bsd_init(void); 945 946 extern int serverperfmode; 947 948 #if defined(XNU_TARGET_OS_OSX) 949 static inline bool 950 kernel_is_macos_or_server(void) 951 { 952 return true; 953 } 954 #else /* XNU_TARGET_OS_OSX */ 955 static inline bool 956 kernel_is_macos_or_server(void) 957 { 958 return serverperfmode != 0; 959 } 960 #endif /* XNU_TARGET_OS_OSX */ 961 962 #endif /* MACH_BSD */ 963 964 __exported_pop 965 966 __END_DECLS 967 968 #endif /* _KERN_STARTUP_H_ */ 969 970 #endif /* XNU_KERNEL_PRIVATE */ 971