1 /* 2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 32 #ifdef XNU_KERNEL_PRIVATE 33 34 #ifndef _KERN_STARTUP_H_ 35 #define _KERN_STARTUP_H_ 36 37 #include <stdbool.h> 38 #include <stdint.h> 39 40 #include <libkern/section_keywords.h> 41 42 __BEGIN_DECLS 43 44 #pragma GCC visibility push(hidden) 45 46 /*! 47 * @enum startup_subsystem_id_t 48 * 49 * @abstract 50 * Represents a stage of kernel intialization, ubnd allows for subsystems 51 * to register initializers for a specific stage. 52 * 53 * @discussion 54 * Documentation of each subsystem initialization sequence exists in 55 * @file doc/startup.md. 56 */ 57 __enum_decl(startup_subsystem_id_t, uint32_t, { 58 STARTUP_SUB_NONE = 0, /**< reserved for the startup subsystem */ 59 60 STARTUP_SUB_TUNABLES, /**< support for the tunables subsystem */ 61 STARTUP_SUB_TIMEOUTS, /**< configurable machine timeouts */ 62 STARTUP_SUB_LOCKS, /**< various subsystem locks */ 63 STARTUP_SUB_KPRINTF, /**< kprintf initialization */ 64 65 STARTUP_SUB_PMAP_STEAL, /**< to perform various pmap carveouts */ 66 STARTUP_SUB_KMEM, /**< once kmem_alloc is ready */ 67 STARTUP_SUB_ZALLOC, /**< initialize zalloc and kalloc */ 68 STARTUP_SUB_KTRACE, /**< initialize kernel trace */ 69 STARTUP_SUB_PERCPU, /**< initialize the percpu subsystem */ 70 STARTUP_SUB_EVENT, /**< initiailze the event subsystem */ 71 72 STARTUP_SUB_CODESIGNING, /**< codesigning subsystem */ 73 STARTUP_SUB_OSLOG, /**< oslog and kernel logging */ 74 STARTUP_SUB_MACH_IPC, /**< Mach IPC */ 75 STARTUP_SUB_THREAD_CALL, /**< Thread calls */ 76 STARTUP_SUB_SYSCTL, /**< registers sysctls */ 77 STARTUP_SUB_EARLY_BOOT, /**< interrupts/preemption are turned on */ 78 79 STARTUP_SUB_LOCKDOWN = ~0u, /**< reserved for the startup subsystem */ 80 }); 81 82 /*! 83 * Stores the last subsystem to have been fully initialized; 84 */ 85 extern startup_subsystem_id_t startup_phase; 86 87 /*! 88 * @enum startup_debug_t 89 * 90 * @abstract 91 * Flags set in the @c startup_debug global to configure startup debugging. 92 */ 93 __options_decl(startup_debug_t, uint32_t, { 94 STARTUP_DEBUG_NONE = 0x00000000, 95 STARTUP_DEBUG_VERBOSE = 0x00000001, 96 }); 97 98 extern startup_debug_t startup_debug; 99 100 /*! 101 * @enum startup_rank 102 * 103 * @abstract 104 * Specifies in which rank a given initializer runs within a given section 105 * to register initializers for a specific rank within the subsystem. 106 * 107 * @description 108 * A startup function, declared with @c STARTUP or @c STARTUP_ARG, can specify 109 * a rank within the subsystem they initialize. 110 * 111 * @c STARTUP_RANK_NTH(n) will let callbacks be run at stage @c n (0-based). 112 * 113 * @c STARTUP_RANK_FIRST, @c STARTUP_RANK_SECOND, @c STARTUP_RANK_THIRD and 114 * @c STARTUP_RANK_FOURTH are given as convenient names for these. 115 * 116 * @c STARTUP_RANK_MIDDLE is a reserved value that will let startup functions 117 * run after all the @c STARTUP_RANK_NTH(n) ones have. 118 * 119 * @c STARTUP_RANK_NTH_LATE_NTH(n) will let callbacks be run then in @c n rank 120 * after the @c STARTUP_RANK_MIDDLE ones (0-based). 121 * 122 * @c STARTUP_RANK_LAST callbacks will run absolutely last after everything 123 * else did for this subsystem. 124 */ 125 __enum_decl(startup_rank_t, uint32_t, { 126 #define STARTUP_RANK_NTH(n) ((startup_rank_t)(n - 1)) 127 STARTUP_RANK_FIRST = 0, 128 STARTUP_RANK_SECOND = 1, 129 STARTUP_RANK_THIRD = 2, 130 STARTUP_RANK_FOURTH = 3, 131 132 STARTUP_RANK_MIDDLE = 0x7fffffff, 133 134 #define STARTUP_RANK_LATE_NTH(n) \ 135 ((startup_rank_t)(STARTUP_RANK_MIDDLE + 1 + (n))) 136 137 STARTUP_RANK_LAST = 0xffffffff, 138 }); 139 140 #if KASAN 141 /* 142 * The use of weird sections that get unmapped confuse the hell out of kasan, 143 * so for KASAN leave things in regular __TEXT/__DATA segments 144 */ 145 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 146 #define STARTUP_DATA_SEGSECT "__DATA,__init" 147 #define STARTUP_HOOK_SEGMENT "__DATA" 148 #define STARTUP_HOOK_SECTION "__init_entry_set" 149 #elif defined(__x86_64__) 150 /* Intel doesn't have a __BOOTDATA but doesn't protect __KLD */ 151 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 152 #define STARTUP_DATA_SEGSECT "__KLDDATA,__init" 153 #define STARTUP_HOOK_SEGMENT "__KLDDATA" 154 #define STARTUP_HOOK_SECTION "__init_entry_set" 155 #else 156 /* arm protects __KLD early, so use __BOOTDATA for data */ 157 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 158 #define STARTUP_DATA_SEGSECT "__BOOTDATA,__init" 159 #define STARTUP_HOOK_SEGMENT "__BOOTDATA" 160 #define STARTUP_HOOK_SECTION "__init_entry_set" 161 #endif 162 163 /*! 164 * @macro __startup_func 165 * 166 * @abstract 167 * Attribute to place on functions used only during the kernel startup phase. 168 * 169 * @description 170 * Code marked with this attribute will be unmapped after kernel lockdown. 171 */ 172 #define __startup_func \ 173 __PLACE_IN_SECTION(STARTUP_CODE_SEGSECT) \ 174 __attribute__((cold, visibility("hidden"))) 175 176 /*! 177 * @macro __startup_data 178 * 179 * @abstract 180 * Attribute to place on globals used during the kernel startup phase. 181 * 182 * @description 183 * Data marked with this attribute will be unmapped after kernel lockdown. 184 */ 185 #define __startup_data \ 186 __PLACE_IN_SECTION(STARTUP_DATA_SEGSECT) 187 188 /*! 189 * @macro STARTUP 190 * 191 * @abstract 192 * Declares a kernel startup callback. 193 */ 194 #define STARTUP(subsystem, rank, func) \ 195 __STARTUP(func, __LINE__, subsystem, rank, func) 196 197 /*! 198 * @macro STARTUP_ARG 199 * 200 * @abstract 201 * Declares a kernel startup callback that takes an argument. 202 */ 203 #define STARTUP_ARG(subsystem, rank, func, arg) \ 204 __STARTUP_ARG(func, __LINE__, subsystem, rank, func, arg) 205 206 /*! 207 * @macro TUNABLE 208 * 209 * @abstract 210 * Declares a read-only kernel tunable that is read from a boot-arg with 211 * a default value, without further processing. 212 * 213 * @param type_t 214 * Should be an integer type or bool. 215 * 216 * @param var 217 * The name of the C variable to use for storage. 218 * 219 * @param boot_arg 220 * The name of the boot-arg to parse for initialization 221 * 222 * @param default_value 223 * The default value for the tunable if the boot-arg is absent. 224 */ 225 #define TUNABLE(type_t, var, boot_arg, default_value) \ 226 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \ 227 __TUNABLE(type_t, var, boot_arg) 228 229 /*! 230 * @macro TUNABLE_WRITEABLE 231 * 232 * @abstract 233 * Declares a writeable kernel tunable that is read from a boot-arg with 234 * a default value, without further processing. 235 * 236 * @param type_t 237 * Should be an integer type or bool. 238 * 239 * @param var 240 * The name of the C variable to use for storage. 241 * 242 * @param boot_arg 243 * The name of the boot-arg to parse for initialization 244 * 245 * @param default_value 246 * The default value for the tunable if the boot-arg is absent. 247 */ 248 #define TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) \ 249 type_t var = default_value; \ 250 __TUNABLE(type_t, var, boot_arg) 251 252 #if DEBUG || DEVELOPMENT 253 #define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \ 254 TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) 255 #else 256 #define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \ 257 TUNABLE(type_t, var, boot_arg, default_value) 258 #endif 259 260 /*! 261 * @macro TUNABLE_STR 262 * 263 * @abstract 264 * Declares a read-only kernel tunable that is read from a boot-arg with 265 * a default value, without further processing. 266 * 267 * @param var 268 * The name of the C variable to use for storage. 269 * 270 * @param count 271 * The number of bytes in the buffer. 272 * 273 * @param boot_arg 274 * The name of the boot-arg to parse for initialization 275 * 276 * @param default_value 277 * The default value for the tunable if the boot-arg is absent. 278 */ 279 #define TUNABLE_STR(var, count, boot_arg, default_value) \ 280 char __security_const_late var[count] = default_value; \ 281 __TUNABLE_STR(var, boot_arg) 282 283 /*! 284 * @enum tunable_dt_flags_t 285 * 286 * @abstract 287 * Flags used with the @c TUNABLE_DT* macros. 288 * 289 * @description 290 * If TUNABLE_DT_CHECK_CHOSEN is set, a value in 291 * /chosen/<dt_base>/<dt_name> takes precedence over any value in 292 * /<dt_base>/<dt_name>. /chosen is by convention the area where 293 * synthesized values not coming from the serialized device tree are 294 * being added, so this provides a way for e.g. the boot-loader to 295 * set/override tunables. 296 */ 297 __options_decl(tunable_dt_flags_t, uint32_t, { 298 TUNABLE_DT_NONE = 0x00000000, 299 TUNABLE_DT_CHECK_CHOSEN = 0x00000001, 300 }); 301 302 /*! 303 * @macro TUNABLE_DT 304 * 305 * @abstract 306 * Like TUNABLE, but gets the initial value from both Device Tree and 307 * boot-args. The order in which the initial value is resolved is as 308 * follows, with later steps overriding previous ones (if they are 309 * specified): 310 * 311 * 1. Device Tree Entry "/<dt_base>/<dt_name>", 312 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry 313 * "/chosen/<dt_base>/<dt_name>" (see the description for 314 * @c tunable_dt_flags_t), 315 * 3. boot-args. 316 * 317 * @param type_t 318 * Should be an integer type or bool. 319 * 320 * @param var 321 * The name of the C variable to use for storage. 322 * 323 * @param dt_base 324 * The name of the DT node containing the property. 325 * 326 * @param dt_name 327 * The name of the DT property containing the default value. 328 * 329 * @param boot_arg 330 * The name of the boot-arg overriding the initial value from the DT. 331 * 332 * @param default_value 333 * The default value for the tunable if both DT entry and boot-arg are 334 * absent. 335 * 336 * @param flags 337 * See the description for @c tunable_dt_flags_t. 338 */ 339 #define TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 340 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \ 341 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags) 342 343 /*! 344 * @macro TUNABLE_DT_WRITEABLE 345 * 346 * @abstract 347 * Like TUNABLE_WRITEABLE, but gets the initial value from both Device 348 * Tree and boot-args. The order in which the initial value is 349 * resolved is as follows, with later steps overriding previous ones 350 * (if they are specified): 351 * 352 * 1. Device Tree Entry "/<dt_base>/<dt_name>", 353 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry 354 * "/chosen/<dt_base>/<dt_name>" (see the description for 355 * @c tunable_dt_flags_t), 356 * 3. boot-args. 357 * 358 * @param type_t 359 * Should be an integer type or bool. 360 * 361 * @param var 362 * The name of the C variable to use for storage. 363 * 364 * @param dt_base 365 * The name of the DT node containing the property. 366 * 367 * @param dt_name 368 * The name of the DT property containing the default value. 369 * 370 * @param boot_arg 371 * The name of the boot-arg overriding the initial value from the DT. 372 * 373 * @param default_value 374 * The default value for the tunable if both DT entry and boot-arg are 375 * absent. 376 * 377 * @param flags 378 * See the description for @c tunable_dt_flags_t. 379 */ 380 #define TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 381 type_t var = default_value; \ 382 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags) 383 384 /* 385 * Machine Timeouts 386 * 387 * Machine Timeouts are timeouts for low level kernel code manifesting 388 * as _Atomic uint64_t variables, whose default value can be 389 * overridden and scaled via the device tree and boot-args. 390 * 391 * Each timeout has a name, looked up directly as the property name in 392 * the device tree in both the "/machine-timeouts" and 393 * "/chosen/machine-timeouts" nodes. The "chosen" property always 394 * overrides the other one. This allows fixed per-device timeouts in 395 * the device tree to be overridden by iBoot in "chosen". 396 * 397 * Additionally, the same name with "-scale" appended is looked up as 398 * properties for optional scale factors. Scale factors are not 399 * overridden by chosen, instead all scale factors (including global 400 * and/or boot-arg scale factors) combine by multiplication. 401 * 402 * The special name "global-scale" provides a scale that applies to 403 * every timeout. 404 * 405 * All property names can be used as boot-args by prefixing 406 * "ml-timeout-", e.g. th global scale is available as the 407 * "ml-timeout-global-scale" boot-arg. 408 * 409 * By convention, if the timeout value resolves to 0, the timeout 410 * should be disabled. 411 */ 412 413 /* 414 * Machine Timeouts types. See the next section for what unit 415 * they are in. 416 * 417 * We use _Atomic, but only with relaxed ordering: This is just to 418 * make sure all devices see consistent values all the time. Since 419 * the actual timeout value will be seen as 0 before initializaton, 420 * relaxed ordering means that code that runs concurrently with 421 * initialization only risks to see a disabled timeout during early 422 * boot. 423 */ 424 typedef _Atomic uint64_t machine_timeout_t; 425 426 /* 427 * Units 428 * 429 * Machine Timeouts are ALWAYS in picoseconds in the device tree or 430 * boot-args, to avoid confusion when changing or comparing timeouts 431 * as a user, but the actual storage value might contain the same 432 * duration in another unit, calculated by the initialization code. 433 * 434 * This is done because otherwise we would likely introduce another 435 * multiplication in potentially hot code paths, given that code that 436 * actually uses the timeout storage variable is unlikely to work with 437 * picosecond values when comparing against the timeout deadline. 438 * 439 * This unit scale is *only* applied during initialization at early 440 * boot, and only if the timeout's default value was overridden 441 * through the device tree or a boot-arg. 442 */ 443 #define MACHINE_TIMEOUT_UNIT_PSEC 1 444 #define MACHINE_TIMEOUT_UNIT_NSEC 1000 445 #define MACHINE_TIMEOUT_UNIT_USEC (1000*1000) 446 #define MACHINE_TIMEOUT_UNIT_MSEC (1000*1000*1000) 447 // Special unit for timebase ticks (usually 1/24MHz) 448 #define MACHINE_TIMEOUT_UNIT_TIMEBASE 0 449 450 // DT property names are limited to 31 chars, minus "-global" suffix 451 #define MACHINE_TIMEOUT_MAX_NAME_LEN 25 452 struct machine_timeout_spec { 453 void *ptr; 454 uint64_t default_value; 455 uint64_t unit_scale; 456 char name[MACHINE_TIMEOUT_MAX_NAME_LEN + 1]; 457 bool (*skip_predicate)(struct machine_timeout_spec const *); 458 }; 459 460 extern void 461 machine_timeout_init_with_suffix(const struct machine_timeout_spec *spec, char const *phase_suffix); 462 463 extern void 464 machine_timeout_init(const struct machine_timeout_spec *spec); 465 466 #if DEVELOPMENT || DEBUG 467 // Late timeout (re-)initialization, at the end of bsd_init() 468 extern void 469 machine_timeout_bsd_init(void); 470 #endif /* DEVELOPMENT || DEBUG */ 471 472 /*! 473 * @macro MACHINE_TIMEOUT and MACHINE_TIMEOUT_DEV_WRITEABLE 474 * 475 * @abstract 476 * Defines a Machine Timeout that can be overridden and 477 * scaled through the device tree and boot-args. 478 * 479 * The variant with the _DEV_WRITEABLE suffix does not mark the timeout as 480 * SECURITY_READ_ONLY_LATE on DEVELOPMENT kernels, so that e.g. 481 * machine_timeout_init_with_suffix or sysctls can change it after lockdown. 482 * 483 * @param var 484 * The name of the C variable to use for storage. If the storage value 485 * contains 0, the timeout is considered disabled by convention. 486 * 487 * @param timeout_name 488 * The name of the timeout, used for property and boot-arg names. See 489 * the general description of Machine Timeouts above for how this name 490 * ends up being used. 491 * 492 * @param timeout_default 493 * The default value for the timeout if not specified through device 494 * tree or boot-arg. Will still be scaled if a scale factor exists. 495 * 496 * @param var_unit 497 * The unit that the storage variable is in. Note that timeout values 498 * must always be specified as picoseconds in the device tree and 499 * boot-args, but timeout initialization will convert the value to the 500 * unit specified here before writing it to the storage variable. 501 * 502 * @param skip_predicate 503 * Optionally, a function to call to decide whether the timeout should 504 * be set or not. If NULL, the timeout will always be set (if 505 * specified anywhere). A predicate has the following signature: 506 * bool skip_predicate (struct machine_timeout_spec const *) 507 */ 508 509 #define _MACHINE_TIMEOUT(var, timeout_name, timeout_default, var_unit, skip_pred) \ 510 struct machine_timeout_spec \ 511 __machine_timeout_spec_ ## var = { \ 512 .ptr = &var, \ 513 .default_value = timeout_default, \ 514 .unit_scale = var_unit, \ 515 .name = timeout_name, \ 516 .skip_predicate = skip_pred, \ 517 }; \ 518 __STARTUP_ARG(var, __LINE__, TIMEOUTS, STARTUP_RANK_FIRST, \ 519 machine_timeout_init, &__machine_timeout_spec_ ## var) 520 521 #define MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) \ 522 SECURITY_READ_ONLY_LATE(machine_timeout_t) var = 0; \ 523 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 524 525 #if DEVELOPMENT || DEBUG 526 #define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \ 527 machine_timeout_t var = 0; \ 528 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 529 #else 530 #define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \ 531 MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 532 #endif /* DEVELOPMENT || DEBUG */ 533 534 /*! 535 * @macro MACHINE_TIMEOUT_SPEC_REF 536 * 537 * @abstract 538 * References a previously defined MACHINE_TIMEOUT. 539 * 540 * This is primarily useful for overriding individual timeouts 541 * at arbitrary times (even after boot), by manually calling 542 * machine_timeout_init_with_suffix() with this macro 543 * as first argument, and a suffix to apply to both device tree and 544 * boot-arg as second argument. 545 * 546 * @param var 547 * The name of the C variable used for storage, as it was specified 548 * in MACHINE_TIMEOUT. 549 */ 550 #define MACHINE_TIMEOUT_SPEC_REF(var) (&__machine_timeout_spec_ ## var) 551 552 /*! 553 * @macro MACHINE_TIMEOUT_SPEC_DECL 554 * 555 * @abstract 556 * Declaration of machine timeout spec, mostly useful to make it known 557 * for MACHINE_TIMEOUT_SPEC_REF. 558 * 559 * @param var 560 * The name of the C variable used for storage, as it was specified 561 * in MACHINE_TIMEOUT. 562 */ 563 #define MACHINE_TIMEOUT_SPEC_DECL(var) extern struct machine_timeout_spec __machine_timeout_spec_ ## var 564 565 /* 566 * Event subsystem 567 * 568 * This allows to define a few system-wide events that allow for loose coupling 569 * between subsystems interested in those events and the emitter. 570 */ 571 572 /*! 573 * @macro EVENT_DECLARE() 574 * 575 * @brief 576 * Declares an event namespace with a given callback type. 577 * 578 * @param name The name for the event (typically in all caps). 579 * @param cb_type_t A function type for the callbacks. 580 */ 581 #define EVENT_DECLARE(name, cb_type_t) \ 582 struct name##_event { \ 583 struct event_hdr evt_link; \ 584 cb_type_t *evt_cb; \ 585 }; \ 586 extern struct event_hdr name##_HEAD 587 588 /*! 589 * @macro EVENT_DEFINE() 590 * 591 * @brief 592 * Defines the head for the event corresponding to an EVENT_DECLARE() 593 */ 594 #define EVENT_DEFINE(name) \ 595 __security_const_late struct event_hdr name##_HEAD 596 597 /*! 598 * @macro EVENT_REGISTER_HANDLER() 599 * 600 * @brief 601 * Registers a handler for a given event. 602 * 603 * @param name The name for the event as declared in EVENT_DECLARE() 604 * @param handler The handler to register for this event. 605 */ 606 #define EVENT_REGISTER_HANDLER(name, handler) \ 607 __EVENT_REGISTER(name, __LINE__, handler) 608 609 610 /*! 611 * @macro EVENT_INVOKE() 612 * 613 * @brief 614 * Call all the events handlers with the specified arguments. 615 * 616 * @param name The name for the event as declared in EVENT_DECLARE() 617 * @param handler The handler to register for this event. 618 */ 619 #define EVENT_INVOKE(name, ...) \ 620 for (struct event_hdr *__e = &name##_HEAD; (__e = __e->next);) { \ 621 __container_of(__e, struct name##_event, \ 622 evt_link)->evt_cb(__VA_ARGS__); \ 623 } 624 625 626 #if DEBUG || DEVELOPMENT 627 628 /*! 629 * @macro SYSCTL_TEST_REGISTER 630 * 631 * @abstract 632 * Declares a test that will appear under @c debug.test.${name}. 633 * 634 * @param name 635 * An identifier that will be stringified to form the sysctl test name. 636 * 637 * @param cb 638 * The callback to run, of type: 639 * <code> 640 * int (callback *)(int64_t value, int64_t *); 641 * </code> 642 */ 643 #define SYSCTL_TEST_REGISTER(name, cb) \ 644 static __startup_data struct sysctl_test_setup_spec \ 645 __startup_SYSCTL_TEST_ ## name = { \ 646 .st_name = #name, \ 647 .st_func = &cb, \ 648 }; \ 649 STARTUP_ARG(SYSCTL, STARTUP_RANK_MIDDLE, \ 650 sysctl_register_test_startup, &__startup_SYSCTL_TEST_ ## name) 651 652 #endif /* DEBUG || DEVELOPMENT */ 653 #pragma mark - internals 654 655 __END_DECLS 656 657 #ifdef __cplusplus 658 template <typename T> 659 struct __startup_tunable { 660 static const bool value = false; 661 }; 662 663 template <> 664 struct __startup_tunable <bool>{ 665 static const bool value = true; 666 }; 667 #define __startup_type_is_bool(type_t) __startup_tunable<type_t>::value 668 #else 669 #define __startup_type_is_bool(type_t) __builtin_types_compatible_p(bool, type_t) 670 #endif 671 672 __BEGIN_DECLS 673 674 #define __TUNABLE(type_t, var, key) \ 675 static __startup_data char __startup_TUNABLES_name_ ## var[] = key; \ 676 static __startup_data struct startup_tunable_spec \ 677 __startup_TUNABLES_spec_ ## var = { \ 678 .name = __startup_TUNABLES_name_ ## var, \ 679 .var_addr = (void *)&var, \ 680 .var_len = sizeof(type_t), \ 681 .var_is_bool = __startup_type_is_bool(type_t), \ 682 }; \ 683 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 684 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var) 685 686 #define __TUNABLE_STR(var, key) \ 687 static __startup_data char __startup_TUNABLES_name_ ## var[] = key; \ 688 static __startup_data struct startup_tunable_spec \ 689 __startup_TUNABLES_spec_ ## var = { \ 690 .name = __startup_TUNABLES_name_ ## var, \ 691 .var_addr = (void *)&var, \ 692 .var_len = sizeof(var), \ 693 .var_is_str = true, \ 694 }; \ 695 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 696 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var) 697 698 #define __TUNABLE_DT(type_t, var, dt_base_key, dt_name_key, boot_arg_key, flags) \ 699 static __startup_data char __startup_TUNABLES_dt_base_ ## var[] = dt_base_key; \ 700 static __startup_data char __startup_TUNABLES_dt_name_ ## var[] = dt_name_key; \ 701 static __startup_data char __startup_TUNABLES_name_ ## var[] = boot_arg_key; \ 702 static __startup_data struct startup_tunable_dt_spec \ 703 __startup_TUNABLES_DT_spec_ ## var = { \ 704 .dt_base = __startup_TUNABLES_dt_base_ ## var, \ 705 .dt_name = __startup_TUNABLES_dt_name_ ## var, \ 706 .dt_chosen_override = (bool)((flags) & TUNABLE_DT_CHECK_CHOSEN), \ 707 .boot_arg_name = __startup_TUNABLES_name_ ## var, \ 708 .var_addr = (void *)&var, \ 709 .var_len = sizeof(type_t), \ 710 .var_is_bool = __startup_type_is_bool(type_t), \ 711 }; \ 712 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 713 kernel_startup_tunable_dt_init, &__startup_TUNABLES_DT_spec_ ## var) 714 715 #ifdef __cplusplus 716 #define __STARTUP_FUNC_CAST(func, a) \ 717 (void(*)(const void *))func 718 #else 719 #define __STARTUP_FUNC_CAST(func, a) \ 720 (typeof(func(a))(*)(const void *))func 721 #endif 722 723 724 #define __STARTUP1(name, line, subsystem, rank, func, a, b) \ 725 __PLACE_IN_SECTION(STARTUP_HOOK_SEGMENT "," STARTUP_HOOK_SECTION) \ 726 static const struct startup_entry \ 727 __startup_ ## subsystem ## _entry_ ## name ## _ ## line = { \ 728 STARTUP_SUB_ ## subsystem, \ 729 rank, __STARTUP_FUNC_CAST(func, a), b, \ 730 } 731 732 #define __STARTUP(name, line, subsystem, rank, func) \ 733 __STARTUP1(name, line, subsystem, rank, func, , NULL) 734 735 #define __STARTUP_ARG(name, line, subsystem, rank, func, arg) \ 736 __STARTUP1(name, line, subsystem, rank, func, arg, arg) 737 738 struct startup_entry { 739 startup_subsystem_id_t subsystem; 740 startup_rank_t rank; 741 void (*func)(const void *); 742 const void *arg; 743 }; 744 745 struct startup_tunable_spec { 746 const char *name; 747 void *var_addr; 748 int var_len; 749 bool var_is_bool; 750 bool var_is_str; 751 }; 752 753 struct startup_tunable_dt_spec { 754 const char *dt_base; 755 const char *dt_name; 756 bool dt_chosen_override; 757 const char *boot_arg_name; 758 void *var_addr; 759 int var_len; 760 bool var_is_bool; 761 }; 762 763 #if DEBUG || DEVELOPMENT 764 struct sysctl_test_setup_spec { 765 const char *st_name; 766 int (*st_func)(int64_t, int64_t *); 767 }; 768 769 extern void sysctl_register_test_startup( 770 struct sysctl_test_setup_spec *spec); 771 #endif /* DEBUG || DEVELOPMENT */ 772 773 /* 774 * Kernel and machine startup declarations 775 */ 776 777 /* Initialize kernel */ 778 extern void kernel_startup_bootstrap(void); 779 extern void kernel_startup_initialize_upto(startup_subsystem_id_t upto); 780 extern void kernel_startup_tunable_init(const struct startup_tunable_spec *); 781 extern void kernel_startup_tunable_dt_init(const struct startup_tunable_dt_spec *); 782 extern void kernel_bootstrap(void); 783 784 /* Initialize machine dependent stuff */ 785 extern void machine_init(void); 786 787 extern void slave_main(void *machine_param); 788 789 /* 790 * The following must be implemented in machine dependent code. 791 */ 792 793 /* Slave cpu initialization */ 794 extern void slave_machine_init(void *machine_param); 795 796 /* Device subsystem initialization */ 797 extern void device_service_create(void); 798 799 struct event_hdr { 800 struct event_hdr *next; 801 }; 802 803 extern void event_register_handler(struct event_hdr *event_hdr); 804 805 #define __EVENT_REGISTER(name, lno, handler) \ 806 static __security_const_late struct name##_event name##_event_##lno = { \ 807 .evt_link.next = &name##_HEAD, \ 808 .evt_cb = (handler), \ 809 }; \ 810 __STARTUP_ARG(name, lno, EVENT, STARTUP_RANK_FIRST, \ 811 event_register_handler, &name##_event_##lno.evt_link) 812 813 #ifdef MACH_BSD 814 815 /* BSD subsystem initialization */ 816 extern void bsd_init(void); 817 818 #endif /* MACH_BSD */ 819 820 #pragma GCC visibility pop 821 822 __END_DECLS 823 824 #endif /* _KERN_STARTUP_H_ */ 825 826 #endif /* XNU_KERNEL_PRIVATE */ 827