1 /* 2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 32 #ifdef XNU_KERNEL_PRIVATE 33 34 #ifndef _KERN_STARTUP_H_ 35 #define _KERN_STARTUP_H_ 36 37 #include <stdbool.h> 38 #include <stddef.h> 39 #include <stdint.h> 40 41 #include <libkern/section_keywords.h> 42 43 __BEGIN_DECLS 44 45 #pragma GCC visibility push(hidden) 46 47 /*! 48 * @enum startup_subsystem_id_t 49 * 50 * @abstract 51 * Represents a stage of kernel intialization, ubnd allows for subsystems 52 * to register initializers for a specific stage. 53 * 54 * @discussion 55 * Documentation of each subsystem initialization sequence exists in 56 * @file doc/startup.md. 57 */ 58 __enum_decl(startup_subsystem_id_t, uint32_t, { 59 STARTUP_SUB_NONE = 0, /**< reserved for the startup subsystem */ 60 61 STARTUP_SUB_TUNABLES, /**< support for the tunables subsystem */ 62 STARTUP_SUB_TIMEOUTS, /**< configurable machine timeouts */ 63 STARTUP_SUB_LOCKS, /**< various subsystem locks */ 64 STARTUP_SUB_KPRINTF, /**< kprintf initialization */ 65 66 STARTUP_SUB_PMAP_STEAL, /**< to perform various pmap carveouts */ 67 STARTUP_SUB_KMEM, /**< once kmem_alloc is ready */ 68 STARTUP_SUB_ZALLOC, /**< initialize zalloc and kalloc */ 69 STARTUP_SUB_KTRACE, /**< initialize kernel trace */ 70 STARTUP_SUB_PERCPU, /**< initialize the percpu subsystem */ 71 STARTUP_SUB_EVENT, /**< initiailze the event subsystem */ 72 73 STARTUP_SUB_CODESIGNING, /**< codesigning subsystem */ 74 STARTUP_SUB_OSLOG, /**< oslog and kernel logging */ 75 STARTUP_SUB_MACH_IPC, /**< Mach IPC */ 76 STARTUP_SUB_THREAD_CALL, /**< Thread calls */ 77 STARTUP_SUB_SYSCTL, /**< registers sysctls */ 78 STARTUP_SUB_EXCLAVES, /**< initialize exclaves */ 79 STARTUP_SUB_EARLY_BOOT, /**< interrupts/preemption are turned on */ 80 81 STARTUP_SUB_LOCKDOWN = ~0u, /**< reserved for the startup subsystem */ 82 }); 83 84 /*! 85 * Stores the last subsystem to have been fully initialized; 86 */ 87 extern startup_subsystem_id_t startup_phase; 88 89 /*! 90 * @enum startup_debug_t 91 * 92 * @abstract 93 * Flags set in the @c startup_debug global to configure startup debugging. 94 */ 95 __options_decl(startup_debug_t, uint32_t, { 96 STARTUP_DEBUG_NONE = 0x00000000, 97 STARTUP_DEBUG_VERBOSE = 0x00000001, 98 }); 99 100 extern startup_debug_t startup_debug; 101 102 /*! 103 * @enum startup_rank 104 * 105 * @abstract 106 * Specifies in which rank a given initializer runs within a given section 107 * to register initializers for a specific rank within the subsystem. 108 * 109 * @description 110 * A startup function, declared with @c STARTUP or @c STARTUP_ARG, can specify 111 * a rank within the subsystem they initialize. 112 * 113 * @c STARTUP_RANK_NTH(n) will let callbacks be run at stage @c n (0-based). 114 * 115 * @c STARTUP_RANK_FIRST, @c STARTUP_RANK_SECOND, @c STARTUP_RANK_THIRD and 116 * @c STARTUP_RANK_FOURTH are given as convenient names for these. 117 * 118 * @c STARTUP_RANK_MIDDLE is a reserved value that will let startup functions 119 * run after all the @c STARTUP_RANK_NTH(n) ones have. 120 * 121 * @c STARTUP_RANK_NTH_LATE_NTH(n) will let callbacks be run then in @c n rank 122 * after the @c STARTUP_RANK_MIDDLE ones (0-based). 123 * 124 * @c STARTUP_RANK_LAST callbacks will run absolutely last after everything 125 * else did for this subsystem. 126 */ 127 __enum_decl(startup_rank_t, uint32_t, { 128 #define STARTUP_RANK_NTH(n) ((startup_rank_t)(n - 1)) 129 STARTUP_RANK_FIRST = 0, 130 STARTUP_RANK_SECOND = 1, 131 STARTUP_RANK_THIRD = 2, 132 STARTUP_RANK_FOURTH = 3, 133 134 STARTUP_RANK_MIDDLE = 0x7fffffff, 135 136 #define STARTUP_RANK_LATE_NTH(n) \ 137 ((startup_rank_t)(STARTUP_RANK_MIDDLE + 1 + (n))) 138 139 STARTUP_RANK_LAST = 0xffffffff, 140 }); 141 142 #if KASAN 143 /* 144 * The use of weird sections that get unmapped confuse the hell out of kasan, 145 * so for KASAN leave things in regular __TEXT/__DATA segments 146 */ 147 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 148 #define STARTUP_CONST_SEGSECT "__DATA_CONST,__init" 149 #define STARTUP_DATA_SEGSECT "__DATA,__init" 150 #define STARTUP_HOOK_SEGMENT "__DATA" 151 #define STARTUP_HOOK_SECTION "__init_entry_set" 152 #elif defined(__x86_64__) 153 /* Intel doesn't have a __BOOTDATA but doesn't protect __KLD */ 154 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 155 #define STARTUP_CONST_SEGSECT "__KLDDATA,__const" 156 #define STARTUP_DATA_SEGSECT "__KLDDATA,__init" 157 #define STARTUP_HOOK_SEGMENT "__KLDDATA" 158 #define STARTUP_HOOK_SECTION "__init_entry_set" 159 #else 160 /* arm protects __KLD early, so use __BOOTDATA for data */ 161 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 162 #define STARTUP_CONST_SEGSECT "__KLDDATA,__const" 163 #define STARTUP_DATA_SEGSECT "__BOOTDATA,__init" 164 #define STARTUP_HOOK_SEGMENT "__BOOTDATA" 165 #define STARTUP_HOOK_SECTION "__init_entry_set" 166 #endif 167 168 /*! 169 * @macro __startup_func 170 * 171 * @abstract 172 * Attribute to place on functions used only during the kernel startup phase. 173 * 174 * @description 175 * Code marked with this attribute will be unmapped after kernel lockdown. 176 */ 177 #define __startup_func \ 178 __PLACE_IN_SECTION(STARTUP_CODE_SEGSECT) \ 179 __attribute__((cold, visibility("hidden"))) 180 181 /*! 182 * @macro __startup_data 183 * 184 * @abstract 185 * Attribute to place on globals used during the kernel startup phase. 186 * 187 * @description 188 * Data marked with this attribute will be unmapped after kernel lockdown. 189 */ 190 #define __startup_data \ 191 __PLACE_IN_SECTION(STARTUP_DATA_SEGSECT) 192 193 /*! 194 * @macro __startup_const 195 * 196 * @abstract 197 * Attribute to place on global constants used during the kernel startup phase. 198 * 199 * @description 200 * Data marked with this attribute will be unmapped after kernel lockdown. 201 * 202 * __startup_const implies const. Be mindful that for pointers, the `const` 203 * will end up on the wrong side of the star if you put __startup_const at the 204 * start of the declaration. 205 */ 206 #define __startup_const \ 207 __PLACE_IN_SECTION(STARTUP_CONST_SEGSECT) const 208 209 /*! 210 * @macro STARTUP 211 * 212 * @abstract 213 * Declares a kernel startup callback. 214 */ 215 #define STARTUP(subsystem, rank, func) \ 216 __STARTUP(func, __LINE__, subsystem, rank, func) 217 218 /*! 219 * @macro STARTUP_ARG 220 * 221 * @abstract 222 * Declares a kernel startup callback that takes an argument. 223 */ 224 #define STARTUP_ARG(subsystem, rank, func, arg) \ 225 __STARTUP_ARG(func, __LINE__, subsystem, rank, func, arg) 226 227 /*! 228 * @macro TUNABLE 229 * 230 * @abstract 231 * Declares a read-only kernel tunable that is read from a boot-arg with 232 * a default value, without further processing. 233 * 234 * @param type_t 235 * Should be an integer type or bool. 236 * 237 * @param var 238 * The name of the C variable to use for storage. 239 * 240 * @param boot_arg 241 * The name of the boot-arg to parse for initialization 242 * 243 * @param default_value 244 * The default value for the tunable if the boot-arg is absent. 245 */ 246 #define TUNABLE(type_t, var, boot_arg, default_value) \ 247 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \ 248 __TUNABLE(type_t, var, boot_arg) 249 250 /*! 251 * @macro TUNABLE_WRITEABLE 252 * 253 * @abstract 254 * Declares a writeable kernel tunable that is read from a boot-arg with 255 * a default value, without further processing. 256 * 257 * @param type_t 258 * Should be an integer type or bool. 259 * 260 * @param var 261 * The name of the C variable to use for storage. 262 * 263 * @param boot_arg 264 * The name of the boot-arg to parse for initialization 265 * 266 * @param default_value 267 * The default value for the tunable if the boot-arg is absent. 268 */ 269 #define TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) \ 270 type_t var = default_value; \ 271 __TUNABLE(type_t, var, boot_arg) 272 273 #if DEBUG || DEVELOPMENT 274 #define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \ 275 TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) 276 #else 277 #define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \ 278 TUNABLE(type_t, var, boot_arg, default_value) 279 #endif 280 281 /*! 282 * @macro TUNABLE_STR 283 * 284 * @abstract 285 * Declares a read-only kernel tunable that is read from a boot-arg with 286 * a default value, without further processing. 287 * 288 * @param var 289 * The name of the C variable to use for storage. 290 * 291 * @param count 292 * The number of bytes in the buffer. 293 * 294 * @param boot_arg 295 * The name of the boot-arg to parse for initialization 296 * 297 * @param default_value 298 * The default value for the tunable if the boot-arg is absent. 299 */ 300 #define TUNABLE_STR(var, count, boot_arg, default_value) \ 301 char __security_const_late var[count] = default_value; \ 302 __TUNABLE_STR(var, boot_arg) 303 304 /*! 305 * @enum tunable_dt_flags_t 306 * 307 * @abstract 308 * Flags used with the @c TUNABLE_DT* macros. 309 * 310 * @description 311 * If TUNABLE_DT_CHECK_CHOSEN is set, a value in 312 * /chosen/<dt_base>/<dt_name> takes precedence over any value in 313 * /<dt_base>/<dt_name>. /chosen is by convention the area where 314 * synthesized values not coming from the serialized device tree are 315 * being added, so this provides a way for e.g. the boot-loader to 316 * set/override tunables. 317 */ 318 __options_decl(tunable_dt_flags_t, uint32_t, { 319 TUNABLE_DT_NONE = 0x00000000, 320 TUNABLE_DT_CHECK_CHOSEN = 0x00000001, 321 }); 322 323 /*! 324 * @macro TUNABLE_DT 325 * 326 * @abstract 327 * Like TUNABLE, but gets the initial value from both Device Tree and 328 * boot-args. The order in which the initial value is resolved is as 329 * follows, with later steps overriding previous ones (if they are 330 * specified): 331 * 332 * 1. Device Tree Entry "/<dt_base>/<dt_name>", 333 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry 334 * "/chosen/<dt_base>/<dt_name>" (see the description for 335 * @c tunable_dt_flags_t), 336 * 3. boot-args. 337 * 338 * @param type_t 339 * Should be an integer type or bool. 340 * 341 * @param var 342 * The name of the C variable to use for storage. 343 * 344 * @param dt_base 345 * The name of the DT node containing the property. 346 * 347 * @param dt_name 348 * The name of the DT property containing the default value. 349 * 350 * @param boot_arg 351 * The name of the boot-arg overriding the initial value from the DT. 352 * 353 * @param default_value 354 * The default value for the tunable if both DT entry and boot-arg are 355 * absent. 356 * 357 * @param flags 358 * See the description for @c tunable_dt_flags_t. 359 */ 360 #define TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 361 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \ 362 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags) 363 364 /*! 365 * @macro TUNABLE_DT_WRITEABLE 366 * 367 * @abstract 368 * Like TUNABLE_WRITEABLE, but gets the initial value from both Device 369 * Tree and boot-args. The order in which the initial value is 370 * resolved is as follows, with later steps overriding previous ones 371 * (if they are specified): 372 * 373 * 1. Device Tree Entry "/<dt_base>/<dt_name>", 374 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry 375 * "/chosen/<dt_base>/<dt_name>" (see the description for 376 * @c tunable_dt_flags_t), 377 * 3. boot-args. 378 * 379 * @param type_t 380 * Should be an integer type or bool. 381 * 382 * @param var 383 * The name of the C variable to use for storage. 384 * 385 * @param dt_base 386 * The name of the DT node containing the property. 387 * 388 * @param dt_name 389 * The name of the DT property containing the default value. 390 * 391 * @param boot_arg 392 * The name of the boot-arg overriding the initial value from the DT. 393 * 394 * @param default_value 395 * The default value for the tunable if both DT entry and boot-arg are 396 * absent. 397 * 398 * @param flags 399 * See the description for @c tunable_dt_flags_t. 400 */ 401 #define TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 402 type_t var = default_value; \ 403 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags) 404 405 #if DEBUG || DEVELOPMENT 406 #define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 407 TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) 408 #else 409 #define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 410 TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) 411 #endif 412 413 /* 414 * Machine Timeouts 415 * 416 * Machine Timeouts are timeouts for low level kernel code manifesting 417 * as _Atomic uint64_t variables, whose default value can be 418 * overridden and scaled via the device tree and boot-args. 419 * 420 * Each timeout has a name, looked up directly as the property name in 421 * the device tree in both the "/machine-timeouts" and 422 * "/chosen/machine-timeouts" nodes. The "chosen" property always 423 * overrides the other one. This allows fixed per-device timeouts in 424 * the device tree to be overridden by iBoot in "chosen". 425 * 426 * Additionally, the same name with "-scale" appended is looked up as 427 * properties for optional scale factors. Scale factors are not 428 * overridden by chosen, instead all scale factors (including global 429 * and/or boot-arg scale factors) combine by multiplication. 430 * 431 * The special name "global-scale" provides a scale that applies to 432 * every timeout. 433 * 434 * All property names can be used as boot-args by prefixing 435 * "ml-timeout-", e.g. th global scale is available as the 436 * "ml-timeout-global-scale" boot-arg. 437 * 438 * By convention, if the timeout value resolves to 0, the timeout 439 * should be disabled. 440 */ 441 442 /* 443 * Machine Timeouts types. See the next section for what unit 444 * they are in. 445 * 446 * We use _Atomic, but only with relaxed ordering: This is just to 447 * make sure all devices see consistent values all the time. Since 448 * the actual timeout value will be seen as 0 before initializaton, 449 * relaxed ordering means that code that runs concurrently with 450 * initialization only risks to see a disabled timeout during early 451 * boot. 452 */ 453 typedef _Atomic uint64_t machine_timeout_t; 454 455 /* 456 * Units 457 * 458 * Machine Timeouts are ALWAYS in picoseconds in the device tree or 459 * boot-args, to avoid confusion when changing or comparing timeouts 460 * as a user, but the actual storage value might contain the same 461 * duration in another unit, calculated by the initialization code. 462 * 463 * This is done because otherwise we would likely introduce another 464 * multiplication in potentially hot code paths, given that code that 465 * actually uses the timeout storage variable is unlikely to work with 466 * picosecond values when comparing against the timeout deadline. 467 * 468 * This unit scale is *only* applied during initialization at early 469 * boot, and only if the timeout's default value was overridden 470 * through the device tree or a boot-arg. 471 */ 472 #define MACHINE_TIMEOUT_UNIT_PSEC 1 473 #define MACHINE_TIMEOUT_UNIT_NSEC 1000 474 #define MACHINE_TIMEOUT_UNIT_USEC (1000*1000) 475 #define MACHINE_TIMEOUT_UNIT_MSEC (1000*1000*1000) 476 // Special unit for timebase ticks (usually 1/24MHz) 477 #define MACHINE_TIMEOUT_UNIT_TIMEBASE 0 478 479 // DT property names are limited to 31 chars, minus "-global" suffix 480 #define MACHINE_TIMEOUT_MAX_NAME_LEN 25 481 struct machine_timeout_spec { 482 void *ptr; 483 uint64_t default_value; 484 uint64_t unit_scale; 485 char name[MACHINE_TIMEOUT_MAX_NAME_LEN + 1]; 486 bool (*skip_predicate)(struct machine_timeout_spec const *); 487 }; 488 489 extern void 490 machine_timeout_init_with_suffix(const struct machine_timeout_spec *spec, char const *phase_suffix); 491 492 extern void 493 machine_timeout_init(const struct machine_timeout_spec *spec); 494 495 #if DEVELOPMENT || DEBUG 496 // Late timeout (re-)initialization, at the end of bsd_init() 497 extern void 498 machine_timeout_bsd_init(void); 499 #endif /* DEVELOPMENT || DEBUG */ 500 501 /*! 502 * @macro MACHINE_TIMEOUT and MACHINE_TIMEOUT_DEV_WRITEABLE 503 * 504 * @abstract 505 * Defines a Machine Timeout that can be overridden and 506 * scaled through the device tree and boot-args. 507 * 508 * The variant with the _DEV_WRITEABLE suffix does not mark the timeout as 509 * SECURITY_READ_ONLY_LATE on DEVELOPMENT kernels, so that e.g. 510 * machine_timeout_init_with_suffix or sysctls can change it after lockdown. 511 * 512 * @param var 513 * The name of the C variable to use for storage. If the storage value 514 * contains 0, the timeout is considered disabled by convention. 515 * 516 * @param timeout_name 517 * The name of the timeout, used for property and boot-arg names. See 518 * the general description of Machine Timeouts above for how this name 519 * ends up being used. 520 * 521 * @param timeout_default 522 * The default value for the timeout if not specified through device 523 * tree or boot-arg. Will still be scaled if a scale factor exists. 524 * 525 * @param var_unit 526 * The unit that the storage variable is in. Note that timeout values 527 * must always be specified as picoseconds in the device tree and 528 * boot-args, but timeout initialization will convert the value to the 529 * unit specified here before writing it to the storage variable. 530 * 531 * @param skip_predicate 532 * Optionally, a function to call to decide whether the timeout should 533 * be set or not. If NULL, the timeout will always be set (if 534 * specified anywhere). A predicate has the following signature: 535 * bool skip_predicate (struct machine_timeout_spec const *) 536 */ 537 538 #define _MACHINE_TIMEOUT(var, timeout_name, timeout_default, var_unit, skip_pred) \ 539 struct machine_timeout_spec \ 540 __machine_timeout_spec_ ## var = { \ 541 .ptr = &var, \ 542 .default_value = timeout_default, \ 543 .unit_scale = var_unit, \ 544 .name = timeout_name, \ 545 .skip_predicate = skip_pred, \ 546 }; \ 547 __STARTUP_ARG(var, __LINE__, TIMEOUTS, STARTUP_RANK_FIRST, \ 548 machine_timeout_init, &__machine_timeout_spec_ ## var) 549 550 #define MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) \ 551 SECURITY_READ_ONLY_LATE(machine_timeout_t) var = 0; \ 552 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 553 554 #if DEVELOPMENT || DEBUG 555 #define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \ 556 machine_timeout_t var = 0; \ 557 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 558 #else 559 #define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \ 560 MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 561 #endif /* DEVELOPMENT || DEBUG */ 562 563 /*! 564 * @macro MACHINE_TIMEOUT_SPEC_REF 565 * 566 * @abstract 567 * References a previously defined MACHINE_TIMEOUT. 568 * 569 * This is primarily useful for overriding individual timeouts 570 * at arbitrary times (even after boot), by manually calling 571 * machine_timeout_init_with_suffix() with this macro 572 * as first argument, and a suffix to apply to both device tree and 573 * boot-arg as second argument. 574 * 575 * @param var 576 * The name of the C variable used for storage, as it was specified 577 * in MACHINE_TIMEOUT. 578 */ 579 #define MACHINE_TIMEOUT_SPEC_REF(var) (&__machine_timeout_spec_ ## var) 580 581 /*! 582 * @macro MACHINE_TIMEOUT_SPEC_DECL 583 * 584 * @abstract 585 * Declaration of machine timeout spec, mostly useful to make it known 586 * for MACHINE_TIMEOUT_SPEC_REF. 587 * 588 * @param var 589 * The name of the C variable used for storage, as it was specified 590 * in MACHINE_TIMEOUT. 591 */ 592 #define MACHINE_TIMEOUT_SPEC_DECL(var) extern struct machine_timeout_spec __machine_timeout_spec_ ## var 593 594 /* 595 * Event subsystem 596 * 597 * This allows to define a few system-wide events that allow for loose coupling 598 * between subsystems interested in those events and the emitter. 599 */ 600 601 /*! 602 * @macro EVENT_DECLARE() 603 * 604 * @brief 605 * Declares an event namespace with a given callback type. 606 * 607 * @param name The name for the event (typically in all caps). 608 * @param cb_type_t A function type for the callbacks. 609 */ 610 #define EVENT_DECLARE(name, cb_type_t) \ 611 struct name##_event { \ 612 struct event_hdr evt_link; \ 613 cb_type_t *evt_cb; \ 614 }; \ 615 extern struct event_hdr name##_HEAD 616 617 /*! 618 * @macro EVENT_DEFINE() 619 * 620 * @brief 621 * Defines the head for the event corresponding to an EVENT_DECLARE() 622 */ 623 #define EVENT_DEFINE(name) \ 624 __security_const_late struct event_hdr name##_HEAD 625 626 /*! 627 * @macro EVENT_REGISTER_HANDLER() 628 * 629 * @brief 630 * Registers a handler for a given event. 631 * 632 * @param name The name for the event as declared in EVENT_DECLARE() 633 * @param handler The handler to register for this event. 634 */ 635 #define EVENT_REGISTER_HANDLER(name, handler) \ 636 __EVENT_REGISTER(name, __LINE__, handler) 637 638 639 /*! 640 * @macro EVENT_INVOKE() 641 * 642 * @brief 643 * Call all the events handlers with the specified arguments. 644 * 645 * @param name The name for the event as declared in EVENT_DECLARE() 646 * @param handler The handler to register for this event. 647 */ 648 #define EVENT_INVOKE(name, ...) \ 649 for (struct event_hdr *__e = &name##_HEAD; (__e = __e->next);) { \ 650 __container_of(__e, struct name##_event, \ 651 evt_link)->evt_cb(__VA_ARGS__); \ 652 } 653 654 655 #if DEBUG || DEVELOPMENT 656 657 /*! 658 * @macro SYSCTL_TEST_REGISTER 659 * 660 * @abstract 661 * Declares a test that will appear under @c debug.test.${name}. 662 * 663 * @param name 664 * An identifier that will be stringified to form the sysctl test name. 665 * 666 * @param cb 667 * The callback to run, of type: 668 * <code> 669 * int (callback *)(int64_t value, int64_t *); 670 * </code> 671 */ 672 #define SYSCTL_TEST_REGISTER(name, cb) \ 673 static __startup_data struct sysctl_test_setup_spec \ 674 __startup_SYSCTL_TEST_ ## name = { \ 675 .st_name = #name, \ 676 .st_func = &cb, \ 677 }; \ 678 STARTUP_ARG(SYSCTL, STARTUP_RANK_MIDDLE, \ 679 sysctl_register_test_startup, &__startup_SYSCTL_TEST_ ## name) 680 681 #endif /* DEBUG || DEVELOPMENT */ 682 #pragma mark - internals 683 684 __END_DECLS 685 686 #ifdef __cplusplus 687 template <typename T> 688 struct __startup_tunable { 689 static const bool value = false; 690 }; 691 692 template <> 693 struct __startup_tunable <bool>{ 694 static const bool value = true; 695 }; 696 #define __startup_type_is_bool(type_t) __startup_tunable<type_t>::value 697 #else 698 #define __startup_type_is_bool(type_t) __builtin_types_compatible_p(bool, type_t) 699 #endif 700 701 __BEGIN_DECLS 702 703 #define __TUNABLE(type_t, var, key) \ 704 static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \ 705 static __startup_const struct startup_tunable_spec \ 706 __startup_TUNABLES_spec_ ## var = { \ 707 .name = __startup_TUNABLES_name_ ## var, \ 708 .var_addr = (void *)&var, \ 709 .var_len = sizeof(type_t), \ 710 .var_is_bool = __startup_type_is_bool(type_t), \ 711 }; \ 712 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 713 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var) 714 715 #define __TUNABLE_STR(var, key) \ 716 static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \ 717 static __startup_const struct startup_tunable_spec \ 718 __startup_TUNABLES_spec_ ## var = { \ 719 .name = __startup_TUNABLES_name_ ## var, \ 720 .var_addr = (void *)&var, \ 721 .var_len = sizeof(var), \ 722 .var_is_str = true, \ 723 }; \ 724 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 725 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var) 726 727 #define __TUNABLE_DT(type_t, var, dt_base_key, dt_name_key, boot_arg_key, flags) \ 728 static __startup_const char __startup_TUNABLES_dt_base_ ## var[] = dt_base_key; \ 729 static __startup_const char __startup_TUNABLES_dt_name_ ## var[] = dt_name_key; \ 730 static __startup_const char __startup_TUNABLES_name_ ## var[] = boot_arg_key; \ 731 static __startup_const struct startup_tunable_dt_spec \ 732 __startup_TUNABLES_DT_spec_ ## var = { \ 733 .dt_base = __startup_TUNABLES_dt_base_ ## var, \ 734 .dt_name = __startup_TUNABLES_dt_name_ ## var, \ 735 .dt_chosen_override = (bool)((flags) & TUNABLE_DT_CHECK_CHOSEN), \ 736 .boot_arg_name = __startup_TUNABLES_name_ ## var, \ 737 .var_addr = (void *)&var, \ 738 .var_len = sizeof(type_t), \ 739 .var_is_bool = __startup_type_is_bool(type_t), \ 740 }; \ 741 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 742 kernel_startup_tunable_dt_init, &__startup_TUNABLES_DT_spec_ ## var) 743 744 #ifdef __cplusplus 745 #define __STARTUP_FUNC_CAST(func, a) \ 746 (void(*)(const void *))func 747 #else 748 #define __STARTUP_FUNC_CAST(func, a) \ 749 (typeof(func(a))(*)(const void *))func 750 #endif 751 752 753 #define __STARTUP1(name, line, subsystem, rank, func, a, b) \ 754 __PLACE_IN_SECTION(STARTUP_HOOK_SEGMENT "," STARTUP_HOOK_SECTION) \ 755 static const struct startup_entry \ 756 __startup_ ## subsystem ## _entry_ ## name ## _ ## line = { \ 757 STARTUP_SUB_ ## subsystem, \ 758 rank, __STARTUP_FUNC_CAST(func, a), b, \ 759 } 760 761 #define __STARTUP(name, line, subsystem, rank, func) \ 762 __STARTUP1(name, line, subsystem, rank, func, , NULL) 763 764 #define __STARTUP_ARG(name, line, subsystem, rank, func, arg) \ 765 __STARTUP1(name, line, subsystem, rank, func, arg, arg) 766 767 struct startup_entry { 768 startup_subsystem_id_t subsystem; 769 startup_rank_t rank; 770 void (*func)(const void *); 771 const void *arg; 772 }; 773 774 struct startup_tunable_spec { 775 const char *name; 776 void *var_addr; 777 int var_len; 778 bool var_is_bool; 779 bool var_is_str; 780 }; 781 782 struct startup_tunable_dt_spec { 783 const char *dt_base; 784 const char *dt_name; 785 bool dt_chosen_override; 786 const char *boot_arg_name; 787 void *var_addr; 788 int var_len; 789 bool var_is_bool; 790 }; 791 792 #if DEBUG || DEVELOPMENT 793 struct sysctl_test_setup_spec { 794 const char *st_name; 795 int (*st_func)(int64_t, int64_t *); 796 }; 797 798 extern void sysctl_register_test_startup( 799 struct sysctl_test_setup_spec *spec); 800 #endif /* DEBUG || DEVELOPMENT */ 801 802 /* 803 * Kernel and machine startup declarations 804 */ 805 806 /* Initialize kernel */ 807 extern void kernel_startup_bootstrap(void); 808 extern void kernel_startup_initialize_upto(startup_subsystem_id_t upto); 809 extern void kernel_startup_tunable_init(const struct startup_tunable_spec *); 810 extern void kernel_startup_tunable_dt_init(const struct startup_tunable_dt_spec *); 811 extern void kernel_bootstrap(void); 812 813 /* Initialize machine dependent stuff */ 814 extern void machine_init(void); 815 816 extern void secondary_cpu_main(void *machine_param); 817 818 /* Secondary cpu initialization */ 819 extern void machine_cpu_reinit(void *machine_param); 820 extern void processor_cpu_reinit(void* machine_param, bool wait_for_cpu_signal, bool is_final_system_sleep); 821 822 /* Device subsystem initialization */ 823 extern void device_service_create(void); 824 825 struct event_hdr { 826 struct event_hdr *next; 827 }; 828 829 extern void event_register_handler(struct event_hdr *event_hdr); 830 831 #define __EVENT_REGISTER(name, lno, handler) \ 832 static __security_const_late struct name##_event name##_event_##lno = { \ 833 .evt_link.next = &name##_HEAD, \ 834 .evt_cb = (handler), \ 835 }; \ 836 __STARTUP_ARG(name, lno, EVENT, STARTUP_RANK_FIRST, \ 837 event_register_handler, &name##_event_##lno.evt_link) 838 839 #ifdef MACH_BSD 840 841 /* BSD subsystem initialization */ 842 extern void bsd_init(void); 843 844 extern int serverperfmode; 845 846 #if defined(XNU_TARGET_OS_OSX) 847 static inline bool 848 kernel_is_macos_or_server(void) 849 { 850 return true; 851 } 852 #else /* XNU_TARGET_OS_OSX */ 853 static inline bool 854 kernel_is_macos_or_server(void) 855 { 856 return !!serverperfmode; 857 } 858 #endif /* XNU_TARGET_OS_OSX */ 859 860 #endif /* MACH_BSD */ 861 862 #pragma GCC visibility pop 863 864 __END_DECLS 865 866 #endif /* _KERN_STARTUP_H_ */ 867 868 #endif /* XNU_KERNEL_PRIVATE */ 869