1 /* 2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* 29 * @OSF_COPYRIGHT@ 30 */ 31 32 #ifdef XNU_KERNEL_PRIVATE 33 34 #ifndef _KERN_STARTUP_H_ 35 #define _KERN_STARTUP_H_ 36 37 #include <stdbool.h> 38 #include <stddef.h> 39 #include <stdint.h> 40 41 #include <libkern/section_keywords.h> 42 43 __BEGIN_DECLS 44 45 #pragma GCC visibility push(hidden) 46 47 /*! 48 * @enum startup_subsystem_id_t 49 * 50 * @abstract 51 * Represents a stage of kernel intialization, ubnd allows for subsystems 52 * to register initializers for a specific stage. 53 * 54 * @discussion 55 * Documentation of each subsystem initialization sequence exists in 56 * @file doc/startup.md. 57 */ 58 __enum_decl(startup_subsystem_id_t, uint32_t, { 59 STARTUP_SUB_NONE = 0, /**< reserved for the startup subsystem */ 60 61 STARTUP_SUB_TUNABLES, /**< support for the tunables subsystem */ 62 STARTUP_SUB_TIMEOUTS, /**< configurable machine timeouts */ 63 STARTUP_SUB_LOCKS, /**< various subsystem locks */ 64 STARTUP_SUB_KPRINTF, /**< kprintf initialization */ 65 66 STARTUP_SUB_PMAP_STEAL, /**< to perform various pmap carveouts */ 67 STARTUP_SUB_KMEM, /**< once kmem_alloc is ready */ 68 STARTUP_SUB_ZALLOC, /**< initialize zalloc and kalloc */ 69 STARTUP_SUB_KTRACE, /**< initialize kernel trace */ 70 STARTUP_SUB_PERCPU, /**< initialize the percpu subsystem */ 71 STARTUP_SUB_EVENT, /**< initiailze the event subsystem */ 72 73 STARTUP_SUB_CODESIGNING, /**< codesigning subsystem */ 74 STARTUP_SUB_OSLOG, /**< oslog and kernel logging */ 75 STARTUP_SUB_MACH_IPC, /**< Mach IPC */ 76 STARTUP_SUB_THREAD_CALL, /**< Thread calls */ 77 STARTUP_SUB_SYSCTL, /**< registers sysctls */ 78 STARTUP_SUB_EARLY_BOOT, /**< interrupts/preemption are turned on */ 79 80 STARTUP_SUB_LOCKDOWN = ~0u, /**< reserved for the startup subsystem */ 81 }); 82 83 /*! 84 * Stores the last subsystem to have been fully initialized; 85 */ 86 extern startup_subsystem_id_t startup_phase; 87 88 /*! 89 * @enum startup_debug_t 90 * 91 * @abstract 92 * Flags set in the @c startup_debug global to configure startup debugging. 93 */ 94 __options_decl(startup_debug_t, uint32_t, { 95 STARTUP_DEBUG_NONE = 0x00000000, 96 STARTUP_DEBUG_VERBOSE = 0x00000001, 97 }); 98 99 extern startup_debug_t startup_debug; 100 101 /*! 102 * @enum startup_rank 103 * 104 * @abstract 105 * Specifies in which rank a given initializer runs within a given section 106 * to register initializers for a specific rank within the subsystem. 107 * 108 * @description 109 * A startup function, declared with @c STARTUP or @c STARTUP_ARG, can specify 110 * a rank within the subsystem they initialize. 111 * 112 * @c STARTUP_RANK_NTH(n) will let callbacks be run at stage @c n (0-based). 113 * 114 * @c STARTUP_RANK_FIRST, @c STARTUP_RANK_SECOND, @c STARTUP_RANK_THIRD and 115 * @c STARTUP_RANK_FOURTH are given as convenient names for these. 116 * 117 * @c STARTUP_RANK_MIDDLE is a reserved value that will let startup functions 118 * run after all the @c STARTUP_RANK_NTH(n) ones have. 119 * 120 * @c STARTUP_RANK_NTH_LATE_NTH(n) will let callbacks be run then in @c n rank 121 * after the @c STARTUP_RANK_MIDDLE ones (0-based). 122 * 123 * @c STARTUP_RANK_LAST callbacks will run absolutely last after everything 124 * else did for this subsystem. 125 */ 126 __enum_decl(startup_rank_t, uint32_t, { 127 #define STARTUP_RANK_NTH(n) ((startup_rank_t)(n - 1)) 128 STARTUP_RANK_FIRST = 0, 129 STARTUP_RANK_SECOND = 1, 130 STARTUP_RANK_THIRD = 2, 131 STARTUP_RANK_FOURTH = 3, 132 133 STARTUP_RANK_MIDDLE = 0x7fffffff, 134 135 #define STARTUP_RANK_LATE_NTH(n) \ 136 ((startup_rank_t)(STARTUP_RANK_MIDDLE + 1 + (n))) 137 138 STARTUP_RANK_LAST = 0xffffffff, 139 }); 140 141 #if KASAN 142 /* 143 * The use of weird sections that get unmapped confuse the hell out of kasan, 144 * so for KASAN leave things in regular __TEXT/__DATA segments 145 */ 146 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 147 #define STARTUP_CONST_SEGSECT "__DATA_CONST,__init" 148 #define STARTUP_DATA_SEGSECT "__DATA,__init" 149 #define STARTUP_HOOK_SEGMENT "__DATA" 150 #define STARTUP_HOOK_SECTION "__init_entry_set" 151 #elif defined(__x86_64__) 152 /* Intel doesn't have a __BOOTDATA but doesn't protect __KLD */ 153 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 154 #define STARTUP_CONST_SEGSECT "__KLDDATA,__const" 155 #define STARTUP_DATA_SEGSECT "__KLDDATA,__init" 156 #define STARTUP_HOOK_SEGMENT "__KLDDATA" 157 #define STARTUP_HOOK_SECTION "__init_entry_set" 158 #else 159 /* arm protects __KLD early, so use __BOOTDATA for data */ 160 #define STARTUP_CODE_SEGSECT "__TEXT,__text" 161 #define STARTUP_CONST_SEGSECT "__KLDDATA,__const" 162 #define STARTUP_DATA_SEGSECT "__BOOTDATA,__init" 163 #define STARTUP_HOOK_SEGMENT "__BOOTDATA" 164 #define STARTUP_HOOK_SECTION "__init_entry_set" 165 #endif 166 167 /*! 168 * @macro __startup_func 169 * 170 * @abstract 171 * Attribute to place on functions used only during the kernel startup phase. 172 * 173 * @description 174 * Code marked with this attribute will be unmapped after kernel lockdown. 175 */ 176 #define __startup_func \ 177 __PLACE_IN_SECTION(STARTUP_CODE_SEGSECT) \ 178 __attribute__((cold, visibility("hidden"))) 179 180 /*! 181 * @macro __startup_data 182 * 183 * @abstract 184 * Attribute to place on globals used during the kernel startup phase. 185 * 186 * @description 187 * Data marked with this attribute will be unmapped after kernel lockdown. 188 */ 189 #define __startup_data \ 190 __PLACE_IN_SECTION(STARTUP_DATA_SEGSECT) 191 192 /*! 193 * @macro __startup_const 194 * 195 * @abstract 196 * Attribute to place on global constants used during the kernel startup phase. 197 * 198 * @description 199 * Data marked with this attribute will be unmapped after kernel lockdown. 200 * 201 * __startup_const implies const. Be mindful that for pointers, the `const` 202 * will end up on the wrong side of the star if you put __startup_const at the 203 * start of the declaration. 204 */ 205 #define __startup_const \ 206 __PLACE_IN_SECTION(STARTUP_CONST_SEGSECT) const 207 208 /*! 209 * @macro STARTUP 210 * 211 * @abstract 212 * Declares a kernel startup callback. 213 */ 214 #define STARTUP(subsystem, rank, func) \ 215 __STARTUP(func, __LINE__, subsystem, rank, func) 216 217 /*! 218 * @macro STARTUP_ARG 219 * 220 * @abstract 221 * Declares a kernel startup callback that takes an argument. 222 */ 223 #define STARTUP_ARG(subsystem, rank, func, arg) \ 224 __STARTUP_ARG(func, __LINE__, subsystem, rank, func, arg) 225 226 /*! 227 * @macro TUNABLE 228 * 229 * @abstract 230 * Declares a read-only kernel tunable that is read from a boot-arg with 231 * a default value, without further processing. 232 * 233 * @param type_t 234 * Should be an integer type or bool. 235 * 236 * @param var 237 * The name of the C variable to use for storage. 238 * 239 * @param boot_arg 240 * The name of the boot-arg to parse for initialization 241 * 242 * @param default_value 243 * The default value for the tunable if the boot-arg is absent. 244 */ 245 #define TUNABLE(type_t, var, boot_arg, default_value) \ 246 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \ 247 __TUNABLE(type_t, var, boot_arg) 248 249 /*! 250 * @macro TUNABLE_WRITEABLE 251 * 252 * @abstract 253 * Declares a writeable kernel tunable that is read from a boot-arg with 254 * a default value, without further processing. 255 * 256 * @param type_t 257 * Should be an integer type or bool. 258 * 259 * @param var 260 * The name of the C variable to use for storage. 261 * 262 * @param boot_arg 263 * The name of the boot-arg to parse for initialization 264 * 265 * @param default_value 266 * The default value for the tunable if the boot-arg is absent. 267 */ 268 #define TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) \ 269 type_t var = default_value; \ 270 __TUNABLE(type_t, var, boot_arg) 271 272 #if DEBUG || DEVELOPMENT 273 #define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \ 274 TUNABLE_WRITEABLE(type_t, var, boot_arg, default_value) 275 #else 276 #define TUNABLE_DEV_WRITEABLE(type_t, var, boot_arg, default_value) \ 277 TUNABLE(type_t, var, boot_arg, default_value) 278 #endif 279 280 /*! 281 * @macro TUNABLE_STR 282 * 283 * @abstract 284 * Declares a read-only kernel tunable that is read from a boot-arg with 285 * a default value, without further processing. 286 * 287 * @param var 288 * The name of the C variable to use for storage. 289 * 290 * @param count 291 * The number of bytes in the buffer. 292 * 293 * @param boot_arg 294 * The name of the boot-arg to parse for initialization 295 * 296 * @param default_value 297 * The default value for the tunable if the boot-arg is absent. 298 */ 299 #define TUNABLE_STR(var, count, boot_arg, default_value) \ 300 char __security_const_late var[count] = default_value; \ 301 __TUNABLE_STR(var, boot_arg) 302 303 /*! 304 * @enum tunable_dt_flags_t 305 * 306 * @abstract 307 * Flags used with the @c TUNABLE_DT* macros. 308 * 309 * @description 310 * If TUNABLE_DT_CHECK_CHOSEN is set, a value in 311 * /chosen/<dt_base>/<dt_name> takes precedence over any value in 312 * /<dt_base>/<dt_name>. /chosen is by convention the area where 313 * synthesized values not coming from the serialized device tree are 314 * being added, so this provides a way for e.g. the boot-loader to 315 * set/override tunables. 316 */ 317 __options_decl(tunable_dt_flags_t, uint32_t, { 318 TUNABLE_DT_NONE = 0x00000000, 319 TUNABLE_DT_CHECK_CHOSEN = 0x00000001, 320 }); 321 322 /*! 323 * @macro TUNABLE_DT 324 * 325 * @abstract 326 * Like TUNABLE, but gets the initial value from both Device Tree and 327 * boot-args. The order in which the initial value is resolved is as 328 * follows, with later steps overriding previous ones (if they are 329 * specified): 330 * 331 * 1. Device Tree Entry "/<dt_base>/<dt_name>", 332 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry 333 * "/chosen/<dt_base>/<dt_name>" (see the description for 334 * @c tunable_dt_flags_t), 335 * 3. boot-args. 336 * 337 * @param type_t 338 * Should be an integer type or bool. 339 * 340 * @param var 341 * The name of the C variable to use for storage. 342 * 343 * @param dt_base 344 * The name of the DT node containing the property. 345 * 346 * @param dt_name 347 * The name of the DT property containing the default value. 348 * 349 * @param boot_arg 350 * The name of the boot-arg overriding the initial value from the DT. 351 * 352 * @param default_value 353 * The default value for the tunable if both DT entry and boot-arg are 354 * absent. 355 * 356 * @param flags 357 * See the description for @c tunable_dt_flags_t. 358 */ 359 #define TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 360 SECURITY_READ_ONLY_LATE(type_t) var = default_value; \ 361 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags) 362 363 /*! 364 * @macro TUNABLE_DT_WRITEABLE 365 * 366 * @abstract 367 * Like TUNABLE_WRITEABLE, but gets the initial value from both Device 368 * Tree and boot-args. The order in which the initial value is 369 * resolved is as follows, with later steps overriding previous ones 370 * (if they are specified): 371 * 372 * 1. Device Tree Entry "/<dt_base>/<dt_name>", 373 * 2. If TUNABLE_DT_CHECK_CHOSEN is set, Device Tree Entry 374 * "/chosen/<dt_base>/<dt_name>" (see the description for 375 * @c tunable_dt_flags_t), 376 * 3. boot-args. 377 * 378 * @param type_t 379 * Should be an integer type or bool. 380 * 381 * @param var 382 * The name of the C variable to use for storage. 383 * 384 * @param dt_base 385 * The name of the DT node containing the property. 386 * 387 * @param dt_name 388 * The name of the DT property containing the default value. 389 * 390 * @param boot_arg 391 * The name of the boot-arg overriding the initial value from the DT. 392 * 393 * @param default_value 394 * The default value for the tunable if both DT entry and boot-arg are 395 * absent. 396 * 397 * @param flags 398 * See the description for @c tunable_dt_flags_t. 399 */ 400 #define TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 401 type_t var = default_value; \ 402 __TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, flags) 403 404 #if DEBUG || DEVELOPMENT 405 #define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 406 TUNABLE_DT_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) 407 #else 408 #define TUNABLE_DT_DEV_WRITEABLE(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) \ 409 TUNABLE_DT(type_t, var, dt_base, dt_name, boot_arg, default_value, flags) 410 #endif 411 412 /* 413 * Machine Timeouts 414 * 415 * Machine Timeouts are timeouts for low level kernel code manifesting 416 * as _Atomic uint64_t variables, whose default value can be 417 * overridden and scaled via the device tree and boot-args. 418 * 419 * Each timeout has a name, looked up directly as the property name in 420 * the device tree in both the "/machine-timeouts" and 421 * "/chosen/machine-timeouts" nodes. The "chosen" property always 422 * overrides the other one. This allows fixed per-device timeouts in 423 * the device tree to be overridden by iBoot in "chosen". 424 * 425 * Additionally, the same name with "-scale" appended is looked up as 426 * properties for optional scale factors. Scale factors are not 427 * overridden by chosen, instead all scale factors (including global 428 * and/or boot-arg scale factors) combine by multiplication. 429 * 430 * The special name "global-scale" provides a scale that applies to 431 * every timeout. 432 * 433 * All property names can be used as boot-args by prefixing 434 * "ml-timeout-", e.g. th global scale is available as the 435 * "ml-timeout-global-scale" boot-arg. 436 * 437 * By convention, if the timeout value resolves to 0, the timeout 438 * should be disabled. 439 */ 440 441 /* 442 * Machine Timeouts types. See the next section for what unit 443 * they are in. 444 * 445 * We use _Atomic, but only with relaxed ordering: This is just to 446 * make sure all devices see consistent values all the time. Since 447 * the actual timeout value will be seen as 0 before initializaton, 448 * relaxed ordering means that code that runs concurrently with 449 * initialization only risks to see a disabled timeout during early 450 * boot. 451 */ 452 typedef _Atomic uint64_t machine_timeout_t; 453 454 /* 455 * Units 456 * 457 * Machine Timeouts are ALWAYS in picoseconds in the device tree or 458 * boot-args, to avoid confusion when changing or comparing timeouts 459 * as a user, but the actual storage value might contain the same 460 * duration in another unit, calculated by the initialization code. 461 * 462 * This is done because otherwise we would likely introduce another 463 * multiplication in potentially hot code paths, given that code that 464 * actually uses the timeout storage variable is unlikely to work with 465 * picosecond values when comparing against the timeout deadline. 466 * 467 * This unit scale is *only* applied during initialization at early 468 * boot, and only if the timeout's default value was overridden 469 * through the device tree or a boot-arg. 470 */ 471 #define MACHINE_TIMEOUT_UNIT_PSEC 1 472 #define MACHINE_TIMEOUT_UNIT_NSEC 1000 473 #define MACHINE_TIMEOUT_UNIT_USEC (1000*1000) 474 #define MACHINE_TIMEOUT_UNIT_MSEC (1000*1000*1000) 475 // Special unit for timebase ticks (usually 1/24MHz) 476 #define MACHINE_TIMEOUT_UNIT_TIMEBASE 0 477 478 // DT property names are limited to 31 chars, minus "-global" suffix 479 #define MACHINE_TIMEOUT_MAX_NAME_LEN 25 480 struct machine_timeout_spec { 481 void *ptr; 482 uint64_t default_value; 483 uint64_t unit_scale; 484 char name[MACHINE_TIMEOUT_MAX_NAME_LEN + 1]; 485 bool (*skip_predicate)(struct machine_timeout_spec const *); 486 }; 487 488 extern void 489 machine_timeout_init_with_suffix(const struct machine_timeout_spec *spec, char const *phase_suffix); 490 491 extern void 492 machine_timeout_init(const struct machine_timeout_spec *spec); 493 494 #if DEVELOPMENT || DEBUG 495 // Late timeout (re-)initialization, at the end of bsd_init() 496 extern void 497 machine_timeout_bsd_init(void); 498 #endif /* DEVELOPMENT || DEBUG */ 499 500 /*! 501 * @macro MACHINE_TIMEOUT and MACHINE_TIMEOUT_DEV_WRITEABLE 502 * 503 * @abstract 504 * Defines a Machine Timeout that can be overridden and 505 * scaled through the device tree and boot-args. 506 * 507 * The variant with the _DEV_WRITEABLE suffix does not mark the timeout as 508 * SECURITY_READ_ONLY_LATE on DEVELOPMENT kernels, so that e.g. 509 * machine_timeout_init_with_suffix or sysctls can change it after lockdown. 510 * 511 * @param var 512 * The name of the C variable to use for storage. If the storage value 513 * contains 0, the timeout is considered disabled by convention. 514 * 515 * @param timeout_name 516 * The name of the timeout, used for property and boot-arg names. See 517 * the general description of Machine Timeouts above for how this name 518 * ends up being used. 519 * 520 * @param timeout_default 521 * The default value for the timeout if not specified through device 522 * tree or boot-arg. Will still be scaled if a scale factor exists. 523 * 524 * @param var_unit 525 * The unit that the storage variable is in. Note that timeout values 526 * must always be specified as picoseconds in the device tree and 527 * boot-args, but timeout initialization will convert the value to the 528 * unit specified here before writing it to the storage variable. 529 * 530 * @param skip_predicate 531 * Optionally, a function to call to decide whether the timeout should 532 * be set or not. If NULL, the timeout will always be set (if 533 * specified anywhere). A predicate has the following signature: 534 * bool skip_predicate (struct machine_timeout_spec const *) 535 */ 536 537 #define _MACHINE_TIMEOUT(var, timeout_name, timeout_default, var_unit, skip_pred) \ 538 struct machine_timeout_spec \ 539 __machine_timeout_spec_ ## var = { \ 540 .ptr = &var, \ 541 .default_value = timeout_default, \ 542 .unit_scale = var_unit, \ 543 .name = timeout_name, \ 544 .skip_predicate = skip_pred, \ 545 }; \ 546 __STARTUP_ARG(var, __LINE__, TIMEOUTS, STARTUP_RANK_FIRST, \ 547 machine_timeout_init, &__machine_timeout_spec_ ## var) 548 549 #define MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) \ 550 SECURITY_READ_ONLY_LATE(machine_timeout_t) var = 0; \ 551 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 552 553 #if DEVELOPMENT || DEBUG 554 #define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \ 555 machine_timeout_t var = 0; \ 556 _MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 557 #else 558 #define MACHINE_TIMEOUT_DEV_WRITEABLE(var, name, default, unit, skip_predicate) \ 559 MACHINE_TIMEOUT(var, name, default, unit, skip_predicate) 560 #endif /* DEVELOPMENT || DEBUG */ 561 562 /*! 563 * @macro MACHINE_TIMEOUT_SPEC_REF 564 * 565 * @abstract 566 * References a previously defined MACHINE_TIMEOUT. 567 * 568 * This is primarily useful for overriding individual timeouts 569 * at arbitrary times (even after boot), by manually calling 570 * machine_timeout_init_with_suffix() with this macro 571 * as first argument, and a suffix to apply to both device tree and 572 * boot-arg as second argument. 573 * 574 * @param var 575 * The name of the C variable used for storage, as it was specified 576 * in MACHINE_TIMEOUT. 577 */ 578 #define MACHINE_TIMEOUT_SPEC_REF(var) (&__machine_timeout_spec_ ## var) 579 580 /*! 581 * @macro MACHINE_TIMEOUT_SPEC_DECL 582 * 583 * @abstract 584 * Declaration of machine timeout spec, mostly useful to make it known 585 * for MACHINE_TIMEOUT_SPEC_REF. 586 * 587 * @param var 588 * The name of the C variable used for storage, as it was specified 589 * in MACHINE_TIMEOUT. 590 */ 591 #define MACHINE_TIMEOUT_SPEC_DECL(var) extern struct machine_timeout_spec __machine_timeout_spec_ ## var 592 593 /* 594 * Event subsystem 595 * 596 * This allows to define a few system-wide events that allow for loose coupling 597 * between subsystems interested in those events and the emitter. 598 */ 599 600 /*! 601 * @macro EVENT_DECLARE() 602 * 603 * @brief 604 * Declares an event namespace with a given callback type. 605 * 606 * @param name The name for the event (typically in all caps). 607 * @param cb_type_t A function type for the callbacks. 608 */ 609 #define EVENT_DECLARE(name, cb_type_t) \ 610 struct name##_event { \ 611 struct event_hdr evt_link; \ 612 cb_type_t *evt_cb; \ 613 }; \ 614 extern struct event_hdr name##_HEAD 615 616 /*! 617 * @macro EVENT_DEFINE() 618 * 619 * @brief 620 * Defines the head for the event corresponding to an EVENT_DECLARE() 621 */ 622 #define EVENT_DEFINE(name) \ 623 __security_const_late struct event_hdr name##_HEAD 624 625 /*! 626 * @macro EVENT_REGISTER_HANDLER() 627 * 628 * @brief 629 * Registers a handler for a given event. 630 * 631 * @param name The name for the event as declared in EVENT_DECLARE() 632 * @param handler The handler to register for this event. 633 */ 634 #define EVENT_REGISTER_HANDLER(name, handler) \ 635 __EVENT_REGISTER(name, __LINE__, handler) 636 637 638 /*! 639 * @macro EVENT_INVOKE() 640 * 641 * @brief 642 * Call all the events handlers with the specified arguments. 643 * 644 * @param name The name for the event as declared in EVENT_DECLARE() 645 * @param handler The handler to register for this event. 646 */ 647 #define EVENT_INVOKE(name, ...) \ 648 for (struct event_hdr *__e = &name##_HEAD; (__e = __e->next);) { \ 649 __container_of(__e, struct name##_event, \ 650 evt_link)->evt_cb(__VA_ARGS__); \ 651 } 652 653 654 #if DEBUG || DEVELOPMENT 655 656 /*! 657 * @macro SYSCTL_TEST_REGISTER 658 * 659 * @abstract 660 * Declares a test that will appear under @c debug.test.${name}. 661 * 662 * @param name 663 * An identifier that will be stringified to form the sysctl test name. 664 * 665 * @param cb 666 * The callback to run, of type: 667 * <code> 668 * int (callback *)(int64_t value, int64_t *); 669 * </code> 670 */ 671 #define SYSCTL_TEST_REGISTER(name, cb) \ 672 static __startup_data struct sysctl_test_setup_spec \ 673 __startup_SYSCTL_TEST_ ## name = { \ 674 .st_name = #name, \ 675 .st_func = &cb, \ 676 }; \ 677 STARTUP_ARG(SYSCTL, STARTUP_RANK_MIDDLE, \ 678 sysctl_register_test_startup, &__startup_SYSCTL_TEST_ ## name) 679 680 #endif /* DEBUG || DEVELOPMENT */ 681 #pragma mark - internals 682 683 __END_DECLS 684 685 #ifdef __cplusplus 686 template <typename T> 687 struct __startup_tunable { 688 static const bool value = false; 689 }; 690 691 template <> 692 struct __startup_tunable <bool>{ 693 static const bool value = true; 694 }; 695 #define __startup_type_is_bool(type_t) __startup_tunable<type_t>::value 696 #else 697 #define __startup_type_is_bool(type_t) __builtin_types_compatible_p(bool, type_t) 698 #endif 699 700 __BEGIN_DECLS 701 702 #define __TUNABLE(type_t, var, key) \ 703 static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \ 704 static __startup_const struct startup_tunable_spec \ 705 __startup_TUNABLES_spec_ ## var = { \ 706 .name = __startup_TUNABLES_name_ ## var, \ 707 .var_addr = (void *)&var, \ 708 .var_len = sizeof(type_t), \ 709 .var_is_bool = __startup_type_is_bool(type_t), \ 710 }; \ 711 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 712 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var) 713 714 #define __TUNABLE_STR(var, key) \ 715 static __startup_const char __startup_TUNABLES_name_ ## var[] = key; \ 716 static __startup_const struct startup_tunable_spec \ 717 __startup_TUNABLES_spec_ ## var = { \ 718 .name = __startup_TUNABLES_name_ ## var, \ 719 .var_addr = (void *)&var, \ 720 .var_len = sizeof(var), \ 721 .var_is_str = true, \ 722 }; \ 723 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 724 kernel_startup_tunable_init, &__startup_TUNABLES_spec_ ## var) 725 726 #define __TUNABLE_DT(type_t, var, dt_base_key, dt_name_key, boot_arg_key, flags) \ 727 static __startup_const char __startup_TUNABLES_dt_base_ ## var[] = dt_base_key; \ 728 static __startup_const char __startup_TUNABLES_dt_name_ ## var[] = dt_name_key; \ 729 static __startup_const char __startup_TUNABLES_name_ ## var[] = boot_arg_key; \ 730 static __startup_const struct startup_tunable_dt_spec \ 731 __startup_TUNABLES_DT_spec_ ## var = { \ 732 .dt_base = __startup_TUNABLES_dt_base_ ## var, \ 733 .dt_name = __startup_TUNABLES_dt_name_ ## var, \ 734 .dt_chosen_override = (bool)((flags) & TUNABLE_DT_CHECK_CHOSEN), \ 735 .boot_arg_name = __startup_TUNABLES_name_ ## var, \ 736 .var_addr = (void *)&var, \ 737 .var_len = sizeof(type_t), \ 738 .var_is_bool = __startup_type_is_bool(type_t), \ 739 }; \ 740 __STARTUP_ARG(var, __LINE__, TUNABLES, STARTUP_RANK_FIRST, \ 741 kernel_startup_tunable_dt_init, &__startup_TUNABLES_DT_spec_ ## var) 742 743 #ifdef __cplusplus 744 #define __STARTUP_FUNC_CAST(func, a) \ 745 (void(*)(const void *))func 746 #else 747 #define __STARTUP_FUNC_CAST(func, a) \ 748 (typeof(func(a))(*)(const void *))func 749 #endif 750 751 752 #define __STARTUP1(name, line, subsystem, rank, func, a, b) \ 753 __PLACE_IN_SECTION(STARTUP_HOOK_SEGMENT "," STARTUP_HOOK_SECTION) \ 754 static const struct startup_entry \ 755 __startup_ ## subsystem ## _entry_ ## name ## _ ## line = { \ 756 STARTUP_SUB_ ## subsystem, \ 757 rank, __STARTUP_FUNC_CAST(func, a), b, \ 758 } 759 760 #define __STARTUP(name, line, subsystem, rank, func) \ 761 __STARTUP1(name, line, subsystem, rank, func, , NULL) 762 763 #define __STARTUP_ARG(name, line, subsystem, rank, func, arg) \ 764 __STARTUP1(name, line, subsystem, rank, func, arg, arg) 765 766 struct startup_entry { 767 startup_subsystem_id_t subsystem; 768 startup_rank_t rank; 769 void (*func)(const void *); 770 const void *arg; 771 }; 772 773 struct startup_tunable_spec { 774 const char *name; 775 void *var_addr; 776 int var_len; 777 bool var_is_bool; 778 bool var_is_str; 779 }; 780 781 struct startup_tunable_dt_spec { 782 const char *dt_base; 783 const char *dt_name; 784 bool dt_chosen_override; 785 const char *boot_arg_name; 786 void *var_addr; 787 int var_len; 788 bool var_is_bool; 789 }; 790 791 #if DEBUG || DEVELOPMENT 792 struct sysctl_test_setup_spec { 793 const char *st_name; 794 int (*st_func)(int64_t, int64_t *); 795 }; 796 797 extern void sysctl_register_test_startup( 798 struct sysctl_test_setup_spec *spec); 799 #endif /* DEBUG || DEVELOPMENT */ 800 801 /* 802 * Kernel and machine startup declarations 803 */ 804 805 /* Initialize kernel */ 806 extern void kernel_startup_bootstrap(void); 807 extern void kernel_startup_initialize_upto(startup_subsystem_id_t upto); 808 extern void kernel_startup_tunable_init(const struct startup_tunable_spec *); 809 extern void kernel_startup_tunable_dt_init(const struct startup_tunable_dt_spec *); 810 extern void kernel_bootstrap(void); 811 812 /* Initialize machine dependent stuff */ 813 extern void machine_init(void); 814 815 extern void slave_main(void *machine_param); 816 817 /* 818 * The following must be implemented in machine dependent code. 819 */ 820 821 /* Slave cpu initialization */ 822 extern void slave_machine_init(void *machine_param); 823 824 /* Device subsystem initialization */ 825 extern void device_service_create(void); 826 827 struct event_hdr { 828 struct event_hdr *next; 829 }; 830 831 extern void event_register_handler(struct event_hdr *event_hdr); 832 833 #define __EVENT_REGISTER(name, lno, handler) \ 834 static __security_const_late struct name##_event name##_event_##lno = { \ 835 .evt_link.next = &name##_HEAD, \ 836 .evt_cb = (handler), \ 837 }; \ 838 __STARTUP_ARG(name, lno, EVENT, STARTUP_RANK_FIRST, \ 839 event_register_handler, &name##_event_##lno.evt_link) 840 841 #ifdef MACH_BSD 842 843 /* BSD subsystem initialization */ 844 extern void bsd_init(void); 845 846 #endif /* MACH_BSD */ 847 848 #pragma GCC visibility pop 849 850 __END_DECLS 851 852 #endif /* _KERN_STARTUP_H_ */ 853 854 #endif /* XNU_KERNEL_PRIVATE */ 855