1 /*
2 * Copyright (c) 2021-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _KERN_SMR_H_
30 #define _KERN_SMR_H_
31
32 #include <sys/cdefs.h>
33 #include <stdbool.h>
34 #include <stdint.h>
35 #include <kern/assert.h>
36 #include <kern/debug.h>
37 #include <kern/smr_types.h>
38 #include <kern/startup.h>
39 #include <os/atomic_private.h>
40
41 __BEGIN_DECLS
42
43 #pragma mark SMR pointers
44
45 /*
46 * SMR Accessors are meant to provide safe access to SMR protected
47 * pointers and prevent misuse and accidental access.
48 *
49 * Accessors are grouped by type:
50 * entered - Use while in a read section (between smr_enter/smr_leave())
51 * serialized - Use while holding a lock that serializes writers.
52 * Updates are synchronized with readers via included barriers.
53 * unserialized - Use after the memory is out of scope and not visible to
54 * readers.
55 *
56 * All acceses include a parameter for an assert to verify the required
57 * synchronization.
58 */
59
60
61 /*!
62 * @macro smr_unsafe_load()
63 *
64 * @brief
65 * Read from an SMR protected pointer without any synchronization.
66 *
67 * @discussion
68 * This returns an integer on purpose as dereference is generally unsafe.
69 */
70 #define smr_unsafe_load(ptr) \
71 ({ (uintptr_t)((ptr)->__smr_ptr); })
72
73 /*!
74 * @macro smr_entered_load()
75 *
76 * @brief
77 * Read from an SMR protected pointer while in a read section.
78 */
79 #define smr_entered_load(ptr) \
80 ({ (ptr)->__smr_ptr; })
81
82 /*!
83 * @macro smr_entered_load_assert()
84 *
85 * @brief
86 * Read from an SMR protected pointer while in a read section.
87 */
88 #define smr_entered_load_assert(ptr, smr) ({ \
89 assert(smr_entered(smr)); \
90 (ptr)->__smr_ptr; \
91 })
92
93 /*!
94 * @macro smr_entered_load_acquire()
95 *
96 * @brief
97 * Read from an SMR protected pointer while in a read section (with acquire
98 * fence).
99 */
100 #define smr_entered_load_acquire(ptr) \
101 os_atomic_load(&(ptr)->__smr_ptr, acquire)
102
103 /*!
104 * @macro smr_entered_load_acquire_assert()
105 *
106 * @brief
107 * Read from an SMR protected pointer while in a read section.
108 */
109 #define smr_entered_load_acquire_assert(ptr, smr) ({ \
110 assert(smr_entered(smr)); \
111 os_atomic_load(&(ptr)->__smr_ptr, acquire); \
112 })
113
114 /*!
115 * @macro smr_serialized_load_assert()
116 *
117 * @brief
118 * Read from an SMR protected pointer while serialized by an
119 * external mechanism.
120 */
121 #define smr_serialized_load_assert(ptr, held_cond) ({ \
122 assertf(held_cond, "smr_serialized_load: lock not held"); \
123 (ptr)->__smr_ptr; \
124 })
125
126 /*!
127 * @macro smr_serialized_load()
128 *
129 * @brief
130 * Read from an SMR protected pointer while serialized by an
131 * external mechanism.
132 */
133 #define smr_serialized_load(ptr) \
134 smr_serialized_load_assert(ptr, true)
135
136 /*!
137 * @macro smr_init_store()
138 *
139 * @brief
140 * Store @c value to an SMR protected pointer during initialization.
141 */
142 #define smr_init_store(ptr, value) \
143 ({ (ptr)->__smr_ptr = value; })
144
145 /*!
146 * @macro smr_clear_store()
147 *
148 * @brief
149 * Clear (sets to 0) an SMR protected pointer (this is always "allowed" to do).
150 */
151 #define smr_clear_store(ptr) \
152 smr_init_store(ptr, 0)
153
154 /*!
155 * @macro smr_serialized_store_assert()
156 *
157 * @brief
158 * Store @c value to an SMR protected pointer while serialized by an
159 * external mechanism.
160 *
161 * @discussion
162 * Writers that are serialized with mutual exclusion or on a single
163 * thread should use smr_serialized_store() rather than swap.
164 */
165 #define smr_serialized_store_assert(ptr, value, held_cond) ({ \
166 assertf(held_cond, "smr_serialized_store: lock not held"); \
167 os_atomic_thread_fence(release); \
168 (ptr)->__smr_ptr = value; \
169 })
170
171 /*!
172 * @macro smr_serialized_store()
173 *
174 * @brief
175 * Store @c value to an SMR protected pointer while serialized by an
176 * external mechanism.
177 *
178 * @discussion
179 * Writers that are serialized with mutual exclusion or on a single
180 * thread should use smr_serialized_store() rather than swap.
181 */
182 #define smr_serialized_store(ptr, value) \
183 smr_serialized_store_assert(ptr, value, true)
184
185 /*!
186 * @macro smr_serialized_store_relaxed_assert()
187 *
188 * @brief
189 * Store @c value to an SMR protected pointer while serialized by an
190 * external mechanism.
191 *
192 * @discussion
193 * This function can be used when storing a value that was already
194 * previously stored with smr_serialized_store() (for example during
195 * a linked list removal).
196 */
197 #define smr_serialized_store_relaxed_assert(ptr, value, held_cond) ({ \
198 assertf(held_cond, "smr_serialized_store_relaxed: lock not held"); \
199 (ptr)->__smr_ptr = value; \
200 })
201
202 /*!
203 * @macro smr_serialized_store_relaxed()
204 *
205 * @brief
206 * Store @c value to an SMR protected pointer while serialized by an
207 * external mechanism.
208 *
209 * @discussion
210 * This function can be used when storing a value that was already
211 * previously stored with smr_serialized_store() (for example during
212 * a linked list removal).
213 */
214 #define smr_serialized_store_relaxed(ptr, value) \
215 smr_serialized_store_relaxed_assert(ptr, value, true)
216
217 /*!
218 * @macro smr_serialized_swap_assert()
219 *
220 * @brief
221 * Swap @c value with an SMR protected pointer and return the old value
222 * while serialized by an external mechanism.
223 *
224 * @discussion
225 * Swap permits multiple writers to update a pointer concurrently.
226 */
227 #define smr_serialized_swap_assert(ptr, value, held_cond) ({ \
228 assertf(held_cond, "smr_serialized_store: lock not held"); \
229 os_atomic_xchg(&(ptr)->__smr_ptr, value, release); \
230 })
231
232 /*!
233 * @macro smr_serialized_swap()
234 *
235 * @brief
236 * Swap @c value with an SMR protected pointer and return the old value
237 * while serialized by an external mechanism.
238 *
239 * @discussion
240 * Swap permits multiple writers to update a pointer concurrently.
241 */
242 #define smr_serialized_swap(ptr, value) \
243 smr_serialized_swap_assert(ptr, value, true)
244
245 /*!
246 * @macro smr_unserialized_load()
247 *
248 * @brief.
249 * Read from an SMR protected pointer when no serialization is required
250 * such as in the destructor callback or when the caller guarantees other
251 * synchronization.
252 */
253 #define smr_unserialized_load(ptr) \
254 ({ (ptr)->__smr_ptr; })
255
256 /*!
257 * @macro smr_unserialized_store()
258 *
259 * @brief.
260 * Store to an SMR protected pointer when no serialiation is required
261 * such as in the destructor callback or when the caller guarantees other
262 * synchronization.
263 */
264 #define smr_unserialized_store(ptr, value) \
265 ({ (ptr)->__smr_ptr = value; })
266
267
268 #pragma mark SMR queues
269
270 /*
271 * SMR queues are queues that are meant to be read under SMR critical sections
272 * concurrently with possible updates to the queue.
273 *
274 * /!\ Such read operations CAN ONLY BE PERFORMED IN FORWARD DIRECTION. /!\
275 *
276 * Queues can be either:
277 * - lists where the head is a single pointer,
278 * and insertions can only be at the head;
279 * - tail queues where the head is two pointers,
280 * and insertions can be either at the head or the tail.
281 *
282 * Queue linkages can either be single forward pointer linkages or double
283 * forward/backward linkages. The latter supports O(1) deletion.
284 *
285 *
286 * The entire API surface uses type inference for the implementations,
287 * which allows to relatively easily change between the 4 types of queues
288 * with very minimal API changes (mostly the types of list heads and fields).
289 */
290
291
292 /*!
293 * @macro smrq_init
294 *
295 * @brief
296 * Initializes an SMR queue head.
297 */
298 #define smrq_init(head) ({ \
299 __auto_type __head = (head); \
300 \
301 smr_init_store(&__head->first, NULL); \
302 if (__smrq_lastp(__head)) { \
303 *__smrq_lastp(__head) = &__head->first; \
304 } \
305 })
306
307
308 /*!
309 * @macro smrq_empty
310 *
311 * @brief
312 * Returns whether an SMR queue is empty, can be called from any context.
313 */
314 #define smrq_empty(head) \
315 (smr_unsafe_load(&(head)->first) == 0)
316
317
318 /*!
319 * @macro smrq_entered_first
320 *
321 * @brief
322 * Returns the first element of an SMR queue, while in a read section.
323 */
324 #define smrq_entered_first(head, type_t, field) \
325 __container_of_safe(smr_entered_load(&(head)->first), type_t, field)
326
327
328 /*!
329 * @macro smrq_entered_next
330 *
331 * @brief
332 * Returns the next element of an SMR queue element, while in a read section.
333 */
334 #define smrq_entered_next(elem, field) \
335 __container_of_safe(smr_entered_load(&(elem)->field.next), \
336 typeof(*(elem)), field)
337
338
339 /*!
340 * @macro smrq_entered_foreach
341 *
342 * @brief
343 * Enumerates an SMR queue, while in a read section.
344 */
345 #define smrq_entered_foreach(it, head, field) \
346 for (__auto_type __it = smr_entered_load(&(head)->first); \
347 ((it) = __container_of_safe(__it, typeof(*(it)), field)); \
348 __it = smr_entered_load(&__it->next))
349
350
351 /*!
352 * @macro smrq_serialized_first
353 *
354 * @brief
355 * Returns the first element of an SMR queue, while being serialized
356 * by an external mechanism.
357 */
358 #define smrq_serialized_first(head, type_t, link) \
359 __container_of_safe(smr_serialized_load(&(head)->first), type_t, link)
360
361 /*!
362 * @macro smrq_serialized_next
363 *
364 * @brief
365 * Returns the next element of an SMR queue element, while being serialized
366 * by an external mechanism.
367 */
368 #define smrq_serialized_next(elem, field) \
369 __container_of_safe(smr_serialized_load(&(elem)->field.next), \
370 typeof(*(elem)), field)
371
372 /*!
373 * @macro smrq_serialized_foreach
374 *
375 * @brief
376 * Enumerates an SMR queue, while being serialized
377 * by an external mechanism.
378 */
379 #define smrq_serialized_foreach(it, head, field) \
380 for (__auto_type __it = smr_serialized_load(&(head)->first); \
381 ((it) = __container_of_safe(__it, typeof(*(it)), field)); \
382 __it = smr_serialized_load(&__it->next))
383
384 /*!
385 * @macro smrq_serialized_foreach_safe
386 *
387 * @brief
388 * Enumerates an SMR queue, while being serialized
389 * by an external mechanism.
390 *
391 * @discussion
392 * This variant supports removing the current element from the queue.
393 */
394 #define smrq_serialized_foreach_safe(it, head, field) \
395 for (__auto_type __it = smr_serialized_load(&(head)->first), \
396 __next_it = __it; \
397 ((it) = __container_of_safe(__it, typeof(*(it)), field)) && \
398 ((__next_it = smr_serialized_load(&__it->next)), 1); \
399 __it = __next_it)
400
401
402 /*!
403 * @macro smrq_serialized_insert_head
404 *
405 * @brief
406 * Inserts an element at the head of an SMR queue, while being serialized
407 * by an external mechanism.
408 */
409 #define smrq_serialized_insert_head(head, elem) ({ \
410 __auto_type __head = (head); \
411 \
412 __smrq_serialized_insert(&__head->first, (elem), \
413 smr_serialized_load(&__head->first), __smrq_lastp(__head)); \
414 })
415
416
417 /*!
418 * @macro smrq_serialized_insert_tail
419 *
420 * @brief
421 * Inserts an element at the tail of an SMR queue, while being serialized
422 * by an external mechanism.
423 */
424 #define smrq_serialized_insert_tail(head, elem) ({ \
425 __auto_type __head = (head); \
426 \
427 __smrq_serialized_insert(__head->last, (elem), \
428 NULL, &__head->last); \
429 })
430
431
432 /*!
433 * @macro smrq_serialized_insert_head_relaxed
434 *
435 * @brief
436 * Inserts an element at the head of an SMR queue, while being serialized
437 * by an external mechanism, without any barrier.
438 */
439 #define smrq_serialized_insert_head_relaxed(head, elem) ({ \
440 __auto_type __head = (head); \
441 \
442 __smrq_serialized_insert_relaxed(&__head->first, (elem), \
443 smr_serialized_load(&__head->first), __smrq_lastp(__head)); \
444 })
445
446
447 /*!
448 * @macro smrq_serialized_insert_tail_relaxed
449 *
450 * @brief
451 * Inserts an element at the tail of an SMR queue, while being serialized
452 * by an external mechanism, without any barrier.
453 */
454 #define smrq_serialized_insert_tail_relaxed(head, elem) ({ \
455 __auto_type __head = (head); \
456 \
457 __smrq_serialized_insert_relaxed(__head->last, (elem), \
458 NULL, &__head->last); \
459 })
460
461
462 /*!
463 * @macro smrq_serialized_remove
464 *
465 * @brief
466 * Removes an element from an SMR queue, while being serialized
467 * by an external mechanism.
468 *
469 * @discussion
470 * The @c head argument is actually unused for the @c smrq_list queue type.
471 * It is still advised to pass it, the compiler should be able to optimize
472 * the code away as computing a list head ought to have no side effects.
473 */
474 #define smrq_serialized_remove(head, elem) ({ \
475 __auto_type __head = (head); \
476 \
477 __smrq_serialized_remove(&__head->first, (elem), __smrq_lastp(__head)); \
478 })
479
480
481 /*!
482 * @macro smrq_serialized_replace
483 *
484 * @brief
485 * Replaces an element on an SMR queue with another at the same spot,
486 * while being serialized by an external mechanism.
487 */
488 #define smrq_serialized_replace(head, old_elem, new_elem) ({ \
489 __auto_type __head = (head); \
490 \
491 __smrq_serialized_replace(&__head->first, \
492 (old_elem), (new_elem), __smrq_lastp(__head)); \
493 })
494
495
496 /*!
497 * @macro smrq_serialized_iter
498 *
499 * @brief
500 * Enumerates an SMR singly linked queue, while being serialized
501 * by an external mechanism.
502 *
503 * @discussion
504 * This is for manual loops that typically perform erasures.
505 *
506 * The body of the loop must move the cursor using (once):
507 * - smrq_serialized_iter_next() to to go the next element,
508 * - smrq_serialized_iter_erase() to erase the current element.
509 *
510 * The iterator variable will _not_ be updated until the next
511 * loop iteration.
512 *
513 * This form is preferred to smrq_serialized_foreach_safe()
514 * for singly linked lists as smrq_serialized_iter_erase()
515 * is O(1) as opposed to smrq_serialized_remove().
516 */
517 #define smrq_serialized_iter(it, head, field) \
518 for (__smrq_slink_t *__prev_##it = &(head)->first, \
519 *__chk_##it = __prev_##it; \
520 ((it) = __container_of_safe(smr_serialized_load(__prev_##it), \
521 typeof(*(it)), field)); \
522 assert(__chk_##it), __chk_##it = __prev_##it)
523
524 /*!
525 * @macro smrq_serialized_iter_next
526 *
527 * @brief
528 * Goes to the next element inside an smrq_serialied_iter() loop.
529 */
530 #define smrq_serialized_iter_next(it, field) ({ \
531 assert(__chk_##it == __prev_##it); \
532 __chk_##it = NULL; \
533 __prev_##it = &(it)->field.next; \
534 })
535
536 /*!
537 * @macro smrq_serialized_iter_erase
538 *
539 * @brief
540 * Erases the element pointed at by the cursor.
541 */
542 #define smrq_serialized_iter_erase(it, field) ({ \
543 assert(__chk_##it == __prev_##it); \
544 __chk_##it = NULL; \
545 __smrq_serialized_remove_one(__prev_##it, &(it)->field, NULL); \
546 })
547
548
549 /*!
550 * @macro smrq_serialized_append
551 *
552 * @brief
553 * Appends a given list at the end of the previous one.
554 *
555 * @discussion
556 * /!\ WARNING /!\: this doesn't "move" the "source" queue like *_CONCAT
557 * for <sys/queue.h>, as it is useful to merge/split hash queues concurrently
558 * with readers while allowing readers to still read via the "source" queue.
559 *
560 * However, the "source" queue needs to be reset to a valid state
561 * if it is to be used again.
562 */
563 #define smrq_serialized_append(dst, src) ({ \
564 __auto_type __src = (src); \
565 __auto_type __dst = (dst); \
566 \
567 __smrq_serialized_append(&__dst->first, __smrq_lastp(__dst), \
568 &__src->first, __smrq_lastp(__src)); \
569 })
570
571
572 #pragma mark SMR domains
573
574 /*!
575 * @enum smr_flags_t
576 *
577 * @brief
578 * Options to pass to smr_domain_create()
579 *
580 * @const SMR_NONE
581 * Default values for the flags.
582 #if XNU_KERNEL_PRIVATE
583 *
584 * @const SMR_SLEEPABLE
585 * Create a sleepable SMR domain.
586 #endif
587 */
588 __options_closed_decl(smr_flags_t, unsigned long, {
589 SMR_NONE = 0x00000000,
590 #if XNU_KERNEL_PRIVATE
591 SMR_SLEEPABLE = 0x00000001,
592 #endif
593 });
594
595 /*!
596 * @function smr_domain_create()
597 *
598 * @brief
599 * Create an SMR domain.
600 *
601 * @discussion
602 * Be mindful when creating SMR domains, and consider carefully
603 * whether to add one or consolidate an existing one.
604 *
605 *
606 * Memory usage
607 * ~~~~~~~~~~~~
608 *
609 * SMR domains are fairly large structures that scale with the number
610 * of cores of the machine. They are meant to be use in a coarse grained
611 * manner.
612 *
613 * In addition to that, when @c smr_call() is used with the domain,
614 * the queues of callbacks are drained based on memory pressure within
615 * the domain. The more domains, the more dormant memory might exist.
616 *
617 * In general, memory considerations drive toward less domains.
618 *
619 *
620 * Scalability
621 * ~~~~~~~~~~~
622 *
623 * An SMR domain is built on top of an atomic state that is used
624 * to perform grace period detection. The more "write" activity
625 * there is on the domain (@c smr_call(), @c smr_advance(), etc...),
626 * the more this atomic might become contended. In particular,
627 * certain usage patterns might scale extremely well independently,
628 * but cause higher contention when sharing a domain.
629 *
630 * Another thing to consider is that when @c smr_call() is being used,
631 * if the callbacks act on vastly different data structures, then as
632 * the callbacks are being drained, cache misses will be higher.
633 *
634 * However, the more domains are in use, the more probable it is
635 * that using it will cause a cache miss.
636 *
637 * Generally, scalability considerations drive toward balanced
638 * coarse-grained domains.
639 *
640 *
641 * Invariants
642 * ~~~~~~~~~~
643 *
644 * The last aspect leading to the decision of creating versus reusing
645 * an SMR domain is about the invariants that these domains protect.
646 *
647 * Object graphs that are protected with SMR and are used together
648 * in many workloads will likely require to share an SMR domain
649 * in order to provide the required guarantees. Having @c smr_call()
650 * callbacks in a given domain cause downstream @c smr_call() into
651 * another different domain regularly is probably a sign that these
652 * domains should be shared.
653 *
654 * Another aspect to consider is that using @c smr_synchronize()
655 * or @c smr_barrier() can lead to two classes of problems:
656 *
657 * - these operations are extremely heavy, and if some subsystem needs to
658 * perform them on several domains, performance will be disappointing.
659 *
660 * - these operations are akin to taking a "write lock" on the domain,
661 * and as such can cause deadlocks when used improperly.
662 * Using a coarser grained unique domain is a good way to simplify
663 * reasoning about the locking dependencies between SMR domains
664 * and other regular locks.
665 *
666 *
667 * Guidance
668 * ~~~~~~~~
669 *
670 * In general, the entire kernel should have relatively few SMR domains,
671 * at the scale of the big subsystems of the kernel (think: Mach IPC, Mach VM,
672 * VFS, Networking, ...).
673 *
674 * When write operations (@c smr_call(), @c smr_synchronize, ...) are used
675 * rarely, consider using the system wide default domains.
676 */
677 extern smr_t smr_domain_create(smr_flags_t flags, const char *name);
678
679 /*!
680 * @function smr_domain_free()
681 *
682 * @brief
683 * Destroys an SMR domain previously create with @c smr_domain_create().
684 */
685 extern void smr_domain_free(smr_t smr);
686
687
688 /*!
689 * @function smr_entered()
690 *
691 * @brief
692 * Returns whether an SMR critical section is entered.
693 */
694 extern bool smr_entered(smr_t smr) __result_use_check;
695
696 /*!
697 * @function smr_enter()
698 *
699 * @brief
700 * Enter a non preemptible SMR critical section.
701 *
702 * @discussion
703 * Entering an SMR critical section is non reentrant.
704 * (entering it recursively is undefined and will panic on development kernels)
705 *
706 * @c smr_leave() must be called to end this section.
707 *
708 * This function can'be be used in interrupt context.
709 */
710 extern void smr_enter(smr_t smr);
711
712 /*!
713 * @function smr_leave()
714 *
715 * @brief
716 * Leave a non preemptible SMR critical section.
717 */
718 extern void smr_leave(smr_t smr);
719
720
721 /*!
722 * @function smr_call()
723 *
724 * @brief
725 * Defer making a call until it is safe to assume all readers
726 * will observe any update prior to this call.
727 *
728 * @discussion
729 * The target SMR domain must NOT be entered when making this call.
730 *
731 * The passed @c size doesn't have to be precise, it should be a rough
732 * estimate of the memory that will be reclaimed when when the call is made.
733 *
734 * This function gives no guarantee of forward progress,
735 * unless the magic SMR_CALL_EXPEDITE size is passed to @c smr_call().
736 *
737 * This function can'be be used in interrupt context.
738 */
739 extern void smr_call(smr_t smr, smr_node_t node, vm_size_t size, smr_cb_t cb);
740
741 #define SMR_CALL_EXPEDITE ((vm_size_t)~0)
742
743 /*!
744 * @function smr_synchronize()
745 *
746 * @brief
747 * Wait until all readers have observed any updates made prior to this call.
748 *
749 * @discussion
750 * The target SMR domain must NOT be entered when making this call.
751 *
752 * This function is quite expensive, and asynchronous deferred processing
753 * using @c smr_call() should be used instead when possible.
754 *
755 * Reserve using this call for events that are extremely rare (like system
756 * configuration events such as configuring networking interfaces, changing
757 * system wide security policies, or loading/unloading a kernel extension).
758 *
759 * This function should typically be called with preemption enabled,
760 * and no locks held.
761 */
762 extern void smr_synchronize(smr_t smr);
763
764 /*!
765 * @function smr_barrier()
766 *
767 * @brief
768 * Wait until all readers have observed any updates made prior to this call,
769 * and all @c smr_call() callbacks dispatched prior to this call on any core
770 * have completed.
771 *
772 * @discussion
773 * The target SMR domain must NOT be entered when making this call.
774 *
775 * This function is typically used when some data structure is being
776 * accessed by @c smr_call() callbacks and that data structure needs
777 * to be retired.
778 *
779 * Reserve using this call for events that are extremely rare (like system
780 * configuration events such as configuring networking interfaces, changing
781 * system wide security policies, or loading/unloading a kernel extension).
782 *
783 * This function should typically be called with preemption enabled,
784 * and no locks held.
785 */
786 extern void smr_barrier(smr_t smr);
787
788
789 #ifdef XNU_KERNEL_PRIVATE
790 #pragma GCC visibility push(hidden)
791 #pragma mark - XNU only
792 #pragma mark XNU only: SMR domains advanced
793
794 #define SMR_SEQ_INVALID ((smr_seq_t)0)
795 #define SMR_SEQ_SLEEPABLE ((smr_seq_t)1) /* only on smr_pcpu::rd_seq */
796 #define SMR_SEQ_INIT ((smr_seq_t)2)
797 #define SMR_SEQ_INC ((smr_seq_t)4)
798
799 typedef long smr_delta_t;
800
801 #define SMR_SEQ_DELTA(a, b) ((smr_delta_t)((a) - (b)))
802 #define SMR_SEQ_CMP(a, op, b) (SMR_SEQ_DELTA(a, b) op 0)
803
804 /*!
805 * @typedef smr_clock_t
806 *
807 * @brief
808 * Represents an SMR domain clock, internal type not manipulated by clients.
809 */
810 typedef struct {
811 smr_seq_t s_rd_seq;
812 smr_seq_t s_wr_seq;
813 } smr_clock_t;
814
815 #define SMR_NAME_MAX 24
816
817 /*!
818 * @typedef smr_t
819 *
820 * @brief
821 * Declares an SMR domain of synchronization.
822 */
823 struct smr {
824 smr_clock_t smr_clock;
825 struct smr_pcpu *smr_pcpu;
826 unsigned long smr_flags;
827 unsigned long smr_early;
828 char smr_name[SMR_NAME_MAX];
829 } __attribute__((aligned(64)));
830
831 /*!
832 * @macro SMR_DEFINE_FLAGS
833 *
834 * @brief
835 * Define an SMR domain with specific create flags.
836 */
837 #define SMR_DEFINE_FLAGS(var, name, flags) \
838 struct smr var = { \
839 .smr_clock.s_rd_seq = SMR_SEQ_INIT, \
840 .smr_clock.s_wr_seq = SMR_SEQ_INIT, \
841 .smr_flags = (flags), \
842 .smr_name = "" name, \
843 }; \
844 STARTUP_ARG(TUNABLES, STARTUP_RANK_LAST, __smr_domain_init, &(var)); \
845 STARTUP_ARG(ZALLOC, STARTUP_RANK_LAST, __smr_domain_init, &(var))
846
847 /*!
848 * @macro SMR_DEFINE
849 *
850 * @brief
851 * Define an SMR domain.
852 */
853 #define SMR_DEFINE(var, name) \
854 SMR_DEFINE_FLAGS(var, name, SMR_NONE)
855
856
857 /*!
858 * @macro SMR_DEFINE_SLEEPABLE
859 *
860 * @brief
861 * Define a sleepable SMR domain.
862 */
863 #define SMR_DEFINE_SLEEPABLE(var, name) \
864 SMR_DEFINE_FLAGS(var, name, SMR_SLEEPABLE)
865
866
867 /*!
868 * @function smr_advance()
869 *
870 * @brief
871 * Advance the write sequence and return the value
872 * for use as a wait goal.
873 *
874 * @discussion
875 * This guarantees that any changes made by the calling thread
876 * prior to this call will be visible to all threads after
877 * the read sequence meets or exceeds the return value.
878 */
879 extern smr_seq_t smr_advance(smr_t smr) __result_use_check;
880
881 /*!
882 * @function smr_deferred_advance()
883 *
884 * @brief
885 * Pretend-advance the write sequence and return the value
886 * for use as a wait goal.
887 *
888 * @discussion
889 * This guarantees that any changes made by the calling thread
890 * prior to this call will be visible to all threads after
891 * the read sequence meets or exceeds the return value.
892 *
893 * Unlike smr_advance(), the global clock isn't really advanced,
894 * it only sets a goal in the future. This can be used to control
895 * the pace of updating the global clock and avoid global atomics.
896 *
897 * In order for the clock to advance, clients of this API must call
898 * @c smr_deferred_advance_commit() with the goal returned by this call.
899 *
900 * Note that calls to @c smr_advance() or @c smr_wait() when passed
901 * the goal returned by this function would also allow the clock
902 * to make progress and are legal (yet less efficient) calls to make.
903 */
904 extern smr_seq_t smr_deferred_advance(smr_t smr) __result_use_check;
905
906 /*!
907 * @function smr_deferred_advance_commit()
908 *
909 * @brief
910 * Actually advance the write sequence to the goal returned by a previous
911 * call to @c smr_deferred_advance().
912 */
913 extern void smr_deferred_advance_commit(smr_t smr, smr_seq_t seq);
914
915
916 /*!
917 * @function smr_poll
918 *
919 * @brief
920 * Poll to determine whether all readers have observed the @c goal
921 * write sequence number.
922 *
923 * @discussion
924 * This function is safe to be called from preemption disabled context
925 * and its worst complexity is O(ncpu).
926 *
927 * @returns true if the goal is met and false if not.
928 */
929 extern bool smr_poll(smr_t smr, smr_seq_t goal) __result_use_check;
930
931 /*!
932 * @function smr_wait
933 *
934 * @brief
935 * Wait until all readers have observed
936 * the @c goal write sequence number.
937 *
938 * @discussion
939 * This function is safe to be called from preemption disabled context
940 * as it never explicitly blocks, however this is not recommended.
941 */
942 extern void smr_wait(smr_t smr, smr_seq_t goal);
943
944
945 #pragma mark XNU only: major sleepable SMR domains
946 /*
947 * Note: this is private for now because sleepable sections that do "bad" things
948 * (such as doing an upcall to userspace, or doing VM allocations) have
949 * the danger that they can stall the reclamation worker threads,
950 * which are a singleton resource.
951 *
952 * Until this can be mitigated or designed better, this stays private.
953 */
954
955 /*!
956 * @typedef smr_tracker_t
957 *
958 * @brief
959 * Structure used to track active sleepable SMR sections.
960 *
961 * @field smrt_domain the entered SMR domain
962 * @field smrt_seq the SMR sequence at the time of smr_enter_sleepable().
963 * @field smrt_link linkage used to track stalled sections.
964 * @field smrt_stack linkage used to track entered sections.
965 * @field smrt_ctid (if stalled) the ctid of the thread in this section.
966 * @field smrt_cpu (if stalled) the cpu the thread was on when stalled.
967 */
968 typedef struct smr_tracker {
969 smr_t smrt_domain;
970 smr_seq_t smrt_seq;
971 struct smrq_link smrt_link;
972 struct smrq_slink smrt_stack;
973 uint32_t smrt_ctid;
974 int smrt_cpu;
975 } *smr_tracker_t;
976
977 /*!
978 * @function smr_enter_sleepable()
979 *
980 * @brief
981 * Enter a sleepable SMR critical section.
982 *
983 * @discussion
984 * Entering an SMR critical section is non recursive
985 * (entering it recursively is undefined and will panic on development kernels)
986 *
987 * @c smr_leave_sleepable() must be called to end this section,
988 * passing the same tracker pointer.
989 *
990 * The SMR domain must have been created with the @c SMR_SLEEPABLE flag.
991 *
992 * It is permitted to do operations that might block under such a transaction,
993 * such as acquiring a lock, or freeing memory.
994 *
995 * It is forbidden to perform operations that wait for an unbounded amount of
996 * time such as waiting for networking packets or even a hardware driver event,
997 * as these could cause grace periods (and memory reclamation) to stall for
998 * a very long time.
999 */
1000 extern void smr_enter_sleepable(smr_t smr, smr_tracker_t tracker);
1001
1002 /*!
1003 * @function smr_leave_sleepable()
1004 *
1005 * @brief
1006 * Leave a sleepable SMR critical section entered with @c smr_enter_sleepable().
1007 */
1008 extern void smr_leave_sleepable(smr_t smr, smr_tracker_t tracker);
1009
1010
1011 #pragma mark XNU only: major subsystems SMR domains
1012
1013 /*!
1014 * @brief
1015 * A global system wide non preemptible domain.
1016 *
1017 * @discussion
1018 * This is provided as a fallback for when a specific SMR domain
1019 * would be overkill.
1020 *
1021 * Try not use the @c smr_system name directly, instead define
1022 * a subsystem domain that happens to be defined to it, so that
1023 * understanding the invariants being provided is easier.
1024 */
1025 extern struct smr smr_system;
1026
1027 /*!
1028 * @brief
1029 * A global system wide sleepable domain.
1030 *
1031 * @discussion
1032 * This is provided as a fallback for when a specific SMR domain
1033 * would be overkill.
1034 *
1035 * Try not use the @c smr_system_sleepable name directly,
1036 * instead define a subsystem domain that happens to be defined to it,
1037 * so that understanding the invariants being provided is easier.
1038 */
1039 extern struct smr smr_system_sleepable;
1040
1041
1042 /*!
1043 * @macro smr_ipc
1044 *
1045 * @brief
1046 * The SMR domain for the Mach IPC subsystem.
1047 */
1048 #define smr_ipc smr_system
1049 #define smr_ipc_entered() smr_entered(&smr_ipc)
1050 #define smr_ipc_enter() smr_enter(&smr_ipc)
1051 #define smr_ipc_leave() smr_leave(&smr_ipc)
1052
1053 #define smr_ipc_call(n, sz, cb) smr_call(&smr_ipc, n, sz, cb)
1054 #define smr_ipc_synchronize() smr_synchronize(&smr_ipc)
1055 #define smr_ipc_barrier() smr_barrier(&smr_ipc)
1056
1057
1058 /*!
1059 * @macro smr_proc_task
1060 *
1061 * @brief
1062 * The SMR domain for the proc/task and adjacent objects.
1063 */
1064 #define smr_proc_task smr_system
1065 #define smr_proc_task_entered() smr_entered(&smr_proc_task)
1066 #define smr_proc_task_enter() smr_enter(&smr_proc_task)
1067 #define smr_proc_task_leave() smr_leave(&smr_proc_task)
1068
1069 #define smr_proc_task_call(n, sz, cb) smr_call(&smr_proc_task, n, sz, cb)
1070 #define smr_proc_task_synchronize() smr_synchronize(&smr_proc_task)
1071 #define smr_proc_task_barrier() smr_barrier(&smr_proc_task)
1072
1073
1074 /*!
1075 * @macro smr_iokit
1076 *
1077 * @brief
1078 * The SMR domain for IOKit
1079 */
1080 #define smr_iokit smr_system
1081 #define smr_iokit_entered() smr_entered(&smr_iokit)
1082 #define smr_iokit_enter() smr_enter(&smr_iokit)
1083 #define smr_iokit_leave() smr_leave(&smr_iokit)
1084
1085 #define smr_iokit_call(n, sz, cb) smr_call(&smr_iokit, n, sz, cb)
1086 #define smr_iokit_synchronize() smr_synchronize(&smr_iokit)
1087 #define smr_iokit_barrier() smr_barrier(&smr_iokit)
1088
1089
1090 /*!
1091 * @macro smr_oslog
1092 *
1093 * @brief
1094 * The SMR domain for kernel OSLog handles.
1095 */
1096 #define smr_oslog smr_system
1097 #define smr_oslog_entered() smr_entered(&smr_oslog)
1098 #define smr_oslog_enter() smr_enter(&smr_oslog)
1099 #define smr_oslog_leave() smr_leave(&smr_oslog)
1100
1101 #define smr_oslog_call(n, sz, cb) smr_call(&smr_oslog, n, sz, cb)
1102 #define smr_oslog_synchronize() smr_synchronize(&smr_oslog)
1103 #define smr_oslog_barrier() smr_barrier(&smr_oslog)
1104
1105
1106 #pragma mark XNU only: implementation details
1107
1108 extern void __smr_domain_init(smr_t);
1109
1110 #ifdef MACH_KERNEL_PRIVATE
1111 struct processor;
1112
1113 extern bool smr_entered_cpu_noblock(smr_t smr, int cpu) __result_use_check;
1114
1115 extern void smr_ack_ipi(void);
1116
1117 extern void smr_mark_active_trackers_stalled(struct thread *self);
1118
1119 __options_closed_decl(smr_cpu_reason_t, uint8_t, {
1120 SMR_CPU_REASON_NONE = 0x00,
1121 SMR_CPU_REASON_OFFLINE = 0x01,
1122 SMR_CPU_REASON_IGNORED = 0x02,
1123 SMR_CPU_REASON_ALL = 0x03,
1124 });
1125
1126 extern void smr_cpu_init(struct processor *);
1127 extern void smr_cpu_up(struct processor *, smr_cpu_reason_t);
1128 extern void smr_cpu_down(struct processor *, smr_cpu_reason_t);
1129
1130 extern void smr_cpu_join(struct processor *, uint64_t ctime);
1131 extern void smr_cpu_tick(uint64_t ctime, bool safe_point);
1132 extern void smr_cpu_leave(struct processor *, uint64_t ctime);
1133
1134 extern void smr_maintenance(uint64_t ctime);
1135
1136 #if CONFIG_QUIESCE_COUNTER
1137 extern void cpu_quiescent_set_storage(uint64_t _Atomic *ptr);
1138 #endif
1139 #endif /* MACH_KERNEL_PRIVATE */
1140
1141 extern uint32_t smr_cpu_checkin_get_min_interval_us(void);
1142
1143 extern uint32_t smr_cpu_checkin_get_min_interval_us(void);
1144
1145 extern void smr_cpu_checkin_set_min_interval_us(uint32_t new_value);
1146
1147 #pragma GCC visibility pop
1148 #endif /* XNU_KERNEL_PRIVATE */
1149 #pragma mark - implementation details
1150 #pragma mark implementation details: SMR queues
1151
1152 extern void __smr_linkage_invalid(__smrq_link_t *link) __abortlike;
1153 extern void __smr_stail_invalid(__smrq_slink_t *link, __smrq_slink_t *last) __abortlike;
1154 extern void __smr_tail_invalid(__smrq_link_t *link, __smrq_link_t *last) __abortlike;
1155
1156 __attribute__((always_inline, overloadable))
1157 static inline __smrq_slink_t **
__smrq_lastp(struct smrq_slist_head * head __unused)1158 __smrq_lastp(struct smrq_slist_head *head __unused)
1159 {
1160 return NULL;
1161 }
1162
1163 __attribute__((always_inline, overloadable))
1164 static inline __smrq_link_t **
__smrq_lastp(struct smrq_list_head * head __unused)1165 __smrq_lastp(struct smrq_list_head *head __unused)
1166 {
1167 return NULL;
1168 }
1169
1170 __attribute__((always_inline, overloadable))
1171 static inline __smrq_slink_t **
__smrq_lastp(struct smrq_stailq_head * head)1172 __smrq_lastp(struct smrq_stailq_head *head)
1173 {
1174 __smrq_slink_t **last = &head->last;
1175
1176 __builtin_assume(last != NULL);
1177 return last;
1178 }
1179
1180 __attribute__((always_inline, overloadable))
1181 static inline __smrq_link_t **
__smrq_lastp(struct smrq_tailq_head * head)1182 __smrq_lastp(struct smrq_tailq_head *head)
1183 {
1184 __smrq_link_t **last = &head->last;
1185
1186 __builtin_assume(last != NULL);
1187 return last;
1188 }
1189
1190
1191 __attribute__((always_inline, overloadable))
1192 static inline void
__smrq_serialized_insert(__smrq_slink_t * prev,struct smrq_slink * elem,struct smrq_slink * next,__smrq_slink_t ** lastp)1193 __smrq_serialized_insert(
1194 __smrq_slink_t *prev,
1195 struct smrq_slink *elem,
1196 struct smrq_slink *next,
1197 __smrq_slink_t **lastp)
1198 {
1199 if (next == NULL && lastp) {
1200 if (*lastp != prev || smr_serialized_load(prev)) {
1201 __smr_stail_invalid(prev, *lastp);
1202 }
1203 }
1204
1205 smr_serialized_store_relaxed(&elem->next, next);
1206 smr_serialized_store(prev, elem);
1207 if (next == NULL && lastp) {
1208 *lastp = &elem->next;
1209 }
1210 }
1211
1212 __attribute__((always_inline, overloadable))
1213 static inline void
__smrq_serialized_insert(__smrq_link_t * prev,struct smrq_link * elem,struct smrq_link * next,__smrq_link_t ** lastp)1214 __smrq_serialized_insert(
1215 __smrq_link_t *prev,
1216 struct smrq_link *elem,
1217 struct smrq_link *next,
1218 __smrq_link_t **lastp)
1219 {
1220 if (next != NULL && next->prev != prev) {
1221 __smr_linkage_invalid(prev);
1222 }
1223 if (next == NULL && lastp) {
1224 if (*lastp != prev || smr_serialized_load(prev)) {
1225 __smr_tail_invalid(prev, *lastp);
1226 }
1227 }
1228
1229 smr_serialized_store_relaxed(&elem->next, next);
1230 elem->prev = prev;
1231 smr_serialized_store(prev, elem);
1232
1233 if (next != NULL) {
1234 next->prev = &elem->next;
1235 } else if (lastp) {
1236 *lastp = &elem->next;
1237 }
1238 }
1239
1240
1241 __attribute__((always_inline, overloadable))
1242 static inline void
__smrq_serialized_insert_relaxed(__smrq_slink_t * prev,struct smrq_slink * elem,struct smrq_slink * next,__smrq_slink_t ** lastp)1243 __smrq_serialized_insert_relaxed(
1244 __smrq_slink_t *prev,
1245 struct smrq_slink *elem,
1246 struct smrq_slink *next,
1247 __smrq_slink_t **lastp)
1248 {
1249 if (next == NULL && lastp) {
1250 if (*lastp != prev || smr_serialized_load(prev)) {
1251 __smr_stail_invalid(prev, *lastp);
1252 }
1253 }
1254
1255 smr_serialized_store_relaxed(&elem->next, next);
1256 smr_serialized_store_relaxed(prev, elem);
1257 if (next == NULL && lastp) {
1258 *lastp = &elem->next;
1259 }
1260 }
1261
1262 __attribute__((always_inline, overloadable))
1263 static inline void
__smrq_serialized_insert_relaxed(__smrq_link_t * prev,struct smrq_link * elem,struct smrq_link * next,__smrq_link_t ** lastp)1264 __smrq_serialized_insert_relaxed(
1265 __smrq_link_t *prev,
1266 struct smrq_link *elem,
1267 struct smrq_link *next,
1268 __smrq_link_t **lastp)
1269 {
1270 if (next != NULL && next->prev != prev) {
1271 __smr_linkage_invalid(prev);
1272 }
1273 if (next == NULL && lastp) {
1274 if (*lastp != prev || smr_serialized_load(prev)) {
1275 __smr_tail_invalid(prev, *lastp);
1276 }
1277 }
1278
1279 smr_serialized_store_relaxed(&elem->next, next);
1280 elem->prev = prev;
1281 smr_serialized_store_relaxed(prev, elem);
1282
1283 if (next != NULL) {
1284 next->prev = &elem->next;
1285 } else if (lastp) {
1286 *lastp = &elem->next;
1287 }
1288 }
1289
1290
1291 __attribute__((always_inline, overloadable))
1292 static inline void
__smrq_serialized_remove_one(__smrq_slink_t * prev,struct smrq_slink * elem,__smrq_slink_t ** lastp)1293 __smrq_serialized_remove_one(
1294 __smrq_slink_t *prev,
1295 struct smrq_slink *elem,
1296 __smrq_slink_t **lastp)
1297 {
1298 struct smrq_slink *next;
1299
1300 /*
1301 * Removal "skips" a link this way:
1302 *
1303 * e1 ---> e2 ---> e3 becomes e1 -----------> e3
1304 *
1305 * When e3 was inserted, a release barrier was issued
1306 * by smr_serialized_store(). We do not need to issue
1307 * a release barrier upon removal because `next` carries
1308 * a dependency on that smr_serialized_store()d value.
1309 */
1310 next = smr_serialized_load(&elem->next);
1311 smr_serialized_store_relaxed(prev, next);
1312 if (next == NULL && lastp) {
1313 *lastp = prev;
1314 }
1315 }
1316
1317 __attribute__((always_inline, overloadable))
1318 static inline void
__smrq_serialized_remove_one(__smrq_link_t * prev,struct smrq_link * elem,__smrq_link_t ** lastp)1319 __smrq_serialized_remove_one(
1320 __smrq_link_t *prev,
1321 struct smrq_link *elem,
1322 __smrq_link_t **lastp)
1323 {
1324 struct smrq_link *next;
1325
1326 next = smr_serialized_load(&elem->next);
1327
1328 if (smr_serialized_load(prev) != elem) {
1329 __smr_linkage_invalid(prev);
1330 }
1331 if (next && next->prev != &elem->next) {
1332 __smr_linkage_invalid(&elem->next);
1333 }
1334
1335 /*
1336 * Removal "skips" a link this way:
1337 *
1338 * e1 ---> e2 ---> e3 becomes e1 -----------> e3
1339 *
1340 * When e3 was inserted, a release barrier was issued
1341 * by smr_serialized_store(). We do not need to issue
1342 * a release barrier upon removal because `next` carries
1343 * a dependency on that smr_serialized_store()d value.
1344 */
1345 smr_serialized_store_relaxed(prev, next);
1346
1347 if (next != NULL) {
1348 next->prev = prev;
1349 } else if (lastp) {
1350 *lastp = prev;
1351 }
1352 elem->prev = NULL;
1353 }
1354
1355
1356 __attribute__((always_inline, overloadable))
1357 static inline void
__smrq_serialized_remove(__smrq_slink_t * first,struct smrq_slink * elem,__smrq_slink_t ** lastp)1358 __smrq_serialized_remove(
1359 __smrq_slink_t *first,
1360 struct smrq_slink *elem,
1361 __smrq_slink_t **lastp)
1362 {
1363 __smrq_slink_t *prev = first;
1364 struct smrq_slink *cur;
1365
1366 while ((cur = smr_serialized_load(prev)) != elem) {
1367 prev = &cur->next;
1368 }
1369
1370 __smrq_serialized_remove_one(prev, elem, lastp);
1371 }
1372
1373 __attribute__((always_inline, overloadable))
1374 static inline void
__smrq_serialized_remove(__smrq_link_t * first __unused,struct smrq_link * elem,__smrq_link_t ** lastp)1375 __smrq_serialized_remove(
1376 __smrq_link_t *first __unused,
1377 struct smrq_link *elem,
1378 __smrq_link_t **lastp)
1379 {
1380 __smrq_serialized_remove_one(elem->prev, elem, lastp);
1381 }
1382
1383
1384 __attribute__((always_inline, overloadable))
1385 static inline void
__smrq_serialized_replace(__smrq_slink_t * first,struct smrq_slink * old_elem,struct smrq_slink * new_elem,__smrq_slink_t ** lastp)1386 __smrq_serialized_replace(
1387 __smrq_slink_t *first,
1388 struct smrq_slink *old_elem,
1389 struct smrq_slink *new_elem,
1390 __smrq_slink_t **lastp)
1391 {
1392 __smrq_slink_t *prev = first;
1393 struct smrq_slink *cur;
1394 struct smrq_slink *next;
1395
1396 while ((cur = smr_serialized_load(prev)) != old_elem) {
1397 prev = &cur->next;
1398 }
1399
1400 next = smr_serialized_load(&old_elem->next);
1401 smr_serialized_store_relaxed(&new_elem->next, next);
1402 smr_serialized_store(prev, new_elem);
1403
1404 if (next == NULL && lastp) {
1405 *lastp = &new_elem->next;
1406 }
1407 }
1408
1409 __attribute__((always_inline, overloadable))
1410 static inline void
__smrq_serialized_replace(__smrq_link_t * first __unused,struct smrq_link * old_elem,struct smrq_link * new_elem,__smrq_link_t ** lastp)1411 __smrq_serialized_replace(
1412 __smrq_link_t *first __unused,
1413 struct smrq_link *old_elem,
1414 struct smrq_link *new_elem,
1415 __smrq_link_t **lastp)
1416 {
1417 __smrq_link_t *prev;
1418 struct smrq_link *next;
1419
1420 prev = old_elem->prev;
1421 next = smr_serialized_load(&old_elem->next);
1422
1423 if (smr_serialized_load(prev) != old_elem) {
1424 __smr_linkage_invalid(prev);
1425 }
1426 if (next && next->prev != &old_elem->next) {
1427 __smr_linkage_invalid(&old_elem->next);
1428 }
1429
1430 smr_serialized_store_relaxed(&new_elem->next, next);
1431 new_elem->prev = prev;
1432 smr_serialized_store(prev, new_elem);
1433
1434 if (next != NULL) {
1435 next->prev = &new_elem->next;
1436 } else if (lastp) {
1437 *lastp = &new_elem->next;
1438 }
1439 old_elem->prev = NULL;
1440 }
1441
1442 __attribute__((always_inline, overloadable))
1443 static inline void
__smrq_serialized_append(__smrq_slink_t * dst_first,__smrq_slink_t ** dst_lastp,__smrq_slink_t * src_first,__smrq_slink_t ** src_lastp)1444 __smrq_serialized_append(
1445 __smrq_slink_t *dst_first,
1446 __smrq_slink_t **dst_lastp,
1447 __smrq_slink_t *src_first,
1448 __smrq_slink_t **src_lastp)
1449 {
1450 struct smrq_slink *src = smr_serialized_load(src_first);
1451 struct smrq_slink *dst;
1452
1453 if (dst_lastp) {
1454 if (src) {
1455 smr_serialized_store_relaxed(*dst_lastp, src);
1456 *dst_lastp = *src_lastp;
1457 }
1458 } else {
1459 while ((dst = smr_serialized_load(dst_first))) {
1460 dst_first = &dst->next;
1461 }
1462 smr_serialized_store_relaxed(dst_first, src);
1463 }
1464 }
1465
1466 __attribute__((always_inline, overloadable))
1467 static inline void
__smrq_serialized_append(__smrq_link_t * dst_first,__smrq_link_t ** dst_lastp,__smrq_link_t * src_first,__smrq_link_t ** src_lastp)1468 __smrq_serialized_append(
1469 __smrq_link_t *dst_first,
1470 __smrq_link_t **dst_lastp,
1471 __smrq_link_t *src_first,
1472 __smrq_link_t **src_lastp)
1473 {
1474 struct smrq_link *src = smr_serialized_load(src_first);
1475 struct smrq_link *dst;
1476
1477 if (dst_lastp) {
1478 if (src) {
1479 smr_serialized_store_relaxed(*dst_lastp, src);
1480 src->prev = *dst_lastp;
1481 *dst_lastp = *src_lastp;
1482 }
1483 } else {
1484 while ((dst = smr_serialized_load(dst_first))) {
1485 dst_first = &dst->next;
1486 }
1487 smr_serialized_store_relaxed(dst_first, src);
1488 src->prev = &dst->next;
1489 }
1490 }
1491
1492 __END_DECLS
1493
1494 #endif /* _KERN_SMR_H_ */
1495