1 /* 2 * Copyright (c) 2017 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _OS_REFCNT_H_ 30 #define _OS_REFCNT_H_ 31 32 /* 33 * os_refcnt reference counting API 34 * 35 * Two flavors are provided: atomic and locked. Atomic internally uses C11 atomic 36 * operations and requires no external synchronization, whereas the locked flavor 37 * assumes the refcnt object is locked by the caller. It is NOT safe to 38 * mix-and-match locked and atomic calls. 39 * 40 * 'refgrp's are a way to (hierarchically) group like refcount objects for 41 * debugging purposes. The group keeps track of the total number and aggregate 42 * reference count of member refcounts, and the "rlog=" boot-arg is used to enable 43 * refcount logging by group name. Named groups can be created explicitly with 44 * os_refgrp_decl(), or implicitly by passing NULL for the refgrp when 45 * initializing a refcnt object. In the latter case, the group name is the same as 46 * the function enclosing the init call. Groups are only available on DEV or DEBUG 47 * builds, and are otherwise compiled out. 48 */ 49 50 #include <stdatomic.h> 51 #include <stdbool.h> 52 #include <os/base.h> 53 54 struct os_refcnt; 55 struct os_refgrp; 56 typedef struct os_refcnt os_refcnt_t; 57 58 /* type of the internal counter */ 59 typedef uint32_t os_ref_count_t; 60 typedef _Atomic(os_ref_count_t) os_ref_atomic_t; 61 62 /* 63 * OS_REF_INITIALIZER 64 * OS_REF_ATOMIC_INITIALIZER 65 * 66 * Static initializers that create refcnt objects with safe initial values for use 67 * between declaration and initialization (os_ref*_init()). Equivalent to zeroing. 68 */ 69 70 #ifndef KERNEL 71 # include <stdlib.h> 72 # include <stdio.h> 73 # ifndef __improbable 74 # define __improbable(x) x 75 # endif 76 # ifndef panic 77 # define panic(x, ...) do { fprintf(stderr, x, __VA_ARGS__); abort(); } while (0) 78 # endif 79 #endif 80 81 #ifndef OS_REFCNT_DEBUG 82 # if DEVELOPMENT || DEBUG 83 # define OS_REFCNT_DEBUG 1 84 # else 85 # define OS_REFCNT_DEBUG 0 86 # endif 87 #endif 88 89 #if __has_attribute(diagnose_if) 90 # define os_error_if(cond, msg) __attribute__((diagnose_if((cond), (msg), "error"))) 91 #else 92 # define os_error_if(...) 93 #endif 94 95 __BEGIN_DECLS 96 97 /* 98 * os_ref_init: initialize an os_refcnt with a count of 1 99 * os_ref_init_count: initialize an os_refcnt with a specific count >= 1 100 */ 101 #define os_ref_init(rc, grp) os_ref_init_count((rc), (grp), 1) 102 static void os_ref_init_count(struct os_refcnt *, struct os_refgrp *, os_ref_count_t count) 103 os_error_if(count == 0, "Reference count must be non-zero initialized"); 104 105 /* 106 * os_refgrp_decl(qual, var, name, parent): declare a refgroup object 'var' with 107 * given name string and parent group. 108 */ 109 110 /* 111 * 112 * os_ref_retain: acquire a reference (increment reference count by 1) atomically. 113 * 114 * os_ref_release: release a reference (decrement reference count) atomically and 115 * return the new count. Memory is synchronized such that the dealloc block 116 * (i.e. code handling the final release() == 0 call) sees up-to-date memory 117 * with respect to all prior release()s on the same refcnt object. This 118 * memory ordering is sufficient for most use cases. 119 * 120 * os_ref_release_relaxed: same as release() but with weaker relaxed memory ordering. 121 * This can be used when the dealloc block is already synchronized with other 122 * accesses to the object (for example, with a lock). 123 * 124 * os_ref_release_live: release a reference that is guaranteed not to be the last one. 125 */ 126 static void os_ref_retain(struct os_refcnt *); 127 static os_ref_count_t os_ref_release(struct os_refcnt *) OS_WARN_RESULT; 128 static os_ref_count_t os_ref_release_relaxed(struct os_refcnt *) OS_WARN_RESULT; 129 static void os_ref_release_live(struct os_refcnt *); 130 131 /* 132 * os_ref_retain_try: a variant of atomic retain that fails for objects with a 133 * zero reference count. The caller must therefore ensure that the object 134 * remains alive for any possible retain_try() caller, usually by using a 135 * lock protecting both the retain and dealloc paths. This variant is useful 136 * for objects stored in a collection, because no lock is required on the 137 * release() side until the object is deallocated. 138 */ 139 static bool os_ref_retain_try(struct os_refcnt *) OS_WARN_RESULT; 140 141 /* 142 * os_ref_retain_locked: acquire a reference on an object protected by a held 143 * lock. The caller must ensure mutual exclusivity of retain_locked() and 144 * release_locked() calls on the same object. 145 * 146 * os_ref_release_locked: release a reference on an object protected by a held 147 * lock. 148 */ 149 static void os_ref_retain_locked(struct os_refcnt *); 150 static os_ref_count_t os_ref_release_locked(struct os_refcnt *) OS_WARN_RESULT; 151 152 /* 153 * os_ref_get_count: return the current reference count. This is unsafe for 154 * synchronization. 155 */ 156 static os_ref_count_t os_ref_get_count(struct os_refcnt *rc); 157 158 159 #if XNU_KERNEL_PRIVATE 160 #pragma GCC visibility push(hidden) 161 162 /* 163 * Raw API that uses a plain atomic counter (os_ref_atomic_t) and a separate 164 * refgroup. This can be used in situations where the refcount object must be 165 * fixed size, for example for embedding in structures with ABI stability 166 * requirements. 167 */ 168 169 #define os_ref_init_raw(rc, grp) os_ref_init_count_raw((rc), (grp), 1) 170 static void os_ref_init_count_raw(os_ref_atomic_t *, struct os_refgrp *, os_ref_count_t count) 171 os_error_if(count == 0, "Reference count must be non-zero initialized"); 172 static void os_ref_retain_floor(struct os_refcnt *, os_ref_count_t f) 173 os_error_if(!__builtin_constant_p(f) || f == 0, "refcount floor must be >= 1"); 174 static void os_ref_retain_raw(os_ref_atomic_t *, struct os_refgrp *); 175 static void os_ref_retain_floor_raw(os_ref_atomic_t *, os_ref_count_t f, struct os_refgrp *) 176 os_error_if(!__builtin_constant_p(f) || f == 0, "refcount floor must be >= 1"); 177 static os_ref_count_t os_ref_release_raw(os_ref_atomic_t *, struct os_refgrp *) OS_WARN_RESULT; 178 static os_ref_count_t os_ref_release_raw_relaxed(os_ref_atomic_t *, struct os_refgrp *) OS_WARN_RESULT; 179 static void os_ref_release_live_raw(os_ref_atomic_t *, struct os_refgrp *); 180 static bool os_ref_retain_try_raw(os_ref_atomic_t *, struct os_refgrp *) OS_WARN_RESULT; 181 static bool os_ref_retain_floor_try_raw(os_ref_atomic_t *, os_ref_count_t f, struct os_refgrp *) OS_WARN_RESULT 182 os_error_if(!__builtin_constant_p(f) || f == 0, "refcount floor must be >= 1"); 183 static void os_ref_retain_locked_raw(os_ref_atomic_t *, struct os_refgrp *); 184 static void os_ref_retain_floor_locked_raw(os_ref_atomic_t *, os_ref_count_t f, struct os_refgrp *) 185 os_error_if(!__builtin_constant_p(f) || f == 0, "refcount floor must be >= 1"); 186 static os_ref_count_t os_ref_release_locked_raw(os_ref_atomic_t *, struct os_refgrp *) OS_WARN_RESULT; 187 static os_ref_count_t os_ref_get_count_raw(os_ref_atomic_t *rc); 188 189 190 /* 191 * Bitwise API: like the raw API, but allows some bits in the refcount value to be 192 * reserved for other purposes. 'b' defines the number of trailing (LSB) reserved 193 * bits, which the refcnt_raw API will never modify (except at init()). 194 * 195 * It is assumed that users of this API always use atomic ops on the 196 * os_ref_atomic_t (or hold a lock for the locked variants), and never modify the 197 * top (32 - 'b') bits. 198 * 199 * Due to guard bits, the maximum reference count is 2^(28 - 'b') - 1, and the 200 * maximum 'b' is 26 bits. This API can also be used just to limit the max 201 * refcount. 202 * 203 * The "*_raw_mask" APIs return the raw bit pattern of the refcount (with a type 204 * of uint32_t), that the caller is supposed to decode. Other APIs that return 205 * os_ref_count_t return a normalized refcount where the trailing bits have been 206 * removed. 207 * 208 * "locked" variants aren't provided as the point of these interfaces 209 * is to combine flags into a refcount and be able to manipulate both 210 * atomically with respect to each other. 211 */ 212 213 /* Initialize the reference count and reserved bits */ 214 #define os_ref_init_mask(rc, b, grp, bits) os_ref_init_count_mask((rc), (b), (grp), 1, bits) 215 void os_ref_init_count_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp, 216 os_ref_count_t init_count, uint32_t init_bits) 217 os_error_if(init_count == 0, "Reference count must be non-zero initialized") 218 os_error_if(b > 26, "Bitwise reference count limited to 26 bits") 219 os_error_if(init_bits >= (1U << b), "Bits out of range"); 220 221 static uint32_t os_ref_get_raw_mask(os_ref_atomic_t *rc); 222 static uint32_t os_ref_get_bits_mask(os_ref_atomic_t *rc, uint32_t b); 223 static os_ref_count_t os_ref_get_count_mask(os_ref_atomic_t *rc, uint32_t b); 224 225 static void 226 os_ref_retain_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp); 227 static void 228 os_ref_retain_acquire_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp); 229 static uint32_t 230 os_ref_retain_try_mask(os_ref_atomic_t *, uint32_t b, uint32_t reject_mask, 231 struct os_refgrp *grp) OS_WARN_RESULT; 232 static bool 233 os_ref_retain_try_acquire_mask(os_ref_atomic_t *, uint32_t b, uint32_t reject_mask, 234 struct os_refgrp *grp) OS_WARN_RESULT; 235 236 static uint32_t 237 os_ref_release_raw_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp) OS_WARN_RESULT; 238 static uint32_t 239 os_ref_release_raw_relaxed_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp) OS_WARN_RESULT; 240 static os_ref_count_t 241 os_ref_release_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp) OS_WARN_RESULT; 242 static os_ref_count_t 243 os_ref_release_relaxed_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp) OS_WARN_RESULT; 244 static uint32_t 245 os_ref_release_live_raw_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp); 246 static void 247 os_ref_release_live_mask(os_ref_atomic_t *rc, uint32_t b, struct os_refgrp *grp); 248 249 #pragma GCC visibility pop 250 #endif /* XNU_KERNEL_PRIVATE */ 251 252 __END_DECLS 253 254 #include <os/refcnt_internal.h> 255 #endif 256