1 /* 2 * Copyright (c) 2012 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifdef KERNEL_PRIVATE 30 31 #ifndef _PTHREAD_SHIMS_H_ 32 #define _PTHREAD_SHIMS_H_ 33 34 #ifndef ASSEMBLER 35 36 #include <kern/block_hint.h> 37 #include <kern/clock.h> 38 #include <kern/kern_types.h> 39 #include <kern/kcdata.h> 40 #include <kern/locks.h> 41 #include <kern/turnstile.h> 42 #include <pthread/priority_private.h> 43 #include <sys/user.h> 44 #include <sys/_types.h> 45 #include <sys/_types/_sigset_t.h> 46 #include <sys/kernel_types.h> 47 48 #ifndef PTHREAD_INTERNAL 49 struct uthread; 50 struct ksyn_waitq_element; 51 #define M_PROC 41 52 #endif 53 54 #if !defined(_SCHED_CALL_T_DEFINED) 55 #define _SCHED_CALL_T_DEFINED 56 typedef void (*sched_call_t)(int type, thread_t thread); 57 #endif 58 59 /* 60 * Increment each time new reserved slots are used. When the pthread 61 * kext registers this table, it will include the version of the xnu 62 * headers that it was built against. 63 */ 64 #define PTHREAD_FUNCTIONS_TABLE_VERSION 1 65 66 typedef const struct pthread_functions_s { 67 int version; 68 69 /* internal calls, kernel core -> kext */ 70 void (*pthread_init)(void); 71 72 void *__unused_was_fill_procworkqueue; 73 void *__unused1; 74 void *__unused2; 75 void *__unused_was_workqueue_exit; 76 void *__unused_was_workqueue_mark_exiting; 77 void *__unused_was_workqueue_thread_yielded; 78 79 void (*pth_proc_hashinit)(proc_t p); 80 void (*pth_proc_hashdelete)(proc_t p); 81 82 /* syscall stubs */ 83 int (*bsdthread_create)(struct proc *p, user_addr_t user_func, user_addr_t user_funcarg, user_addr_t user_stack, user_addr_t user_pthread, uint32_t flags, user_addr_t *retval); 84 int (*bsdthread_register)(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, int pthsize, user_addr_t dummy_value, user_addr_t targetconc_ptr, uint64_t dispatchqueue_offset, int32_t *retval); 85 int (*bsdthread_terminate)(struct proc *p, user_addr_t stackaddr, size_t size, uint32_t kthport, uint32_t sem, int32_t *retval); 86 int (*thread_selfid)(struct proc *p, uint64_t *retval); 87 void *__unused_was_workq_kernreturn; 88 void *__unused_was_workq_open; 89 90 /* psynch syscalls */ 91 int (*psynch_mutexwait)(proc_t p, user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags, uint32_t *retval); 92 int (*psynch_mutexdrop)(proc_t p, user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags, uint32_t *retval); 93 int (*psynch_cvbroad)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t *retval); 94 int (*psynch_cvsignal)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t flags, uint32_t *retval); 95 int (*psynch_cvwait)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, user_addr_t mutex, uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec, uint32_t * retval); 96 int (*psynch_cvclrprepost)(proc_t p, user_addr_t cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags, int *retval); 97 int (*psynch_rw_longrdlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval); 98 int (*psynch_rw_rdlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval); 99 int (*psynch_rw_unlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval); 100 int (*psynch_rw_wrlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval); 101 int (*psynch_rw_yieldwrlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval); 102 103 void *__unused_was_workqueue_get_sched_callback; 104 105 /* New register function with TSD offset */ 106 int (*bsdthread_register2)(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, uint32_t flags, user_addr_t stack_addr_hint, user_addr_t targetconc_ptr, uint32_t dispatchqueue_offset, uint32_t tsd_offset, int32_t *retval); 107 108 void *__unused_was_bsdthread_ctl; 109 void *__unused_was_workq_reqthreads; 110 111 void *__unused_was_thread_qos_from_pthread_priority; 112 void *__unused_was_get_pwq_state_kdp; 113 void *__unused3; 114 void *__unused_was_pthread_priority_canonicalize2; 115 void *__unused_was_workq_thread_has_been_unbound; 116 117 void (*pthread_find_owner)(thread_t thread, struct stackshot_thread_waitinfo *waitinfo); 118 void *(*pthread_get_thread_kwq)(thread_t thread); 119 120 void *__unused_was_workq_threadreq; 121 122 int (*workq_handle_stack_events)(proc_t p, thread_t th, vm_map_t map, 123 user_addr_t stackaddr, mach_port_name_t kport, 124 user_addr_t events, int nevents, int upcall_flags); 125 126 int (*workq_create_threadstack)(proc_t p, vm_map_t vmap, 127 mach_vm_offset_t *out_addr); 128 129 int (*workq_destroy_threadstack)(proc_t p, vm_map_t vmap, 130 mach_vm_offset_t stackaddr); 131 132 void (*workq_setup_thread)(proc_t p, thread_t th, vm_map_t map, 133 user_addr_t stackaddr, mach_port_name_t kport, int th_qos, 134 int setup_flags, int upcall_flags); 135 136 void (*workq_markfree_threadstack)(proc_t p, thread_t, vm_map_t map, 137 user_addr_t stackaddr); 138 139 /* padding for future */ 140 void * _pad[83]; 141 } * pthread_functions_t; 142 143 typedef const struct pthread_callbacks_s { 144 int version; 145 146 /* config information */ 147 uint32_t config_thread_max; 148 uint32_t (*get_task_threadmax)(void); 149 150 /* proc.h accessors */ 151 uint64_t (*proc_get_register)(struct proc *t); 152 void (*proc_set_register)(struct proc *t); 153 154 user_addr_t (*proc_get_threadstart)(struct proc *t); 155 void (*proc_set_threadstart)(struct proc *t, user_addr_t addr); 156 user_addr_t (*proc_get_wqthread)(struct proc *t); 157 void (*proc_set_wqthread)(struct proc *t, user_addr_t addr); 158 int (*proc_get_pthsize)(struct proc *t); 159 void (*proc_set_pthsize)(struct proc *t, int size); 160 161 thread_t (*task_findtid)(task_t t, uint64_t tid); 162 void (*thread_deallocate_safe)(thread_t); 163 /* reclaimable after rdar://81291169 */ 164 bool (*proc_get_pthread_jit_allowlist)(struct proc *t); 165 void (*proc_set_dispatchqueue_offset)(struct proc *t, uint64_t offset); 166 bool (*proc_get_pthread_jit_allowlist2)(struct proc *t, bool *late_out); 167 void *__unused_was_proc_get_wqinitingptr; 168 void *__unused_was_proc_get_wqptr; 169 170 wait_result_t (*psynch_wait_prepare)(uintptr_t kwq, 171 struct turnstile **tstore, thread_t owner, block_hint_t block_hint, 172 uint64_t deadline); 173 174 void (*psynch_wait_update_complete)(struct turnstile *turnstile); 175 176 void (*psynch_wait_complete)(uintptr_t kwq, struct turnstile **tstore); 177 178 void (*psynch_wait_cleanup)(void); 179 180 kern_return_t (*psynch_wait_wakeup)(uintptr_t kwq, 181 struct ksyn_waitq_element *kwe, struct turnstile **tstore); 182 183 void (*psynch_wait_update_owner)(uintptr_t kwq, thread_t owner, 184 struct turnstile **tstore); 185 186 void* (*proc_get_pthhash)(struct proc *t); 187 void (*proc_set_pthhash)(struct proc *t, void* ptr); 188 189 /* bsd/sys/user.h */ 190 void *__unused_was_uthread_get_threadlist; 191 void *__unused_was_uthread_set_threadlist; 192 void *__unused_was_uthread_get_sigmask; 193 void *__unused_was_uthread_set_sigmask; 194 195 void* (*uthread_get_uukwe)(struct uthread *t); 196 void *__unused_was_uthread_get_returnval; 197 void (*uthread_set_returnval)(struct uthread *t, int val); 198 int (*uthread_is_cancelled)(struct uthread *t); 199 200 /* vm_protos.h calls */ 201 ipc_space_t (*task_get_ipcspace)(task_t t); 202 void *__unused_was_ipc_port_copyout_send; 203 204 /* osfmk/vm/vm_map.h */ 205 kern_return_t (*vm_map_page_info)(vm_map_t map, vm_map_offset_t offset, vm_page_info_flavor_t flavor, vm_page_info_t info, mach_msg_type_number_t *count); 206 mach_port_name_t (*ipc_port_copyout_send_pinned)(ipc_port_t sright, ipc_space_t space); 207 208 /* wq functions */ 209 kern_return_t (*thread_set_wq_state32)(thread_t thread, thread_state_t state); 210 #if !defined(__arm__) 211 kern_return_t (*thread_set_wq_state64)(thread_t thread, thread_state_t state); 212 #endif 213 214 /* sched_prim.h */ 215 void (*thread_exception_return)(void); 216 void (*thread_bootstrap_return)(void); 217 218 void *__unused_was_absolutetime_to_microtime; 219 void *__unused_was_thread_set_workq_pri; 220 void *__unused_was_thread_set_workq_qos; 221 222 /* osfmk/kern/thread.h */ 223 struct uthread* (*get_bsdthread_info)(thread_t th); 224 void *__unused_was_thread_sched_call; 225 void *__unused_was_thread_static_param; 226 void *__unused_was_thread_create_workq_waiting_parameter; 227 kern_return_t (*thread_policy_set_internal)(thread_t t, thread_policy_flavor_t flavour, thread_policy_t info, mach_msg_type_number_t count); 228 229 void *__unused_was_thread_affinity_set; 230 231 /* bsd/sys/systm.h */ 232 void (*unix_syscall_return)(int error); 233 234 void *__unused_was_zalloc; 235 void *__unused_was_zfree; 236 void *__unused_was_zinit; 237 238 /* bsd/kern/kern_sig.c */ 239 void (*__pthread_testcancel)(int); 240 241 /* calls without portfolio */ 242 kern_return_t (*mach_port_deallocate)(ipc_space_t space, mach_port_name_t name); 243 kern_return_t (*semaphore_signal_internal_trap)(mach_port_name_t sema_name); 244 vm_map_t (*current_map)(void); 245 246 void *__unused_was_convert_thread_to_port; 247 void *__unused_was_thread_create; 248 249 /* mach/thread_act.h */ 250 kern_return_t (*thread_resume)(thread_act_t target_act); 251 252 /* bsd/sys/event.h */ 253 int (*kevent_workq_internal)(struct proc *p, 254 user_addr_t changelist, int nchanges, 255 user_addr_t eventlist, int nevents, 256 user_addr_t data_out, user_size_t *data_available, 257 unsigned int flags, int32_t *retval); 258 259 #if defined(__arm__) 260 void *__unused_was_map_is_1gb; 261 #endif 262 263 void *__unused_was_proc_get_dispatchqueue_serialno_offset; 264 void *__unused_was_proc_set_dispatchqueue_serialno_offset; 265 266 void *__unused_was_proc_usynch_thread_qos_add_override_for_resource_check_owner; 267 void (*proc_set_workqueue_quantum_offset)(struct proc *p, uint64_t offset); 268 269 uint32_t (*proc_get_pthread_tsd_offset)(struct proc *p); 270 void (*proc_set_pthread_tsd_offset)(struct proc *p, uint32_t pthread_tsd_offset); 271 272 kern_return_t (*thread_set_tsd_base)(thread_t thread, mach_vm_offset_t tsd_base); 273 274 int (*proc_usynch_get_requested_thread_qos)(struct uthread *); 275 uint64_t (*proc_get_mach_thread_self_tsd_offset)(struct proc *p); 276 void (*proc_set_mach_thread_self_tsd_offset)(struct proc *p, uint64_t mach_thread_self_tsd_offset); 277 278 kern_return_t (*thread_policy_get)(thread_t t, thread_policy_flavor_t flavor, thread_policy_t info, mach_msg_type_number_t *count, boolean_t *get_default); 279 boolean_t (*qos_main_thread_active)(void); 280 281 kern_return_t (*thread_set_voucher_name)(mach_port_name_t name); 282 283 boolean_t (*proc_usynch_thread_qos_add_override_for_resource)(task_t task, struct uthread *, uint64_t tid, int override_qos, boolean_t first_override_for_resource, user_addr_t resource, int resource_type); 284 boolean_t (*proc_usynch_thread_qos_remove_override_for_resource)(task_t task, struct uthread *, uint64_t tid, user_addr_t resource, int resource_type); 285 void *__unused_was_proc_usynch_thread_qos_reset_override_for_resource; 286 287 void *__unused_was_proc_init_wqptr_or_wait; 288 289 uint16_t (*thread_set_tag)(thread_t thread, uint16_t tag); 290 uint16_t (*thread_get_tag)(thread_t thread); 291 292 void *__unused_was_thread_create_pinned; 293 294 kern_return_t (*thread_terminate_pinned)(thread_t thread); 295 ipc_port_t (*convert_thread_to_port_pinned)(thread_t th); 296 297 user_addr_t (*proc_get_stack_addr_hint)(struct proc *p); 298 void (*proc_set_stack_addr_hint)(struct proc *p, user_addr_t stack_addr_hint); 299 300 kern_return_t (*thread_create_immovable)(task_t parent_task, thread_t *new_thread); 301 void (*proc_set_return_to_kernel_offset)(struct proc *t, uint64_t offset); 302 303 void *__unused_was_workloop_fulfill_threadreq; 304 void (*thread_will_park_or_terminate)(thread_t thread); 305 306 void *__unused_was_qos_max_parallelism; 307 308 /* proc_internal.h: struct proc user_stack accessor */ 309 user_addr_t (*proc_get_user_stack)(struct proc *p); 310 void *__unused_was_proc_set_user_stack; 311 312 /* padding for future */ 313 void* _pad[69]; 314 } *pthread_callbacks_t; 315 316 void 317 pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks); 318 319 #ifdef BSD_KERNEL_PRIVATE 320 void thread_will_park_or_terminate(thread_t thread); 321 void pthread_init(void); 322 extern pthread_callbacks_t pthread_kern; 323 extern pthread_functions_t pthread_functions; 324 #endif 325 326 #endif /* ASSEMBLER */ 327 #endif /* _PTHREAD_SHIMS_H_ */ 328 #endif /* KERNEL_PRIVATE */ 329