1 /*
2 * Copyright (c) 1999-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Kernel Control domain - allows control connections to
31 * and to read/write data.
32 *
33 * Vincent Lubet, 040506
34 * Christophe Allie, 010928
35 * Justin C. Walker, 990319
36 */
37
38 #include <sys/types.h>
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/syslog.h>
42 #include <sys/socket.h>
43 #include <sys/socketvar.h>
44 #include <sys/protosw.h>
45 #include <sys/domain.h>
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/sys_domain.h>
49 #include <sys/kern_event.h>
50 #include <sys/kern_control.h>
51 #include <sys/kauth.h>
52 #include <sys/sysctl.h>
53 #include <sys/proc_info.h>
54 #include <net/if_var.h>
55
56 #include <mach/vm_types.h>
57
58 #include <kern/thread.h>
59
60 struct kctl {
61 TAILQ_ENTRY(kctl) next; /* controller chain */
62 kern_ctl_ref kctlref;
63
64 /* controller information provided when registering */
65 char name[MAX_KCTL_NAME]; /* unique identifier */
66 u_int32_t id;
67 u_int32_t reg_unit;
68
69 /* misc communication information */
70 u_int32_t flags; /* support flags */
71 u_int32_t recvbufsize; /* request more than the default buffer size */
72 u_int32_t sendbufsize; /* request more than the default buffer size */
73
74 /* Dispatch functions */
75 ctl_setup_func setup; /* Setup contact */
76 ctl_bind_func bind; /* Prepare contact */
77 ctl_connect_func connect; /* Make contact */
78 ctl_disconnect_func disconnect; /* Break contact */
79 ctl_send_func send; /* Send data to nke */
80 ctl_send_list_func send_list; /* Send list of packets */
81 ctl_setopt_func setopt; /* set kctl configuration */
82 ctl_getopt_func getopt; /* get kctl configuration */
83 ctl_rcvd_func rcvd; /* Notify nke when client reads data */
84
85 TAILQ_HEAD(, ctl_cb) kcb_head;
86 u_int32_t lastunit;
87 };
88
89 #if DEVELOPMENT || DEBUG
90 enum ctl_status {
91 KCTL_DISCONNECTED = 0,
92 KCTL_CONNECTING = 1,
93 KCTL_CONNECTED = 2
94 };
95 #endif /* DEVELOPMENT || DEBUG */
96
97 struct ctl_cb {
98 TAILQ_ENTRY(ctl_cb) next; /* controller chain */
99 lck_mtx_t mtx;
100 struct socket *so; /* controlling socket */
101 struct kctl *kctl; /* back pointer to controller */
102 void *userdata;
103 struct sockaddr_ctl sac;
104 u_int32_t usecount;
105 u_int32_t kcb_usecount;
106 u_int32_t require_clearing_count;
107 #if DEVELOPMENT || DEBUG
108 enum ctl_status status;
109 #endif /* DEVELOPMENT || DEBUG */
110 };
111
112 #ifndef ROUNDUP64
113 #define ROUNDUP64(x) P2ROUNDUP((x), sizeof (u_int64_t))
114 #endif
115
116 #ifndef ADVANCE64
117 #define ADVANCE64(p, n) (void*)((char *)(p) + ROUNDUP64(n))
118 #endif
119
120 /*
121 * Definitions and vars for we support
122 */
123
124 #define CTL_SENDSIZE (2 * 1024) /* default buffer size */
125 #define CTL_RECVSIZE (8 * 1024) /* default buffer size */
126
127 /*
128 * Definitions and vars for we support
129 */
130
131 const u_int32_t ctl_maxunit = 65536;
132 static LCK_ATTR_DECLARE(ctl_lck_attr, 0, 0);
133 static LCK_GRP_DECLARE(ctl_lck_grp, "Kernel Control Protocol");
134 static LCK_MTX_DECLARE_ATTR(ctl_mtx, &ctl_lck_grp, &ctl_lck_attr);
135
136 /* all the controllers are chained */
137 TAILQ_HEAD(kctl_list, kctl) ctl_head = TAILQ_HEAD_INITIALIZER(ctl_head);
138
139 static int ctl_attach(struct socket *, int, struct proc *);
140 static int ctl_detach(struct socket *);
141 static int ctl_sofreelastref(struct socket *so);
142 static int ctl_bind(struct socket *, struct sockaddr *, struct proc *);
143 static int ctl_connect(struct socket *, struct sockaddr *, struct proc *);
144 static int ctl_disconnect(struct socket *);
145 static int ctl_ioctl(struct socket *so, u_long cmd, caddr_t data,
146 struct ifnet *ifp, struct proc *p);
147 static int ctl_send(struct socket *, int, struct mbuf *,
148 struct sockaddr *, struct mbuf *, struct proc *);
149 static int ctl_send_list(struct socket *, int, struct mbuf *,
150 struct sockaddr *, struct mbuf *, struct proc *);
151 static int ctl_ctloutput(struct socket *, struct sockopt *);
152 static int ctl_peeraddr(struct socket *so, struct sockaddr **nam);
153 static int ctl_usr_rcvd(struct socket *so, int flags);
154
155 static struct kctl *ctl_find_by_name(const char *);
156 static struct kctl *ctl_find_by_id_unit(u_int32_t id, u_int32_t unit);
157
158 static struct socket *kcb_find_socket(kern_ctl_ref kctlref, u_int32_t unit,
159 u_int32_t *);
160 static struct ctl_cb *kcb_find(struct kctl *, u_int32_t unit);
161 static void ctl_post_msg(u_int32_t event_code, u_int32_t id);
162
163 static int ctl_lock(struct socket *, int, void *);
164 static int ctl_unlock(struct socket *, int, void *);
165 static lck_mtx_t * ctl_getlock(struct socket *, int);
166
167 static struct pr_usrreqs ctl_usrreqs = {
168 .pru_attach = ctl_attach,
169 .pru_bind = ctl_bind,
170 .pru_connect = ctl_connect,
171 .pru_control = ctl_ioctl,
172 .pru_detach = ctl_detach,
173 .pru_disconnect = ctl_disconnect,
174 .pru_peeraddr = ctl_peeraddr,
175 .pru_rcvd = ctl_usr_rcvd,
176 .pru_send = ctl_send,
177 .pru_send_list = ctl_send_list,
178 .pru_sosend = sosend,
179 .pru_sosend_list = sosend_list,
180 .pru_soreceive = soreceive,
181 .pru_soreceive_list = soreceive_list,
182 };
183
184 static struct protosw kctlsw[] = {
185 {
186 .pr_type = SOCK_DGRAM,
187 .pr_protocol = SYSPROTO_CONTROL,
188 .pr_flags = PR_ATOMIC | PR_CONNREQUIRED | PR_PCBLOCK | PR_WANTRCVD,
189 .pr_ctloutput = ctl_ctloutput,
190 .pr_usrreqs = &ctl_usrreqs,
191 .pr_lock = ctl_lock,
192 .pr_unlock = ctl_unlock,
193 .pr_getlock = ctl_getlock,
194 },
195 {
196 .pr_type = SOCK_STREAM,
197 .pr_protocol = SYSPROTO_CONTROL,
198 .pr_flags = PR_CONNREQUIRED | PR_PCBLOCK | PR_WANTRCVD,
199 .pr_ctloutput = ctl_ctloutput,
200 .pr_usrreqs = &ctl_usrreqs,
201 .pr_lock = ctl_lock,
202 .pr_unlock = ctl_unlock,
203 .pr_getlock = ctl_getlock,
204 }
205 };
206
207 __private_extern__ int kctl_reg_list SYSCTL_HANDLER_ARGS;
208 __private_extern__ int kctl_pcblist SYSCTL_HANDLER_ARGS;
209 __private_extern__ int kctl_getstat SYSCTL_HANDLER_ARGS;
210
211
212 SYSCTL_NODE(_net_systm, OID_AUTO, kctl,
213 CTLFLAG_RW | CTLFLAG_LOCKED, 0, "Kernel control family");
214
215 struct kctlstat kctlstat;
216 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, stats,
217 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
218 kctl_getstat, "S,kctlstat", "");
219
220 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, reg_list,
221 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
222 kctl_reg_list, "S,xkctl_reg", "");
223
224 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, pcblist,
225 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
226 kctl_pcblist, "S,xkctlpcb", "");
227
228 u_int32_t ctl_autorcvbuf_max = 256 * 1024;
229 SYSCTL_INT(_net_systm_kctl, OID_AUTO, autorcvbufmax,
230 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_autorcvbuf_max, 0, "");
231
232 u_int32_t ctl_autorcvbuf_high = 0;
233 SYSCTL_INT(_net_systm_kctl, OID_AUTO, autorcvbufhigh,
234 CTLFLAG_RD | CTLFLAG_LOCKED, &ctl_autorcvbuf_high, 0, "");
235
236 u_int32_t ctl_debug = 0;
237 SYSCTL_INT(_net_systm_kctl, OID_AUTO, debug,
238 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_debug, 0, "");
239
240 #if DEVELOPMENT || DEBUG
241 u_int32_t ctl_panic_debug = 0;
242 SYSCTL_INT(_net_systm_kctl, OID_AUTO, panicdebug,
243 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_panic_debug, 0, "");
244 #endif /* DEVELOPMENT || DEBUG */
245
246 #define KCTL_TBL_INC 16
247
248 static uintptr_t kctl_tbl_size = 0;
249 static u_int32_t kctl_tbl_growing = 0;
250 static u_int32_t kctl_tbl_growing_waiting = 0;
251 static uintptr_t kctl_tbl_count = 0;
252 static struct kctl **kctl_table = NULL;
253 static uintptr_t kctl_ref_gencnt = 0;
254
255 static void kctl_tbl_grow(void);
256 static kern_ctl_ref kctl_make_ref(struct kctl *kctl);
257 static void kctl_delete_ref(kern_ctl_ref);
258 static struct kctl *kctl_from_ref(kern_ctl_ref);
259
260 /*
261 * Install the protosw's for the Kernel Control manager.
262 */
263 __private_extern__ void
kern_control_init(struct domain * dp)264 kern_control_init(struct domain *dp)
265 {
266 struct protosw *pr;
267 int i;
268 int kctl_proto_count = (sizeof(kctlsw) / sizeof(struct protosw));
269
270 VERIFY(!(dp->dom_flags & DOM_INITIALIZED));
271 VERIFY(dp == systemdomain);
272
273 for (i = 0, pr = &kctlsw[0]; i < kctl_proto_count; i++, pr++) {
274 net_add_proto(pr, dp, 1);
275 }
276 }
277
278 static void
kcb_delete(struct ctl_cb * kcb)279 kcb_delete(struct ctl_cb *kcb)
280 {
281 if (kcb != 0) {
282 lck_mtx_destroy(&kcb->mtx, &ctl_lck_grp);
283 kfree_type(struct ctl_cb, kcb);
284 }
285 }
286
287 /*
288 * Kernel Controller user-request functions
289 * attach function must exist and succeed
290 * detach not necessary
291 * we need a pcb for the per socket mutex
292 */
293 static int
ctl_attach(struct socket * so,int proto,struct proc * p)294 ctl_attach(struct socket *so, int proto, struct proc *p)
295 {
296 #pragma unused(proto, p)
297 struct ctl_cb *kcb = 0;
298
299 kcb = kalloc_type(struct ctl_cb, Z_WAITOK | Z_ZERO | Z_NOFAIL);
300
301 lck_mtx_init(&kcb->mtx, &ctl_lck_grp, &ctl_lck_attr);
302 kcb->so = so;
303 so->so_pcb = (caddr_t)kcb;
304
305 return 0;
306 }
307
308 static int
ctl_sofreelastref(struct socket * so)309 ctl_sofreelastref(struct socket *so)
310 {
311 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
312
313 so->so_pcb = 0;
314
315 if (kcb != 0) {
316 struct kctl *kctl;
317 if ((kctl = kcb->kctl) != 0) {
318 lck_mtx_lock(&ctl_mtx);
319 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
320 kctlstat.kcs_pcbcount--;
321 kctlstat.kcs_gencnt++;
322 lck_mtx_unlock(&ctl_mtx);
323 }
324 kcb_delete(kcb);
325 }
326 sofreelastref(so, 1);
327 return 0;
328 }
329
330 /*
331 * Use this function and ctl_kcb_require_clearing to serialize
332 * critical calls into the kctl subsystem
333 */
334 static void
ctl_kcb_increment_use_count(struct ctl_cb * kcb,lck_mtx_t * mutex_held)335 ctl_kcb_increment_use_count(struct ctl_cb *kcb, lck_mtx_t *mutex_held)
336 {
337 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
338 while (kcb->require_clearing_count > 0) {
339 msleep(&kcb->require_clearing_count, mutex_held, PSOCK | PCATCH, "kcb_require_clearing", NULL);
340 }
341 kcb->kcb_usecount++;
342 }
343
344 static void
ctl_kcb_require_clearing(struct ctl_cb * kcb,lck_mtx_t * mutex_held)345 ctl_kcb_require_clearing(struct ctl_cb *kcb, lck_mtx_t *mutex_held)
346 {
347 assert(kcb->kcb_usecount != 0);
348 kcb->require_clearing_count++;
349 kcb->kcb_usecount--;
350 while (kcb->kcb_usecount > 0) { // we need to wait until no one else is running
351 msleep(&kcb->kcb_usecount, mutex_held, PSOCK | PCATCH, "kcb_usecount", NULL);
352 }
353 kcb->kcb_usecount++;
354 }
355
356 static void
ctl_kcb_done_clearing(struct ctl_cb * kcb)357 ctl_kcb_done_clearing(struct ctl_cb *kcb)
358 {
359 assert(kcb->require_clearing_count != 0);
360 kcb->require_clearing_count--;
361 wakeup((caddr_t)&kcb->require_clearing_count);
362 }
363
364 static void
ctl_kcb_decrement_use_count(struct ctl_cb * kcb)365 ctl_kcb_decrement_use_count(struct ctl_cb *kcb)
366 {
367 assert(kcb->kcb_usecount != 0);
368 kcb->kcb_usecount--;
369 wakeup((caddr_t)&kcb->kcb_usecount);
370 }
371
372 static int
ctl_detach(struct socket * so)373 ctl_detach(struct socket *so)
374 {
375 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
376
377 if (kcb == 0) {
378 return 0;
379 }
380
381 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
382 ctl_kcb_increment_use_count(kcb, mtx_held);
383 ctl_kcb_require_clearing(kcb, mtx_held);
384
385 if (kcb->kctl != NULL && kcb->kctl->bind != NULL &&
386 kcb->userdata != NULL && !(so->so_state & SS_ISCONNECTED)) {
387 // The unit was bound, but not connected
388 // Invoke the disconnected call to cleanup
389 if (kcb->kctl->disconnect != NULL) {
390 socket_unlock(so, 0);
391 (*kcb->kctl->disconnect)(kcb->kctl->kctlref,
392 kcb->sac.sc_unit, kcb->userdata);
393 socket_lock(so, 0);
394 }
395 }
396
397 soisdisconnected(so);
398 #if DEVELOPMENT || DEBUG
399 kcb->status = KCTL_DISCONNECTED;
400 #endif /* DEVELOPMENT || DEBUG */
401 so->so_flags |= SOF_PCBCLEARING;
402 ctl_kcb_done_clearing(kcb);
403 ctl_kcb_decrement_use_count(kcb);
404 return 0;
405 }
406
407 static int
ctl_setup_kctl(struct socket * so,struct sockaddr * nam,struct proc * p)408 ctl_setup_kctl(struct socket *so, struct sockaddr *nam, struct proc *p)
409 {
410 struct kctl *kctl = NULL;
411 int error = 0;
412 struct sockaddr_ctl sa;
413 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
414 struct ctl_cb *kcb_next = NULL;
415 u_quad_t sbmaxsize;
416 u_int32_t recvbufsize, sendbufsize;
417
418 if (kcb == 0) {
419 panic("ctl_setup_kctl so_pcb null");
420 }
421
422 if (kcb->kctl != NULL) {
423 // Already set up, skip
424 return 0;
425 }
426
427 if (nam->sa_len != sizeof(struct sockaddr_ctl)) {
428 return EINVAL;
429 }
430
431 bcopy(nam, &sa, sizeof(struct sockaddr_ctl));
432
433 lck_mtx_lock(&ctl_mtx);
434 kctl = ctl_find_by_id_unit(sa.sc_id, sa.sc_unit);
435 if (kctl == NULL) {
436 lck_mtx_unlock(&ctl_mtx);
437 return ENOENT;
438 }
439
440 if (((kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&
441 (so->so_type != SOCK_STREAM)) ||
442 (!(kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&
443 (so->so_type != SOCK_DGRAM))) {
444 lck_mtx_unlock(&ctl_mtx);
445 return EPROTOTYPE;
446 }
447
448 if (kctl->flags & CTL_FLAG_PRIVILEGED) {
449 if (p == 0) {
450 lck_mtx_unlock(&ctl_mtx);
451 return EINVAL;
452 }
453 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
454 lck_mtx_unlock(&ctl_mtx);
455 return EPERM;
456 }
457 }
458
459 if ((kctl->flags & CTL_FLAG_REG_ID_UNIT) || sa.sc_unit != 0) {
460 if (kcb_find(kctl, sa.sc_unit) != NULL) {
461 lck_mtx_unlock(&ctl_mtx);
462 return EBUSY;
463 }
464 } else if (kctl->setup != NULL) {
465 error = (*kctl->setup)(&sa.sc_unit, &kcb->userdata);
466 if (error != 0) {
467 lck_mtx_unlock(&ctl_mtx);
468 return error;
469 }
470 } else {
471 /* Find an unused ID, assumes control IDs are in order */
472 u_int32_t unit = 1;
473
474 TAILQ_FOREACH(kcb_next, &kctl->kcb_head, next) {
475 if (kcb_next->sac.sc_unit > unit) {
476 /* Found a gap, lets fill it in */
477 break;
478 }
479 unit = kcb_next->sac.sc_unit + 1;
480 if (unit == ctl_maxunit) {
481 break;
482 }
483 }
484
485 if (unit == ctl_maxunit) {
486 lck_mtx_unlock(&ctl_mtx);
487 return EBUSY;
488 }
489
490 sa.sc_unit = unit;
491 }
492
493 bcopy(&sa, &kcb->sac, sizeof(struct sockaddr_ctl));
494 kcb->kctl = kctl;
495 if (kcb_next != NULL) {
496 TAILQ_INSERT_BEFORE(kcb_next, kcb, next);
497 } else {
498 TAILQ_INSERT_TAIL(&kctl->kcb_head, kcb, next);
499 }
500 kctlstat.kcs_pcbcount++;
501 kctlstat.kcs_gencnt++;
502 kctlstat.kcs_connections++;
503 lck_mtx_unlock(&ctl_mtx);
504
505 /*
506 * rdar://15526688: Limit the send and receive sizes to sb_max
507 * by using the same scaling as sbreserve()
508 */
509 sbmaxsize = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES);
510
511 if (kctl->sendbufsize > sbmaxsize) {
512 sendbufsize = (u_int32_t)sbmaxsize;
513 } else {
514 sendbufsize = kctl->sendbufsize;
515 }
516
517 if (kctl->recvbufsize > sbmaxsize) {
518 recvbufsize = (u_int32_t)sbmaxsize;
519 } else {
520 recvbufsize = kctl->recvbufsize;
521 }
522
523 error = soreserve(so, sendbufsize, recvbufsize);
524 if (error) {
525 if (ctl_debug) {
526 printf("%s - soreserve(%llx, %u, %u) error %d\n",
527 __func__, (uint64_t)VM_KERNEL_ADDRPERM(so),
528 sendbufsize, recvbufsize, error);
529 }
530 goto done;
531 }
532
533 done:
534 if (error) {
535 soisdisconnected(so);
536 #if DEVELOPMENT || DEBUG
537 kcb->status = KCTL_DISCONNECTED;
538 #endif /* DEVELOPMENT || DEBUG */
539 lck_mtx_lock(&ctl_mtx);
540 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
541 kcb->kctl = NULL;
542 kcb->sac.sc_unit = 0;
543 kctlstat.kcs_pcbcount--;
544 kctlstat.kcs_gencnt++;
545 kctlstat.kcs_conn_fail++;
546 lck_mtx_unlock(&ctl_mtx);
547 }
548 return error;
549 }
550
551 static int
ctl_bind(struct socket * so,struct sockaddr * nam,struct proc * p)552 ctl_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
553 {
554 int error = 0;
555 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
556
557 if (kcb == NULL) {
558 panic("ctl_bind so_pcb null");
559 }
560
561 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
562 ctl_kcb_increment_use_count(kcb, mtx_held);
563 ctl_kcb_require_clearing(kcb, mtx_held);
564
565 error = ctl_setup_kctl(so, nam, p);
566 if (error) {
567 goto out;
568 }
569
570 if (kcb->kctl == NULL) {
571 panic("ctl_bind kctl null");
572 }
573
574 if (kcb->kctl->bind == NULL) {
575 error = EINVAL;
576 goto out;
577 }
578
579 socket_unlock(so, 0);
580 error = (*kcb->kctl->bind)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata);
581 socket_lock(so, 0);
582
583 out:
584 ctl_kcb_done_clearing(kcb);
585 ctl_kcb_decrement_use_count(kcb);
586 return error;
587 }
588
589 static int
ctl_connect(struct socket * so,struct sockaddr * nam,struct proc * p)590 ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
591 {
592 int error = 0;
593 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
594
595 if (kcb == NULL) {
596 panic("ctl_connect so_pcb null");
597 }
598
599 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
600 ctl_kcb_increment_use_count(kcb, mtx_held);
601 ctl_kcb_require_clearing(kcb, mtx_held);
602
603 #if DEVELOPMENT || DEBUG
604 if (kcb->status != KCTL_DISCONNECTED && ctl_panic_debug) {
605 panic("kctl already connecting/connected");
606 }
607 kcb->status = KCTL_CONNECTING;
608 #endif /* DEVELOPMENT || DEBUG */
609
610 error = ctl_setup_kctl(so, nam, p);
611 if (error) {
612 goto out;
613 }
614
615 if (kcb->kctl == NULL) {
616 panic("ctl_connect kctl null");
617 }
618
619 soisconnecting(so);
620 socket_unlock(so, 0);
621 error = (*kcb->kctl->connect)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata);
622 socket_lock(so, 0);
623 if (error) {
624 goto end;
625 }
626 soisconnected(so);
627 #if DEVELOPMENT || DEBUG
628 kcb->status = KCTL_CONNECTED;
629 #endif /* DEVELOPMENT || DEBUG */
630
631 end:
632 if (error && kcb->kctl->disconnect) {
633 /*
634 * XXX Make sure we Don't check the return value
635 * of disconnect here.
636 * ipsec/utun_ctl_disconnect will return error when
637 * disconnect gets called after connect failure.
638 * However if we decide to check for disconnect return
639 * value here. Please make sure to revisit
640 * ipsec/utun_ctl_disconnect.
641 */
642 socket_unlock(so, 0);
643 (*kcb->kctl->disconnect)(kcb->kctl->kctlref, kcb->sac.sc_unit, kcb->userdata);
644 socket_lock(so, 0);
645 }
646 if (error) {
647 soisdisconnected(so);
648 #if DEVELOPMENT || DEBUG
649 kcb->status = KCTL_DISCONNECTED;
650 #endif /* DEVELOPMENT || DEBUG */
651 lck_mtx_lock(&ctl_mtx);
652 TAILQ_REMOVE(&kcb->kctl->kcb_head, kcb, next);
653 kcb->kctl = NULL;
654 kcb->sac.sc_unit = 0;
655 kctlstat.kcs_pcbcount--;
656 kctlstat.kcs_gencnt++;
657 kctlstat.kcs_conn_fail++;
658 lck_mtx_unlock(&ctl_mtx);
659 }
660 out:
661 ctl_kcb_done_clearing(kcb);
662 ctl_kcb_decrement_use_count(kcb);
663 return error;
664 }
665
666 static int
ctl_disconnect(struct socket * so)667 ctl_disconnect(struct socket *so)
668 {
669 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
670
671 if ((kcb = (struct ctl_cb *)so->so_pcb)) {
672 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
673 ctl_kcb_increment_use_count(kcb, mtx_held);
674 ctl_kcb_require_clearing(kcb, mtx_held);
675 struct kctl *kctl = kcb->kctl;
676
677 if (kctl && kctl->disconnect) {
678 socket_unlock(so, 0);
679 (*kctl->disconnect)(kctl->kctlref, kcb->sac.sc_unit,
680 kcb->userdata);
681 socket_lock(so, 0);
682 }
683
684 soisdisconnected(so);
685 #if DEVELOPMENT || DEBUG
686 kcb->status = KCTL_DISCONNECTED;
687 #endif /* DEVELOPMENT || DEBUG */
688
689 socket_unlock(so, 0);
690 lck_mtx_lock(&ctl_mtx);
691 kcb->kctl = 0;
692 kcb->sac.sc_unit = 0;
693 while (kcb->usecount != 0) {
694 msleep(&kcb->usecount, &ctl_mtx, 0, "kcb->usecount", 0);
695 }
696 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
697 kctlstat.kcs_pcbcount--;
698 kctlstat.kcs_gencnt++;
699 lck_mtx_unlock(&ctl_mtx);
700 socket_lock(so, 0);
701 ctl_kcb_done_clearing(kcb);
702 ctl_kcb_decrement_use_count(kcb);
703 }
704 return 0;
705 }
706
707 static int
ctl_peeraddr(struct socket * so,struct sockaddr ** nam)708 ctl_peeraddr(struct socket *so, struct sockaddr **nam)
709 {
710 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
711 struct kctl *kctl;
712 struct sockaddr_ctl sc;
713
714 if (kcb == NULL) { /* sanity check */
715 return ENOTCONN;
716 }
717
718 if ((kctl = kcb->kctl) == NULL) {
719 return EINVAL;
720 }
721
722 bzero(&sc, sizeof(struct sockaddr_ctl));
723 sc.sc_len = sizeof(struct sockaddr_ctl);
724 sc.sc_family = AF_SYSTEM;
725 sc.ss_sysaddr = AF_SYS_CONTROL;
726 sc.sc_id = kctl->id;
727 sc.sc_unit = kcb->sac.sc_unit;
728
729 *nam = dup_sockaddr((struct sockaddr *)&sc, 1);
730
731 return 0;
732 }
733
734 static void
ctl_sbrcv_trim(struct socket * so)735 ctl_sbrcv_trim(struct socket *so)
736 {
737 struct sockbuf *sb = &so->so_rcv;
738
739 if (sb->sb_hiwat > sb->sb_idealsize) {
740 u_int32_t diff;
741 int32_t trim;
742
743 /*
744 * The difference between the ideal size and the
745 * current size is the upper bound of the trimage
746 */
747 diff = sb->sb_hiwat - sb->sb_idealsize;
748 /*
749 * We cannot trim below the outstanding data
750 */
751 trim = sb->sb_hiwat - sb->sb_cc;
752
753 trim = imin(trim, (int32_t)diff);
754
755 if (trim > 0) {
756 sbreserve(sb, (sb->sb_hiwat - trim));
757
758 if (ctl_debug) {
759 printf("%s - shrunk to %d\n",
760 __func__, sb->sb_hiwat);
761 }
762 }
763 }
764 }
765
766 static int
ctl_usr_rcvd(struct socket * so,int flags)767 ctl_usr_rcvd(struct socket *so, int flags)
768 {
769 int error = 0;
770 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
771 struct kctl *kctl;
772
773 if (kcb == NULL) {
774 return ENOTCONN;
775 }
776
777 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
778 ctl_kcb_increment_use_count(kcb, mtx_held);
779
780 if ((kctl = kcb->kctl) == NULL) {
781 error = EINVAL;
782 goto out;
783 }
784
785 if (kctl->rcvd) {
786 socket_unlock(so, 0);
787 (*kctl->rcvd)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata, flags);
788 socket_lock(so, 0);
789 }
790
791 ctl_sbrcv_trim(so);
792
793 out:
794 ctl_kcb_decrement_use_count(kcb);
795 return error;
796 }
797
798 static int
ctl_send(struct socket * so,int flags,struct mbuf * m,struct sockaddr * addr,struct mbuf * control,struct proc * p)799 ctl_send(struct socket *so, int flags, struct mbuf *m,
800 struct sockaddr *addr, struct mbuf *control,
801 struct proc *p)
802 {
803 #pragma unused(addr, p)
804 int error = 0;
805 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
806 struct kctl *kctl;
807
808 if (control) {
809 m_freem(control);
810 }
811
812 if (kcb == NULL) { /* sanity check */
813 m_freem(m);
814 return ENOTCONN;
815 }
816
817 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
818 ctl_kcb_increment_use_count(kcb, mtx_held);
819
820 if (error == 0 && (kctl = kcb->kctl) == NULL) {
821 error = EINVAL;
822 }
823
824 if (error == 0 && kctl->send) {
825 so_tc_update_stats(m, so, m_get_service_class(m));
826 socket_unlock(so, 0);
827 error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata,
828 m, flags);
829 socket_lock(so, 0);
830 } else {
831 m_freem(m);
832 if (error == 0) {
833 error = ENOTSUP;
834 }
835 }
836 if (error != 0) {
837 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_send_fail);
838 }
839 ctl_kcb_decrement_use_count(kcb);
840
841 return error;
842 }
843
844 static int
ctl_send_list(struct socket * so,int flags,struct mbuf * m,__unused struct sockaddr * addr,struct mbuf * control,__unused struct proc * p)845 ctl_send_list(struct socket *so, int flags, struct mbuf *m,
846 __unused struct sockaddr *addr, struct mbuf *control,
847 __unused struct proc *p)
848 {
849 int error = 0;
850 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
851 struct kctl *kctl;
852
853 if (control) {
854 m_freem_list(control);
855 }
856
857 if (kcb == NULL) { /* sanity check */
858 m_freem_list(m);
859 return ENOTCONN;
860 }
861
862 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
863 ctl_kcb_increment_use_count(kcb, mtx_held);
864
865 if (error == 0 && (kctl = kcb->kctl) == NULL) {
866 error = EINVAL;
867 }
868
869 if (error == 0 && kctl->send_list) {
870 struct mbuf *nxt;
871
872 for (nxt = m; nxt != NULL; nxt = nxt->m_nextpkt) {
873 so_tc_update_stats(nxt, so, m_get_service_class(nxt));
874 }
875
876 socket_unlock(so, 0);
877 error = (*kctl->send_list)(kctl->kctlref, kcb->sac.sc_unit,
878 kcb->userdata, m, flags);
879 socket_lock(so, 0);
880 } else if (error == 0 && kctl->send) {
881 while (m != NULL && error == 0) {
882 struct mbuf *nextpkt = m->m_nextpkt;
883
884 m->m_nextpkt = NULL;
885 so_tc_update_stats(m, so, m_get_service_class(m));
886 socket_unlock(so, 0);
887 error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit,
888 kcb->userdata, m, flags);
889 socket_lock(so, 0);
890 m = nextpkt;
891 }
892 if (m != NULL) {
893 m_freem_list(m);
894 }
895 } else {
896 m_freem_list(m);
897 if (error == 0) {
898 error = ENOTSUP;
899 }
900 }
901 if (error != 0) {
902 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_send_list_fail);
903 }
904 ctl_kcb_decrement_use_count(kcb);
905
906 return error;
907 }
908
909 static errno_t
ctl_rcvbspace(struct socket * so,size_t datasize,u_int32_t kctlflags,u_int32_t flags)910 ctl_rcvbspace(struct socket *so, size_t datasize,
911 u_int32_t kctlflags, u_int32_t flags)
912 {
913 struct sockbuf *sb = &so->so_rcv;
914 u_int32_t space = sbspace(sb);
915 errno_t error;
916
917 if ((kctlflags & CTL_FLAG_REG_CRIT) == 0) {
918 if ((u_int32_t) space >= datasize) {
919 error = 0;
920 } else {
921 error = ENOBUFS;
922 }
923 } else if ((flags & CTL_DATA_CRIT) == 0) {
924 /*
925 * Reserve 25% for critical messages
926 */
927 if (space < (sb->sb_hiwat >> 2) ||
928 space < datasize) {
929 error = ENOBUFS;
930 } else {
931 error = 0;
932 }
933 } else {
934 size_t autorcvbuf_max;
935
936 /*
937 * Allow overcommit of 25%
938 */
939 autorcvbuf_max = min(sb->sb_idealsize + (sb->sb_idealsize >> 2),
940 ctl_autorcvbuf_max);
941
942 if ((u_int32_t) space >= datasize) {
943 error = 0;
944 } else if (tcp_cansbgrow(sb) &&
945 sb->sb_hiwat < autorcvbuf_max) {
946 /*
947 * Grow with a little bit of leeway
948 */
949 size_t grow = datasize - space + MSIZE;
950 u_int32_t cc = (u_int32_t)MIN(MIN((sb->sb_hiwat + grow), autorcvbuf_max), UINT32_MAX);
951
952 if (sbreserve(sb, cc) == 1) {
953 if (sb->sb_hiwat > ctl_autorcvbuf_high) {
954 ctl_autorcvbuf_high = sb->sb_hiwat;
955 }
956
957 /*
958 * A final check
959 */
960 if ((u_int32_t) sbspace(sb) >= datasize) {
961 error = 0;
962 } else {
963 error = ENOBUFS;
964 }
965
966 if (ctl_debug) {
967 printf("%s - grown to %d error %d\n",
968 __func__, sb->sb_hiwat, error);
969 }
970 } else {
971 error = ENOBUFS;
972 }
973 } else {
974 error = ENOBUFS;
975 }
976 }
977 return error;
978 }
979
980 errno_t
ctl_enqueuembuf(kern_ctl_ref kctlref,u_int32_t unit,struct mbuf * m,u_int32_t flags)981 ctl_enqueuembuf(kern_ctl_ref kctlref, u_int32_t unit, struct mbuf *m,
982 u_int32_t flags)
983 {
984 struct socket *so;
985 errno_t error = 0;
986 int len = m->m_pkthdr.len;
987 u_int32_t kctlflags;
988
989 so = kcb_find_socket(kctlref, unit, &kctlflags);
990 if (so == NULL) {
991 return EINVAL;
992 }
993
994 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
995 error = ENOBUFS;
996 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
997 goto bye;
998 }
999 if ((flags & CTL_DATA_EOR)) {
1000 m->m_flags |= M_EOR;
1001 }
1002
1003 so_recv_data_stat(so, m, 0);
1004 if (sbappend_nodrop(&so->so_rcv, m) != 0) {
1005 if ((flags & CTL_DATA_NOWAKEUP) == 0) {
1006 sorwakeup(so);
1007 }
1008 } else {
1009 error = ENOBUFS;
1010 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1011 }
1012 bye:
1013 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1014 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1015 __func__, error, len,
1016 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1017 }
1018
1019 socket_unlock(so, 1);
1020 if (error != 0) {
1021 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1022 }
1023
1024 return error;
1025 }
1026
1027 /*
1028 * Compute space occupied by mbuf like sbappendrecord
1029 */
1030 static int
m_space(struct mbuf * m)1031 m_space(struct mbuf *m)
1032 {
1033 int space = 0;
1034 struct mbuf *nxt;
1035
1036 for (nxt = m; nxt != NULL; nxt = nxt->m_next) {
1037 space += nxt->m_len;
1038 }
1039
1040 return space;
1041 }
1042
1043 errno_t
ctl_enqueuembuf_list(void * kctlref,u_int32_t unit,struct mbuf * m_list,u_int32_t flags,struct mbuf ** m_remain)1044 ctl_enqueuembuf_list(void *kctlref, u_int32_t unit, struct mbuf *m_list,
1045 u_int32_t flags, struct mbuf **m_remain)
1046 {
1047 struct socket *so = NULL;
1048 errno_t error = 0;
1049 struct mbuf *m, *nextpkt;
1050 int needwakeup = 0;
1051 int len = 0;
1052 u_int32_t kctlflags;
1053
1054 /*
1055 * Need to point the beginning of the list in case of early exit
1056 */
1057 m = m_list;
1058
1059 /*
1060 * kcb_find_socket takes the socket lock with a reference
1061 */
1062 so = kcb_find_socket(kctlref, unit, &kctlflags);
1063 if (so == NULL) {
1064 error = EINVAL;
1065 goto done;
1066 }
1067
1068 if (kctlflags & CTL_FLAG_REG_SOCK_STREAM) {
1069 error = EOPNOTSUPP;
1070 goto done;
1071 }
1072 if (flags & CTL_DATA_EOR) {
1073 error = EINVAL;
1074 goto done;
1075 }
1076
1077 for (m = m_list; m != NULL; m = nextpkt) {
1078 nextpkt = m->m_nextpkt;
1079
1080 if (m->m_pkthdr.len == 0 && ctl_debug) {
1081 printf("%s: %llx m_pkthdr.len is 0",
1082 __func__, (uint64_t)VM_KERNEL_ADDRPERM(m));
1083 }
1084
1085 /*
1086 * The mbuf is either appended or freed by sbappendrecord()
1087 * so it's not reliable from a data standpoint
1088 */
1089 len = m_space(m);
1090 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
1091 error = ENOBUFS;
1092 OSIncrementAtomic64(
1093 (SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1094 break;
1095 } else {
1096 /*
1097 * Unlink from the list, m is on its own
1098 */
1099 m->m_nextpkt = NULL;
1100 so_recv_data_stat(so, m, 0);
1101 if (sbappendrecord_nodrop(&so->so_rcv, m) != 0) {
1102 needwakeup = 1;
1103 } else {
1104 /*
1105 * We free or return the remaining
1106 * mbufs in the list
1107 */
1108 m = nextpkt;
1109 error = ENOBUFS;
1110 OSIncrementAtomic64(
1111 (SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1112 break;
1113 }
1114 }
1115 }
1116 if (needwakeup && (flags & CTL_DATA_NOWAKEUP) == 0) {
1117 sorwakeup(so);
1118 }
1119
1120 done:
1121 if (so != NULL) {
1122 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1123 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1124 __func__, error, len,
1125 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1126 }
1127
1128 socket_unlock(so, 1);
1129 }
1130 if (m_remain) {
1131 *m_remain = m;
1132
1133 if (m != NULL && socket_debug && so != NULL &&
1134 (so->so_options & SO_DEBUG)) {
1135 struct mbuf *n;
1136
1137 printf("%s m_list %llx\n", __func__,
1138 (uint64_t) VM_KERNEL_ADDRPERM(m_list));
1139 for (n = m; n != NULL; n = n->m_nextpkt) {
1140 printf(" remain %llx m_next %llx\n",
1141 (uint64_t) VM_KERNEL_ADDRPERM(n),
1142 (uint64_t) VM_KERNEL_ADDRPERM(n->m_next));
1143 }
1144 }
1145 } else {
1146 if (m != NULL) {
1147 m_freem_list(m);
1148 }
1149 }
1150 if (error != 0) {
1151 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1152 }
1153 return error;
1154 }
1155
1156 errno_t
ctl_enqueuedata(void * kctlref,u_int32_t unit,void * data,size_t len,u_int32_t flags)1157 ctl_enqueuedata(void *kctlref, u_int32_t unit, void *data, size_t len,
1158 u_int32_t flags)
1159 {
1160 struct socket *so;
1161 struct mbuf *m;
1162 errno_t error = 0;
1163 unsigned int num_needed;
1164 struct mbuf *n;
1165 size_t curlen = 0;
1166 u_int32_t kctlflags;
1167
1168 so = kcb_find_socket(kctlref, unit, &kctlflags);
1169 if (so == NULL) {
1170 return EINVAL;
1171 }
1172
1173 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
1174 error = ENOBUFS;
1175 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1176 goto bye;
1177 }
1178
1179 num_needed = 1;
1180 m = m_allocpacket_internal(&num_needed, len, NULL, M_NOWAIT, 1, 0);
1181 if (m == NULL) {
1182 kctlstat.kcs_enqdata_mb_alloc_fail++;
1183 if (ctl_debug) {
1184 printf("%s: m_allocpacket_internal(%lu) failed\n",
1185 __func__, len);
1186 }
1187 error = ENOMEM;
1188 goto bye;
1189 }
1190
1191 for (n = m; n != NULL; n = n->m_next) {
1192 size_t mlen = mbuf_maxlen(n);
1193
1194 if (mlen + curlen > len) {
1195 mlen = len - curlen;
1196 }
1197 n->m_len = (int32_t)mlen;
1198 bcopy((char *)data + curlen, n->m_data, mlen);
1199 curlen += mlen;
1200 }
1201 mbuf_pkthdr_setlen(m, curlen);
1202
1203 if ((flags & CTL_DATA_EOR)) {
1204 m->m_flags |= M_EOR;
1205 }
1206 so_recv_data_stat(so, m, 0);
1207 /*
1208 * No need to call the "nodrop" variant of sbappend
1209 * because the mbuf is local to the scope of the function
1210 */
1211 if (sbappend(&so->so_rcv, m) != 0) {
1212 if ((flags & CTL_DATA_NOWAKEUP) == 0) {
1213 sorwakeup(so);
1214 }
1215 } else {
1216 kctlstat.kcs_enqdata_sbappend_fail++;
1217 error = ENOBUFS;
1218 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1219 }
1220
1221 bye:
1222 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1223 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1224 __func__, error, (int)len,
1225 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1226 }
1227
1228 socket_unlock(so, 1);
1229 if (error != 0) {
1230 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1231 }
1232 return error;
1233 }
1234
1235 errno_t
ctl_getenqueuepacketcount(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * pcnt)1236 ctl_getenqueuepacketcount(kern_ctl_ref kctlref, u_int32_t unit, u_int32_t *pcnt)
1237 {
1238 struct socket *so;
1239 u_int32_t cnt;
1240 struct mbuf *m1;
1241
1242 if (pcnt == NULL) {
1243 return EINVAL;
1244 }
1245
1246 so = kcb_find_socket(kctlref, unit, NULL);
1247 if (so == NULL) {
1248 return EINVAL;
1249 }
1250
1251 cnt = 0;
1252 m1 = so->so_rcv.sb_mb;
1253 while (m1 != NULL) {
1254 if (m1->m_type == MT_DATA ||
1255 m1->m_type == MT_HEADER ||
1256 m1->m_type == MT_OOBDATA) {
1257 cnt += 1;
1258 }
1259 m1 = m1->m_nextpkt;
1260 }
1261 *pcnt = cnt;
1262
1263 socket_unlock(so, 1);
1264
1265 return 0;
1266 }
1267
1268 errno_t
ctl_getenqueuespace(kern_ctl_ref kctlref,u_int32_t unit,size_t * space)1269 ctl_getenqueuespace(kern_ctl_ref kctlref, u_int32_t unit, size_t *space)
1270 {
1271 struct socket *so;
1272 long avail;
1273
1274 if (space == NULL) {
1275 return EINVAL;
1276 }
1277
1278 so = kcb_find_socket(kctlref, unit, NULL);
1279 if (so == NULL) {
1280 return EINVAL;
1281 }
1282
1283 avail = sbspace(&so->so_rcv);
1284 *space = (avail < 0) ? 0 : avail;
1285 socket_unlock(so, 1);
1286
1287 return 0;
1288 }
1289
1290 errno_t
ctl_getenqueuereadable(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * difference)1291 ctl_getenqueuereadable(kern_ctl_ref kctlref, u_int32_t unit,
1292 u_int32_t *difference)
1293 {
1294 struct socket *so;
1295
1296 if (difference == NULL) {
1297 return EINVAL;
1298 }
1299
1300 so = kcb_find_socket(kctlref, unit, NULL);
1301 if (so == NULL) {
1302 return EINVAL;
1303 }
1304
1305 if (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat) {
1306 *difference = 0;
1307 } else {
1308 *difference = (so->so_rcv.sb_lowat - so->so_rcv.sb_cc);
1309 }
1310 socket_unlock(so, 1);
1311
1312 return 0;
1313 }
1314
1315 static int
ctl_ctloutput(struct socket * so,struct sockopt * sopt)1316 ctl_ctloutput(struct socket *so, struct sockopt *sopt)
1317 {
1318 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
1319 struct kctl *kctl;
1320 int error = 0;
1321 void *data = NULL;
1322 size_t data_len = 0;
1323 size_t len;
1324
1325 if (sopt->sopt_level != SYSPROTO_CONTROL) {
1326 return EINVAL;
1327 }
1328
1329 if (kcb == NULL) { /* sanity check */
1330 return ENOTCONN;
1331 }
1332
1333 if ((kctl = kcb->kctl) == NULL) {
1334 return EINVAL;
1335 }
1336
1337 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
1338 ctl_kcb_increment_use_count(kcb, mtx_held);
1339
1340 switch (sopt->sopt_dir) {
1341 case SOPT_SET:
1342 if (kctl->setopt == NULL) {
1343 error = ENOTSUP;
1344 goto out;
1345 }
1346 if (sopt->sopt_valsize != 0) {
1347 data_len = sopt->sopt_valsize;
1348 data = kalloc_data(data_len, Z_WAITOK | Z_ZERO);
1349 if (data == NULL) {
1350 data_len = 0;
1351 error = ENOMEM;
1352 goto out;
1353 }
1354 error = sooptcopyin(sopt, data,
1355 sopt->sopt_valsize, sopt->sopt_valsize);
1356 }
1357 if (error == 0) {
1358 socket_unlock(so, 0);
1359 error = (*kctl->setopt)(kctl->kctlref,
1360 kcb->sac.sc_unit, kcb->userdata, sopt->sopt_name,
1361 data, sopt->sopt_valsize);
1362 socket_lock(so, 0);
1363 }
1364
1365 kfree_data(data, data_len);
1366 break;
1367
1368 case SOPT_GET:
1369 if (kctl->getopt == NULL) {
1370 error = ENOTSUP;
1371 goto out;
1372 }
1373
1374 if (sopt->sopt_valsize && sopt->sopt_val) {
1375 data_len = sopt->sopt_valsize;
1376 data = kalloc_data(data_len, Z_WAITOK | Z_ZERO);
1377 if (data == NULL) {
1378 data_len = 0;
1379 error = ENOMEM;
1380 goto out;
1381 }
1382 /*
1383 * 4108337 - copy user data in case the
1384 * kernel control needs it
1385 */
1386 error = sooptcopyin(sopt, data,
1387 sopt->sopt_valsize, sopt->sopt_valsize);
1388 }
1389
1390 if (error == 0) {
1391 len = sopt->sopt_valsize;
1392 socket_unlock(so, 0);
1393 error = (*kctl->getopt)(kctl->kctlref, kcb->sac.sc_unit,
1394 kcb->userdata, sopt->sopt_name,
1395 data, &len);
1396 if (data != NULL && len > sopt->sopt_valsize) {
1397 panic_plain("ctl_ctloutput: ctl %s returned "
1398 "len (%lu) > sopt_valsize (%lu)\n",
1399 kcb->kctl->name, len,
1400 sopt->sopt_valsize);
1401 }
1402 socket_lock(so, 0);
1403 if (error == 0) {
1404 if (data != NULL) {
1405 error = sooptcopyout(sopt, data, len);
1406 } else {
1407 sopt->sopt_valsize = len;
1408 }
1409 }
1410 }
1411
1412 kfree_data(data, data_len);
1413 break;
1414 }
1415
1416 out:
1417 ctl_kcb_decrement_use_count(kcb);
1418 return error;
1419 }
1420
1421 static int
ctl_ioctl(struct socket * so,u_long cmd,caddr_t data,struct ifnet * ifp,struct proc * p)1422 ctl_ioctl(struct socket *so, u_long cmd, caddr_t data,
1423 struct ifnet *ifp, struct proc *p)
1424 {
1425 #pragma unused(so, ifp, p)
1426 int error = ENOTSUP;
1427
1428 switch (cmd) {
1429 /* get the number of controllers */
1430 case CTLIOCGCOUNT: {
1431 struct kctl *kctl;
1432 u_int32_t n = 0;
1433
1434 lck_mtx_lock(&ctl_mtx);
1435 TAILQ_FOREACH(kctl, &ctl_head, next)
1436 n++;
1437 lck_mtx_unlock(&ctl_mtx);
1438
1439 bcopy(&n, data, sizeof(n));
1440 error = 0;
1441 break;
1442 }
1443 case CTLIOCGINFO: {
1444 struct ctl_info ctl_info;
1445 struct kctl *kctl = 0;
1446 size_t name_len;
1447
1448 bcopy(data, &ctl_info, sizeof(ctl_info));
1449 name_len = strnlen(ctl_info.ctl_name, MAX_KCTL_NAME);
1450
1451 if (name_len == 0 || name_len + 1 > MAX_KCTL_NAME) {
1452 error = EINVAL;
1453 break;
1454 }
1455 lck_mtx_lock(&ctl_mtx);
1456 kctl = ctl_find_by_name(ctl_info.ctl_name);
1457 lck_mtx_unlock(&ctl_mtx);
1458 if (kctl == 0) {
1459 error = ENOENT;
1460 break;
1461 }
1462 ctl_info.ctl_id = kctl->id;
1463 bcopy(&ctl_info, data, sizeof(ctl_info));
1464 error = 0;
1465 break;
1466 }
1467
1468 /* add controls to get list of NKEs */
1469 }
1470
1471 return error;
1472 }
1473
1474 static void
kctl_tbl_grow(void)1475 kctl_tbl_grow(void)
1476 {
1477 struct kctl **new_table;
1478 uintptr_t new_size;
1479
1480 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1481
1482 if (kctl_tbl_growing) {
1483 /* Another thread is allocating */
1484 kctl_tbl_growing_waiting++;
1485
1486 do {
1487 (void) msleep((caddr_t) &kctl_tbl_growing, &ctl_mtx,
1488 PSOCK | PCATCH, "kctl_tbl_growing", 0);
1489 } while (kctl_tbl_growing);
1490 kctl_tbl_growing_waiting--;
1491 }
1492 /* Another thread grew the table */
1493 if (kctl_table != NULL && kctl_tbl_count < kctl_tbl_size) {
1494 return;
1495 }
1496
1497 /* Verify we have a sane size */
1498 if (kctl_tbl_size + KCTL_TBL_INC >= UINT16_MAX) {
1499 kctlstat.kcs_tbl_size_too_big++;
1500 if (ctl_debug) {
1501 printf("%s kctl_tbl_size %lu too big\n",
1502 __func__, kctl_tbl_size);
1503 }
1504 return;
1505 }
1506 kctl_tbl_growing = 1;
1507
1508 new_size = kctl_tbl_size + KCTL_TBL_INC;
1509
1510 lck_mtx_unlock(&ctl_mtx);
1511 new_table = kalloc_type(struct kctl *, new_size, Z_WAITOK | Z_ZERO);
1512 lck_mtx_lock(&ctl_mtx);
1513
1514 if (new_table != NULL) {
1515 if (kctl_table != NULL) {
1516 bcopy(kctl_table, new_table,
1517 kctl_tbl_size * sizeof(struct kctl *));
1518
1519 kfree_type(struct kctl *, kctl_tbl_size, kctl_table);
1520 }
1521 kctl_table = new_table;
1522 kctl_tbl_size = new_size;
1523 }
1524
1525 kctl_tbl_growing = 0;
1526
1527 if (kctl_tbl_growing_waiting) {
1528 wakeup(&kctl_tbl_growing);
1529 }
1530 }
1531
1532 #define KCTLREF_INDEX_MASK 0x0000FFFF
1533 #define KCTLREF_GENCNT_MASK 0xFFFF0000
1534 #define KCTLREF_GENCNT_SHIFT 16
1535
1536 static kern_ctl_ref
kctl_make_ref(struct kctl * kctl)1537 kctl_make_ref(struct kctl *kctl)
1538 {
1539 uintptr_t i;
1540
1541 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1542
1543 if (kctl_tbl_count >= kctl_tbl_size) {
1544 kctl_tbl_grow();
1545 }
1546
1547 kctl->kctlref = NULL;
1548 for (i = 0; i < kctl_tbl_size; i++) {
1549 if (kctl_table[i] == NULL) {
1550 uintptr_t ref;
1551
1552 /*
1553 * Reference is index plus one
1554 */
1555 kctl_ref_gencnt += 1;
1556
1557 /*
1558 * Add generation count as salt to reference to prevent
1559 * use after deregister
1560 */
1561 ref = ((kctl_ref_gencnt << KCTLREF_GENCNT_SHIFT) &
1562 KCTLREF_GENCNT_MASK) +
1563 ((i + 1) & KCTLREF_INDEX_MASK);
1564
1565 kctl->kctlref = (void *)(ref);
1566 kctl_table[i] = kctl;
1567 kctl_tbl_count++;
1568 break;
1569 }
1570 }
1571
1572 if (kctl->kctlref == NULL) {
1573 panic("%s no space in table", __func__);
1574 }
1575
1576 if (ctl_debug > 0) {
1577 printf("%s %p for %p\n",
1578 __func__, kctl->kctlref, kctl);
1579 }
1580
1581 return kctl->kctlref;
1582 }
1583
1584 static void
kctl_delete_ref(kern_ctl_ref kctlref)1585 kctl_delete_ref(kern_ctl_ref kctlref)
1586 {
1587 /*
1588 * Reference is index plus one
1589 */
1590 uintptr_t i = (((uintptr_t)kctlref) & KCTLREF_INDEX_MASK) - 1;
1591
1592 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1593
1594 if (i < kctl_tbl_size) {
1595 struct kctl *kctl = kctl_table[i];
1596
1597 if (kctl->kctlref == kctlref) {
1598 kctl_table[i] = NULL;
1599 kctl_tbl_count--;
1600 } else {
1601 kctlstat.kcs_bad_kctlref++;
1602 }
1603 } else {
1604 kctlstat.kcs_bad_kctlref++;
1605 }
1606 }
1607
1608 static struct kctl *
kctl_from_ref(kern_ctl_ref kctlref)1609 kctl_from_ref(kern_ctl_ref kctlref)
1610 {
1611 /*
1612 * Reference is index plus one
1613 */
1614 uintptr_t i = (((uintptr_t)kctlref) & KCTLREF_INDEX_MASK) - 1;
1615 struct kctl *kctl = NULL;
1616
1617 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1618
1619 if (i >= kctl_tbl_size) {
1620 kctlstat.kcs_bad_kctlref++;
1621 return NULL;
1622 }
1623 kctl = kctl_table[i];
1624 if (kctl->kctlref != kctlref) {
1625 kctlstat.kcs_bad_kctlref++;
1626 return NULL;
1627 }
1628 return kctl;
1629 }
1630
1631 /*
1632 * Register/unregister a NKE
1633 */
1634 errno_t
ctl_register(struct kern_ctl_reg * userkctl,kern_ctl_ref * kctlref)1635 ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref)
1636 {
1637 struct kctl *kctl = NULL;
1638 struct kctl *kctl_next = NULL;
1639 u_int32_t id = 1;
1640 size_t name_len;
1641 int is_extended = 0;
1642 int is_setup = 0;
1643
1644 if (userkctl == NULL) { /* sanity check */
1645 return EINVAL;
1646 }
1647 if (userkctl->ctl_connect == NULL) {
1648 return EINVAL;
1649 }
1650 name_len = strlen(userkctl->ctl_name);
1651 if (name_len == 0 || name_len + 1 > MAX_KCTL_NAME) {
1652 return EINVAL;
1653 }
1654
1655 kctl = kalloc_type(struct kctl, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1656
1657 lck_mtx_lock(&ctl_mtx);
1658
1659 if (kctl_make_ref(kctl) == NULL) {
1660 lck_mtx_unlock(&ctl_mtx);
1661 kfree_type(struct kctl, kctl);
1662 return ENOMEM;
1663 }
1664
1665 /*
1666 * Kernel Control IDs
1667 *
1668 * CTL_FLAG_REG_ID_UNIT indicates the control ID and unit number are
1669 * static. If they do not exist, add them to the list in order. If the
1670 * flag is not set, we must find a new unique value. We assume the
1671 * list is in order. We find the last item in the list and add one. If
1672 * this leads to wrapping the id around, we start at the front of the
1673 * list and look for a gap.
1674 */
1675
1676 if ((userkctl->ctl_flags & CTL_FLAG_REG_ID_UNIT) == 0) {
1677 /* Must dynamically assign an unused ID */
1678
1679 /* Verify the same name isn't already registered */
1680 if (ctl_find_by_name(userkctl->ctl_name) != NULL) {
1681 kctl_delete_ref(kctl->kctlref);
1682 lck_mtx_unlock(&ctl_mtx);
1683 kfree_type(struct kctl, kctl);
1684 return EEXIST;
1685 }
1686
1687 /* Start with 1 in case the list is empty */
1688 id = 1;
1689 kctl_next = TAILQ_LAST(&ctl_head, kctl_list);
1690
1691 if (kctl_next != NULL) {
1692 /* List was not empty, add one to the last item */
1693 id = kctl_next->id + 1;
1694 kctl_next = NULL;
1695
1696 /*
1697 * If this wrapped the id number, start looking at
1698 * the front of the list for an unused id.
1699 */
1700 if (id == 0) {
1701 /* Find the next unused ID */
1702 id = 1;
1703
1704 TAILQ_FOREACH(kctl_next, &ctl_head, next) {
1705 if (kctl_next->id > id) {
1706 /* We found a gap */
1707 break;
1708 }
1709
1710 id = kctl_next->id + 1;
1711 }
1712 }
1713 }
1714
1715 userkctl->ctl_id = id;
1716 kctl->id = id;
1717 kctl->reg_unit = -1;
1718 } else {
1719 TAILQ_FOREACH(kctl_next, &ctl_head, next) {
1720 if (kctl_next->id > userkctl->ctl_id) {
1721 break;
1722 }
1723 }
1724
1725 if (ctl_find_by_id_unit(userkctl->ctl_id, userkctl->ctl_unit)) {
1726 kctl_delete_ref(kctl->kctlref);
1727 lck_mtx_unlock(&ctl_mtx);
1728 kfree_type(struct kctl, kctl);
1729 return EEXIST;
1730 }
1731 kctl->id = userkctl->ctl_id;
1732 kctl->reg_unit = userkctl->ctl_unit;
1733 }
1734
1735 is_extended = (userkctl->ctl_flags & CTL_FLAG_REG_EXTENDED);
1736 is_setup = (userkctl->ctl_flags & CTL_FLAG_REG_SETUP);
1737
1738 strlcpy(kctl->name, userkctl->ctl_name, MAX_KCTL_NAME);
1739 kctl->flags = userkctl->ctl_flags;
1740
1741 /*
1742 * Let the caller know the default send and receive sizes
1743 */
1744 if (userkctl->ctl_sendsize == 0) {
1745 kctl->sendbufsize = CTL_SENDSIZE;
1746 userkctl->ctl_sendsize = kctl->sendbufsize;
1747 } else {
1748 kctl->sendbufsize = userkctl->ctl_sendsize;
1749 }
1750 if (userkctl->ctl_recvsize == 0) {
1751 kctl->recvbufsize = CTL_RECVSIZE;
1752 userkctl->ctl_recvsize = kctl->recvbufsize;
1753 } else {
1754 kctl->recvbufsize = userkctl->ctl_recvsize;
1755 }
1756
1757 if (is_setup) {
1758 kctl->setup = userkctl->ctl_setup;
1759 }
1760 kctl->bind = userkctl->ctl_bind;
1761 kctl->connect = userkctl->ctl_connect;
1762 kctl->disconnect = userkctl->ctl_disconnect;
1763 kctl->send = userkctl->ctl_send;
1764 kctl->setopt = userkctl->ctl_setopt;
1765 kctl->getopt = userkctl->ctl_getopt;
1766 if (is_extended) {
1767 kctl->rcvd = userkctl->ctl_rcvd;
1768 kctl->send_list = userkctl->ctl_send_list;
1769 }
1770
1771 TAILQ_INIT(&kctl->kcb_head);
1772
1773 if (kctl_next) {
1774 TAILQ_INSERT_BEFORE(kctl_next, kctl, next);
1775 } else {
1776 TAILQ_INSERT_TAIL(&ctl_head, kctl, next);
1777 }
1778
1779 kctlstat.kcs_reg_count++;
1780 kctlstat.kcs_gencnt++;
1781
1782 lck_mtx_unlock(&ctl_mtx);
1783
1784 *kctlref = kctl->kctlref;
1785
1786 ctl_post_msg(KEV_CTL_REGISTERED, kctl->id);
1787 return 0;
1788 }
1789
1790 errno_t
ctl_deregister(void * kctlref)1791 ctl_deregister(void *kctlref)
1792 {
1793 struct kctl *kctl;
1794
1795 lck_mtx_lock(&ctl_mtx);
1796 if ((kctl = kctl_from_ref(kctlref)) == NULL) {
1797 kctlstat.kcs_bad_kctlref++;
1798 lck_mtx_unlock(&ctl_mtx);
1799 if (ctl_debug != 0) {
1800 printf("%s invalid kctlref %p\n",
1801 __func__, kctlref);
1802 }
1803 return EINVAL;
1804 }
1805
1806 if (!TAILQ_EMPTY(&kctl->kcb_head)) {
1807 lck_mtx_unlock(&ctl_mtx);
1808 return EBUSY;
1809 }
1810
1811 TAILQ_REMOVE(&ctl_head, kctl, next);
1812
1813 kctlstat.kcs_reg_count--;
1814 kctlstat.kcs_gencnt++;
1815
1816 kctl_delete_ref(kctl->kctlref);
1817 lck_mtx_unlock(&ctl_mtx);
1818
1819 ctl_post_msg(KEV_CTL_DEREGISTERED, kctl->id);
1820 kfree_type(struct kctl, kctl);
1821 return 0;
1822 }
1823
1824 /*
1825 * Must be called with global ctl_mtx lock taked
1826 */
1827 static struct kctl *
ctl_find_by_name(const char * name)1828 ctl_find_by_name(const char *name)
1829 {
1830 struct kctl *kctl;
1831
1832 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1833
1834 TAILQ_FOREACH(kctl, &ctl_head, next)
1835 if (strncmp(kctl->name, name, sizeof(kctl->name)) == 0) {
1836 return kctl;
1837 }
1838
1839 return NULL;
1840 }
1841
1842 u_int32_t
ctl_id_by_name(const char * name)1843 ctl_id_by_name(const char *name)
1844 {
1845 u_int32_t ctl_id = 0;
1846 struct kctl *kctl;
1847
1848 lck_mtx_lock(&ctl_mtx);
1849 kctl = ctl_find_by_name(name);
1850 if (kctl) {
1851 ctl_id = kctl->id;
1852 }
1853 lck_mtx_unlock(&ctl_mtx);
1854
1855 return ctl_id;
1856 }
1857
1858 errno_t
ctl_name_by_id(u_int32_t id,char * out_name,size_t maxsize)1859 ctl_name_by_id(u_int32_t id, char *out_name, size_t maxsize)
1860 {
1861 int found = 0;
1862 struct kctl *kctl;
1863
1864 lck_mtx_lock(&ctl_mtx);
1865 TAILQ_FOREACH(kctl, &ctl_head, next) {
1866 if (kctl->id == id) {
1867 break;
1868 }
1869 }
1870
1871 if (kctl) {
1872 if (maxsize > MAX_KCTL_NAME) {
1873 maxsize = MAX_KCTL_NAME;
1874 }
1875 strlcpy(out_name, kctl->name, maxsize);
1876 found = 1;
1877 }
1878 lck_mtx_unlock(&ctl_mtx);
1879
1880 return found ? 0 : ENOENT;
1881 }
1882
1883 /*
1884 * Must be called with global ctl_mtx lock taked
1885 *
1886 */
1887 static struct kctl *
ctl_find_by_id_unit(u_int32_t id,u_int32_t unit)1888 ctl_find_by_id_unit(u_int32_t id, u_int32_t unit)
1889 {
1890 struct kctl *kctl;
1891
1892 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1893
1894 TAILQ_FOREACH(kctl, &ctl_head, next) {
1895 if (kctl->id == id && (kctl->flags & CTL_FLAG_REG_ID_UNIT) == 0) {
1896 return kctl;
1897 } else if (kctl->id == id && kctl->reg_unit == unit) {
1898 return kctl;
1899 }
1900 }
1901 return NULL;
1902 }
1903
1904 /*
1905 * Must be called with kernel controller lock taken
1906 */
1907 static struct ctl_cb *
kcb_find(struct kctl * kctl,u_int32_t unit)1908 kcb_find(struct kctl *kctl, u_int32_t unit)
1909 {
1910 struct ctl_cb *kcb;
1911
1912 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1913
1914 TAILQ_FOREACH(kcb, &kctl->kcb_head, next)
1915 if (kcb->sac.sc_unit == unit) {
1916 return kcb;
1917 }
1918
1919 return NULL;
1920 }
1921
1922 static struct socket *
kcb_find_socket(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * kctlflags)1923 kcb_find_socket(kern_ctl_ref kctlref, u_int32_t unit, u_int32_t *kctlflags)
1924 {
1925 struct socket *so = NULL;
1926 struct ctl_cb *kcb;
1927 void *lr_saved;
1928 struct kctl *kctl;
1929 int i;
1930
1931 lr_saved = __builtin_return_address(0);
1932
1933 lck_mtx_lock(&ctl_mtx);
1934 /*
1935 * First validate the kctlref
1936 */
1937 if ((kctl = kctl_from_ref(kctlref)) == NULL) {
1938 kctlstat.kcs_bad_kctlref++;
1939 lck_mtx_unlock(&ctl_mtx);
1940 if (ctl_debug != 0) {
1941 printf("%s invalid kctlref %p\n",
1942 __func__, kctlref);
1943 }
1944 return NULL;
1945 }
1946
1947 kcb = kcb_find(kctl, unit);
1948 if (kcb == NULL || kcb->kctl != kctl || (so = kcb->so) == NULL) {
1949 lck_mtx_unlock(&ctl_mtx);
1950 return NULL;
1951 }
1952 /*
1953 * This prevents the socket from being closed
1954 */
1955 kcb->usecount++;
1956 /*
1957 * Respect lock ordering: socket before ctl_mtx
1958 */
1959 lck_mtx_unlock(&ctl_mtx);
1960
1961 socket_lock(so, 1);
1962 /*
1963 * The socket lock history is more useful if we store
1964 * the address of the caller.
1965 */
1966 i = (so->next_lock_lr + SO_LCKDBG_MAX - 1) % SO_LCKDBG_MAX;
1967 so->lock_lr[i] = lr_saved;
1968
1969 lck_mtx_lock(&ctl_mtx);
1970
1971 if ((kctl = kctl_from_ref(kctlref)) == NULL || kcb->kctl == NULL) {
1972 lck_mtx_unlock(&ctl_mtx);
1973 socket_unlock(so, 1);
1974 so = NULL;
1975 lck_mtx_lock(&ctl_mtx);
1976 } else if (kctlflags != NULL) {
1977 *kctlflags = kctl->flags;
1978 }
1979
1980 kcb->usecount--;
1981 if (kcb->usecount == 0) {
1982 wakeup((event_t)&kcb->usecount);
1983 }
1984
1985 lck_mtx_unlock(&ctl_mtx);
1986
1987 return so;
1988 }
1989
1990 static void
ctl_post_msg(u_int32_t event_code,u_int32_t id)1991 ctl_post_msg(u_int32_t event_code, u_int32_t id)
1992 {
1993 struct ctl_event_data ctl_ev_data;
1994 struct kev_msg ev_msg;
1995
1996 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_NOTOWNED);
1997
1998 bzero(&ev_msg, sizeof(struct kev_msg));
1999 ev_msg.vendor_code = KEV_VENDOR_APPLE;
2000
2001 ev_msg.kev_class = KEV_SYSTEM_CLASS;
2002 ev_msg.kev_subclass = KEV_CTL_SUBCLASS;
2003 ev_msg.event_code = event_code;
2004
2005 /* common nke subclass data */
2006 bzero(&ctl_ev_data, sizeof(ctl_ev_data));
2007 ctl_ev_data.ctl_id = id;
2008 ev_msg.dv[0].data_ptr = &ctl_ev_data;
2009 ev_msg.dv[0].data_length = sizeof(ctl_ev_data);
2010
2011 ev_msg.dv[1].data_length = 0;
2012
2013 kev_post_msg(&ev_msg);
2014 }
2015
2016 static int
ctl_lock(struct socket * so,int refcount,void * lr)2017 ctl_lock(struct socket *so, int refcount, void *lr)
2018 {
2019 void *lr_saved;
2020
2021 if (lr == NULL) {
2022 lr_saved = __builtin_return_address(0);
2023 } else {
2024 lr_saved = lr;
2025 }
2026
2027 if (so->so_pcb != NULL) {
2028 lck_mtx_lock(&((struct ctl_cb *)so->so_pcb)->mtx);
2029 } else {
2030 panic("ctl_lock: so=%p NO PCB! lr=%p lrh= %s",
2031 so, lr_saved, solockhistory_nr(so));
2032 /* NOTREACHED */
2033 }
2034
2035 if (so->so_usecount < 0) {
2036 panic("ctl_lock: so=%p so_pcb=%p lr=%p ref=%x lrh= %s",
2037 so, so->so_pcb, lr_saved, so->so_usecount,
2038 solockhistory_nr(so));
2039 /* NOTREACHED */
2040 }
2041
2042 if (refcount) {
2043 so->so_usecount++;
2044 }
2045
2046 so->lock_lr[so->next_lock_lr] = lr_saved;
2047 so->next_lock_lr = (so->next_lock_lr + 1) % SO_LCKDBG_MAX;
2048 return 0;
2049 }
2050
2051 static int
ctl_unlock(struct socket * so,int refcount,void * lr)2052 ctl_unlock(struct socket *so, int refcount, void *lr)
2053 {
2054 void *lr_saved;
2055 lck_mtx_t *mutex_held;
2056
2057 if (lr == NULL) {
2058 lr_saved = __builtin_return_address(0);
2059 } else {
2060 lr_saved = lr;
2061 }
2062
2063 #if (MORE_KCTLLOCK_DEBUG && (DEVELOPMENT || DEBUG))
2064 printf("ctl_unlock: so=%llx sopcb=%x lock=%llx ref=%u lr=%llx\n",
2065 (uint64_t)VM_KERNEL_ADDRPERM(so),
2066 (uint64_t)VM_KERNEL_ADDRPERM(so->so_pcb,
2067 (uint64_t)VM_KERNEL_ADDRPERM(&((struct ctl_cb *)so->so_pcb)->mtx),
2068 so->so_usecount, (uint64_t)VM_KERNEL_ADDRPERM(lr_saved));
2069 #endif /* (MORE_KCTLLOCK_DEBUG && (DEVELOPMENT || DEBUG)) */
2070 if (refcount) {
2071 so->so_usecount--;
2072 }
2073
2074 if (so->so_usecount < 0) {
2075 panic("ctl_unlock: so=%p usecount=%x lrh= %s",
2076 so, so->so_usecount, solockhistory_nr(so));
2077 /* NOTREACHED */
2078 }
2079 if (so->so_pcb == NULL) {
2080 panic("ctl_unlock: so=%p NO PCB usecount=%x lr=%p lrh= %s",
2081 so, so->so_usecount, (void *)lr_saved,
2082 solockhistory_nr(so));
2083 /* NOTREACHED */
2084 }
2085 mutex_held = &((struct ctl_cb *)so->so_pcb)->mtx;
2086
2087 lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
2088 so->unlock_lr[so->next_unlock_lr] = lr_saved;
2089 so->next_unlock_lr = (so->next_unlock_lr + 1) % SO_LCKDBG_MAX;
2090 lck_mtx_unlock(mutex_held);
2091
2092 if (so->so_usecount == 0) {
2093 ctl_sofreelastref(so);
2094 }
2095
2096 return 0;
2097 }
2098
2099 static lck_mtx_t *
2100 ctl_getlock(struct socket *so, int flags)
2101 {
2102 #pragma unused(flags)
2103 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
2104
2105 if (so->so_pcb) {
2106 if (so->so_usecount < 0) {
2107 panic("ctl_getlock: so=%p usecount=%x lrh= %s",
2108 so, so->so_usecount, solockhistory_nr(so));
2109 }
2110 return &kcb->mtx;
2111 } else {
2112 panic("ctl_getlock: so=%p NULL NO so_pcb %s",
2113 so, solockhistory_nr(so));
2114 return so->so_proto->pr_domain->dom_mtx;
2115 }
2116 }
2117
2118 __private_extern__ int
2119 kctl_reg_list SYSCTL_HANDLER_ARGS
2120 {
2121 #pragma unused(oidp, arg1, arg2)
2122 int error = 0;
2123 u_int64_t i, n;
2124 struct xsystmgen xsg;
2125 void *buf = NULL;
2126 struct kctl *kctl;
2127 size_t item_size = ROUNDUP64(sizeof(struct xkctl_reg));
2128
2129 buf = kalloc_data(item_size, Z_WAITOK | Z_ZERO | Z_NOFAIL);
2130
2131 lck_mtx_lock(&ctl_mtx);
2132
2133 n = kctlstat.kcs_reg_count;
2134
2135 if (req->oldptr == USER_ADDR_NULL) {
2136 req->oldidx = (size_t)(n + n / 8) * sizeof(struct xkctl_reg);
2137 goto done;
2138 }
2139 if (req->newptr != USER_ADDR_NULL) {
2140 error = EPERM;
2141 goto done;
2142 }
2143 bzero(&xsg, sizeof(xsg));
2144 xsg.xg_len = sizeof(xsg);
2145 xsg.xg_count = n;
2146 xsg.xg_gen = kctlstat.kcs_gencnt;
2147 xsg.xg_sogen = so_gencnt;
2148 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2149 if (error) {
2150 goto done;
2151 }
2152 /*
2153 * We are done if there is no pcb
2154 */
2155 if (n == 0) {
2156 goto done;
2157 }
2158
2159 for (i = 0, kctl = TAILQ_FIRST(&ctl_head);
2160 i < n && kctl != NULL;
2161 i++, kctl = TAILQ_NEXT(kctl, next)) {
2162 struct xkctl_reg *xkr = (struct xkctl_reg *)buf;
2163 struct ctl_cb *kcb;
2164 u_int32_t pcbcount = 0;
2165
2166 TAILQ_FOREACH(kcb, &kctl->kcb_head, next)
2167 pcbcount++;
2168
2169 bzero(buf, item_size);
2170
2171 xkr->xkr_len = sizeof(struct xkctl_reg);
2172 xkr->xkr_kind = XSO_KCREG;
2173 xkr->xkr_id = kctl->id;
2174 xkr->xkr_reg_unit = kctl->reg_unit;
2175 xkr->xkr_flags = kctl->flags;
2176 xkr->xkr_kctlref = (uint64_t)(kctl->kctlref);
2177 xkr->xkr_recvbufsize = kctl->recvbufsize;
2178 xkr->xkr_sendbufsize = kctl->sendbufsize;
2179 xkr->xkr_lastunit = kctl->lastunit;
2180 xkr->xkr_pcbcount = pcbcount;
2181 xkr->xkr_connect = (uint64_t)VM_KERNEL_UNSLIDE(kctl->connect);
2182 xkr->xkr_disconnect =
2183 (uint64_t)VM_KERNEL_UNSLIDE(kctl->disconnect);
2184 xkr->xkr_send = (uint64_t)VM_KERNEL_UNSLIDE(kctl->send);
2185 xkr->xkr_send_list =
2186 (uint64_t)VM_KERNEL_UNSLIDE(kctl->send_list);
2187 xkr->xkr_setopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->setopt);
2188 xkr->xkr_getopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->getopt);
2189 xkr->xkr_rcvd = (uint64_t)VM_KERNEL_UNSLIDE(kctl->rcvd);
2190 strlcpy(xkr->xkr_name, kctl->name, sizeof(xkr->xkr_name));
2191
2192 error = SYSCTL_OUT(req, buf, item_size);
2193 }
2194
2195 if (error == 0) {
2196 /*
2197 * Give the user an updated idea of our state.
2198 * If the generation differs from what we told
2199 * her before, she knows that something happened
2200 * while we were processing this request, and it
2201 * might be necessary to retry.
2202 */
2203 bzero(&xsg, sizeof(xsg));
2204 xsg.xg_len = sizeof(xsg);
2205 xsg.xg_count = n;
2206 xsg.xg_gen = kctlstat.kcs_gencnt;
2207 xsg.xg_sogen = so_gencnt;
2208 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2209 if (error) {
2210 goto done;
2211 }
2212 }
2213
2214 done:
2215 lck_mtx_unlock(&ctl_mtx);
2216
2217 kfree_data(buf, item_size);
2218
2219 return error;
2220 }
2221
2222 __private_extern__ int
2223 kctl_pcblist SYSCTL_HANDLER_ARGS
2224 {
2225 #pragma unused(oidp, arg1, arg2)
2226 int error = 0;
2227 u_int64_t n, i;
2228 struct xsystmgen xsg;
2229 void *buf = NULL;
2230 struct kctl *kctl;
2231 size_t item_size = ROUNDUP64(sizeof(struct xkctlpcb)) +
2232 ROUNDUP64(sizeof(struct xsocket_n)) +
2233 2 * ROUNDUP64(sizeof(struct xsockbuf_n)) +
2234 ROUNDUP64(sizeof(struct xsockstat_n));
2235
2236 buf = kalloc_data(item_size, Z_WAITOK | Z_ZERO | Z_NOFAIL);
2237
2238 lck_mtx_lock(&ctl_mtx);
2239
2240 n = kctlstat.kcs_pcbcount;
2241
2242 if (req->oldptr == USER_ADDR_NULL) {
2243 req->oldidx = (size_t)(n + n / 8) * item_size;
2244 goto done;
2245 }
2246 if (req->newptr != USER_ADDR_NULL) {
2247 error = EPERM;
2248 goto done;
2249 }
2250 bzero(&xsg, sizeof(xsg));
2251 xsg.xg_len = sizeof(xsg);
2252 xsg.xg_count = n;
2253 xsg.xg_gen = kctlstat.kcs_gencnt;
2254 xsg.xg_sogen = so_gencnt;
2255 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2256 if (error) {
2257 goto done;
2258 }
2259 /*
2260 * We are done if there is no pcb
2261 */
2262 if (n == 0) {
2263 goto done;
2264 }
2265
2266 for (i = 0, kctl = TAILQ_FIRST(&ctl_head);
2267 i < n && kctl != NULL;
2268 kctl = TAILQ_NEXT(kctl, next)) {
2269 struct ctl_cb *kcb;
2270
2271 for (kcb = TAILQ_FIRST(&kctl->kcb_head);
2272 i < n && kcb != NULL;
2273 i++, kcb = TAILQ_NEXT(kcb, next)) {
2274 struct xkctlpcb *xk = (struct xkctlpcb *)buf;
2275 struct xsocket_n *xso = (struct xsocket_n *)
2276 ADVANCE64(xk, sizeof(*xk));
2277 struct xsockbuf_n *xsbrcv = (struct xsockbuf_n *)
2278 ADVANCE64(xso, sizeof(*xso));
2279 struct xsockbuf_n *xsbsnd = (struct xsockbuf_n *)
2280 ADVANCE64(xsbrcv, sizeof(*xsbrcv));
2281 struct xsockstat_n *xsostats = (struct xsockstat_n *)
2282 ADVANCE64(xsbsnd, sizeof(*xsbsnd));
2283
2284 bzero(buf, item_size);
2285
2286 xk->xkp_len = sizeof(struct xkctlpcb);
2287 xk->xkp_kind = XSO_KCB;
2288 xk->xkp_unit = kcb->sac.sc_unit;
2289 xk->xkp_kctpcb = (uint64_t)VM_KERNEL_ADDRPERM(kcb);
2290 xk->xkp_kctlref = (uint64_t)VM_KERNEL_ADDRPERM(kctl);
2291 xk->xkp_kctlid = kctl->id;
2292 strlcpy(xk->xkp_kctlname, kctl->name,
2293 sizeof(xk->xkp_kctlname));
2294
2295 sotoxsocket_n(kcb->so, xso);
2296 sbtoxsockbuf_n(kcb->so ?
2297 &kcb->so->so_rcv : NULL, xsbrcv);
2298 sbtoxsockbuf_n(kcb->so ?
2299 &kcb->so->so_snd : NULL, xsbsnd);
2300 sbtoxsockstat_n(kcb->so, xsostats);
2301
2302 error = SYSCTL_OUT(req, buf, item_size);
2303 }
2304 }
2305
2306 if (error == 0) {
2307 /*
2308 * Give the user an updated idea of our state.
2309 * If the generation differs from what we told
2310 * her before, she knows that something happened
2311 * while we were processing this request, and it
2312 * might be necessary to retry.
2313 */
2314 bzero(&xsg, sizeof(xsg));
2315 xsg.xg_len = sizeof(xsg);
2316 xsg.xg_count = n;
2317 xsg.xg_gen = kctlstat.kcs_gencnt;
2318 xsg.xg_sogen = so_gencnt;
2319 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2320 if (error) {
2321 goto done;
2322 }
2323 }
2324
2325 done:
2326 lck_mtx_unlock(&ctl_mtx);
2327
2328 kfree_data(buf, item_size);
2329 return error;
2330 }
2331
2332 int
2333 kctl_getstat SYSCTL_HANDLER_ARGS
2334 {
2335 #pragma unused(oidp, arg1, arg2)
2336 int error = 0;
2337
2338 lck_mtx_lock(&ctl_mtx);
2339
2340 if (req->newptr != USER_ADDR_NULL) {
2341 error = EPERM;
2342 goto done;
2343 }
2344 if (req->oldptr == USER_ADDR_NULL) {
2345 req->oldidx = sizeof(struct kctlstat);
2346 goto done;
2347 }
2348
2349 error = SYSCTL_OUT(req, &kctlstat,
2350 MIN(sizeof(struct kctlstat), req->oldlen));
2351 done:
2352 lck_mtx_unlock(&ctl_mtx);
2353 return error;
2354 }
2355
2356 void
2357 kctl_fill_socketinfo(struct socket *so, struct socket_info *si)
2358 {
2359 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
2360 struct kern_ctl_info *kcsi =
2361 &si->soi_proto.pri_kern_ctl;
2362 struct kctl *kctl = kcb->kctl;
2363
2364 si->soi_kind = SOCKINFO_KERN_CTL;
2365
2366 if (kctl == 0) {
2367 return;
2368 }
2369
2370 kcsi->kcsi_id = kctl->id;
2371 kcsi->kcsi_reg_unit = kctl->reg_unit;
2372 kcsi->kcsi_flags = kctl->flags;
2373 kcsi->kcsi_recvbufsize = kctl->recvbufsize;
2374 kcsi->kcsi_sendbufsize = kctl->sendbufsize;
2375 kcsi->kcsi_unit = kcb->sac.sc_unit;
2376 strlcpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME);
2377 }
2378