1 /*
2 * Copyright (c) 1999-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Kernel Control domain - allows control connections to
31 * and to read/write data.
32 *
33 * Vincent Lubet, 040506
34 * Christophe Allie, 010928
35 * Justin C. Walker, 990319
36 */
37
38 #include <sys/types.h>
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/syslog.h>
42 #include <sys/socket.h>
43 #include <sys/socketvar.h>
44 #include <sys/protosw.h>
45 #include <sys/domain.h>
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/sys_domain.h>
49 #include <sys/kern_event.h>
50 #include <sys/kern_control.h>
51 #include <sys/kauth.h>
52 #include <sys/sysctl.h>
53 #include <sys/proc_info.h>
54 #include <net/if_var.h>
55
56 #include <mach/vm_types.h>
57
58 #include <kern/thread.h>
59
60 struct kctl {
61 TAILQ_ENTRY(kctl) next; /* controller chain */
62 kern_ctl_ref kctlref;
63
64 /* controller information provided when registering */
65 char name[MAX_KCTL_NAME]; /* unique identifier */
66 u_int32_t id;
67 u_int32_t reg_unit;
68
69 /* misc communication information */
70 u_int32_t flags; /* support flags */
71 u_int32_t recvbufsize; /* request more than the default buffer size */
72 u_int32_t sendbufsize; /* request more than the default buffer size */
73
74 /* Dispatch functions */
75 ctl_setup_func setup; /* Setup contact */
76 ctl_bind_func bind; /* Prepare contact */
77 ctl_connect_func connect; /* Make contact */
78 ctl_disconnect_func disconnect; /* Break contact */
79 ctl_send_func send; /* Send data to nke */
80 ctl_send_list_func send_list; /* Send list of packets */
81 ctl_setopt_func setopt; /* set kctl configuration */
82 ctl_getopt_func getopt; /* get kctl configuration */
83 ctl_rcvd_func rcvd; /* Notify nke when client reads data */
84
85 TAILQ_HEAD(, ctl_cb) kcb_head;
86 u_int32_t lastunit;
87 };
88
89 #if DEVELOPMENT || DEBUG
90 enum ctl_status {
91 KCTL_DISCONNECTED = 0,
92 KCTL_CONNECTING = 1,
93 KCTL_CONNECTED = 2
94 };
95 #endif /* DEVELOPMENT || DEBUG */
96
97 struct ctl_cb {
98 TAILQ_ENTRY(ctl_cb) next; /* controller chain */
99 lck_mtx_t mtx;
100 struct socket *so; /* controlling socket */
101 struct kctl *kctl; /* back pointer to controller */
102 void *userdata;
103 struct sockaddr_ctl sac;
104 u_int32_t usecount;
105 u_int32_t kcb_usecount;
106 u_int32_t require_clearing_count;
107 #if DEVELOPMENT || DEBUG
108 enum ctl_status status;
109 #endif /* DEVELOPMENT || DEBUG */
110 };
111
112 #ifndef ROUNDUP64
113 #define ROUNDUP64(x) P2ROUNDUP((x), sizeof (u_int64_t))
114 #endif
115
116 #ifndef ADVANCE64
117 #define ADVANCE64(p, n) (void*)((char *)(p) + ROUNDUP64(n))
118 #endif
119
120 /*
121 * Definitions and vars for we support
122 */
123
124 #define CTL_SENDSIZE (2 * 1024) /* default buffer size */
125 #define CTL_RECVSIZE (8 * 1024) /* default buffer size */
126
127 /*
128 * Definitions and vars for we support
129 */
130
131 const u_int32_t ctl_maxunit = 65536;
132 static LCK_ATTR_DECLARE(ctl_lck_attr, 0, 0);
133 static LCK_GRP_DECLARE(ctl_lck_grp, "Kernel Control Protocol");
134 static LCK_MTX_DECLARE_ATTR(ctl_mtx, &ctl_lck_grp, &ctl_lck_attr);
135
136 /* all the controllers are chained */
137 TAILQ_HEAD(kctl_list, kctl) ctl_head = TAILQ_HEAD_INITIALIZER(ctl_head);
138
139 static int ctl_attach(struct socket *, int, struct proc *);
140 static int ctl_detach(struct socket *);
141 static int ctl_sofreelastref(struct socket *so);
142 static int ctl_bind(struct socket *, struct sockaddr *, struct proc *);
143 static int ctl_connect(struct socket *, struct sockaddr *, struct proc *);
144 static int ctl_disconnect(struct socket *);
145 static int ctl_ioctl(struct socket *so, u_long cmd, caddr_t data,
146 struct ifnet *ifp, struct proc *p);
147 static int ctl_send(struct socket *, int, struct mbuf *,
148 struct sockaddr *, struct mbuf *, struct proc *);
149 static int ctl_send_list(struct socket *, struct mbuf *, u_int *, int);
150 static int ctl_ctloutput(struct socket *, struct sockopt *);
151 static int ctl_peeraddr(struct socket *so, struct sockaddr **nam);
152 static int ctl_usr_rcvd(struct socket *so, int flags);
153
154 static struct kctl *ctl_find_by_name(const char *);
155 static struct kctl *ctl_find_by_id_unit(u_int32_t id, u_int32_t unit);
156
157 static struct socket *kcb_find_socket(kern_ctl_ref kctlref, u_int32_t unit,
158 u_int32_t *);
159 static struct ctl_cb *kcb_find(struct kctl *, u_int32_t unit);
160 static void ctl_post_msg(u_int32_t event_code, u_int32_t id);
161
162 static int ctl_lock(struct socket *, int, void *);
163 static int ctl_unlock(struct socket *, int, void *);
164 static lck_mtx_t * ctl_getlock(struct socket *, int);
165
166 static struct pr_usrreqs ctl_usrreqs = {
167 .pru_attach = ctl_attach,
168 .pru_bind = ctl_bind,
169 .pru_connect = ctl_connect,
170 .pru_control = ctl_ioctl,
171 .pru_detach = ctl_detach,
172 .pru_disconnect = ctl_disconnect,
173 .pru_peeraddr = ctl_peeraddr,
174 .pru_rcvd = ctl_usr_rcvd,
175 .pru_send = ctl_send,
176 .pru_send_list = ctl_send_list,
177 .pru_sosend = sosend,
178 .pru_sosend_list = sosend_list,
179 .pru_soreceive = soreceive,
180 };
181
182 static struct protosw kctlsw[] = {
183 {
184 .pr_type = SOCK_DGRAM,
185 .pr_protocol = SYSPROTO_CONTROL,
186 .pr_flags = PR_ATOMIC | PR_CONNREQUIRED | PR_PCBLOCK | PR_WANTRCVD,
187 .pr_ctloutput = ctl_ctloutput,
188 .pr_usrreqs = &ctl_usrreqs,
189 .pr_lock = ctl_lock,
190 .pr_unlock = ctl_unlock,
191 .pr_getlock = ctl_getlock,
192 },
193 {
194 .pr_type = SOCK_STREAM,
195 .pr_protocol = SYSPROTO_CONTROL,
196 .pr_flags = PR_CONNREQUIRED | PR_PCBLOCK | PR_WANTRCVD,
197 .pr_ctloutput = ctl_ctloutput,
198 .pr_usrreqs = &ctl_usrreqs,
199 .pr_lock = ctl_lock,
200 .pr_unlock = ctl_unlock,
201 .pr_getlock = ctl_getlock,
202 }
203 };
204
205 __private_extern__ int kctl_reg_list SYSCTL_HANDLER_ARGS;
206 __private_extern__ int kctl_pcblist SYSCTL_HANDLER_ARGS;
207 __private_extern__ int kctl_getstat SYSCTL_HANDLER_ARGS;
208
209
210 SYSCTL_NODE(_net_systm, OID_AUTO, kctl,
211 CTLFLAG_RW | CTLFLAG_LOCKED, 0, "Kernel control family");
212
213 struct kctlstat kctlstat;
214 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, stats,
215 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
216 kctl_getstat, "S,kctlstat", "");
217
218 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, reg_list,
219 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
220 kctl_reg_list, "S,xkctl_reg", "");
221
222 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, pcblist,
223 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
224 kctl_pcblist, "S,xkctlpcb", "");
225
226 u_int32_t ctl_autorcvbuf_max = 256 * 1024;
227 SYSCTL_INT(_net_systm_kctl, OID_AUTO, autorcvbufmax,
228 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_autorcvbuf_max, 0, "");
229
230 u_int32_t ctl_autorcvbuf_high = 0;
231 SYSCTL_INT(_net_systm_kctl, OID_AUTO, autorcvbufhigh,
232 CTLFLAG_RD | CTLFLAG_LOCKED, &ctl_autorcvbuf_high, 0, "");
233
234 u_int32_t ctl_debug = 0;
235 SYSCTL_INT(_net_systm_kctl, OID_AUTO, debug,
236 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_debug, 0, "");
237
238 #if DEVELOPMENT || DEBUG
239 u_int32_t ctl_panic_debug = 0;
240 SYSCTL_INT(_net_systm_kctl, OID_AUTO, panicdebug,
241 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_panic_debug, 0, "");
242 #endif /* DEVELOPMENT || DEBUG */
243
244 u_int32_t ctl_send_list_mode = 1;
245 SYSCTL_INT(_net_systm_kctl, OID_AUTO, send_list_mode,
246 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_send_list_mode, 0, "");
247
248 #define KCTL_TBL_INC 16
249
250 static uintptr_t kctl_tbl_size = 0;
251 static u_int32_t kctl_tbl_growing = 0;
252 static u_int32_t kctl_tbl_growing_waiting = 0;
253 static uintptr_t kctl_tbl_count = 0;
254 static struct kctl **kctl_table = NULL;
255 static uintptr_t kctl_ref_gencnt = 0;
256
257 static void kctl_tbl_grow(void);
258 static kern_ctl_ref kctl_make_ref(struct kctl *kctl);
259 static void kctl_delete_ref(kern_ctl_ref);
260 static struct kctl *kctl_from_ref(kern_ctl_ref);
261
262 /*
263 * Install the protosw's for the Kernel Control manager.
264 */
265 __private_extern__ void
kern_control_init(struct domain * dp)266 kern_control_init(struct domain *dp)
267 {
268 struct protosw *pr;
269 int i;
270 int kctl_proto_count = (sizeof(kctlsw) / sizeof(struct protosw));
271
272 VERIFY(!(dp->dom_flags & DOM_INITIALIZED));
273 VERIFY(dp == systemdomain);
274
275 for (i = 0, pr = &kctlsw[0]; i < kctl_proto_count; i++, pr++) {
276 net_add_proto(pr, dp, 1);
277 }
278 }
279
280 static void
kcb_delete(struct ctl_cb * kcb)281 kcb_delete(struct ctl_cb *kcb)
282 {
283 if (kcb != 0) {
284 lck_mtx_destroy(&kcb->mtx, &ctl_lck_grp);
285 kfree_type(struct ctl_cb, kcb);
286 }
287 }
288
289 /*
290 * Kernel Controller user-request functions
291 * attach function must exist and succeed
292 * detach not necessary
293 * we need a pcb for the per socket mutex
294 */
295 static int
ctl_attach(struct socket * so,int proto,struct proc * p)296 ctl_attach(struct socket *so, int proto, struct proc *p)
297 {
298 #pragma unused(proto, p)
299 struct ctl_cb *kcb = 0;
300
301 kcb = kalloc_type(struct ctl_cb, Z_WAITOK | Z_ZERO | Z_NOFAIL);
302
303 lck_mtx_init(&kcb->mtx, &ctl_lck_grp, &ctl_lck_attr);
304 kcb->so = so;
305 so->so_pcb = (caddr_t)kcb;
306
307 /*
308 * For datagram, use character count for sbspace as its value
309 * may be use for packetization and we do not want to
310 * drop packets based on the sbspace hint that was just provided
311 */
312 if (so->so_proto->pr_type == SOCK_DGRAM) {
313 so->so_rcv.sb_flags |= SB_KCTL;
314 so->so_snd.sb_flags |= SB_KCTL;
315 }
316 return 0;
317 }
318
319 static int
ctl_sofreelastref(struct socket * so)320 ctl_sofreelastref(struct socket *so)
321 {
322 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
323
324 so->so_pcb = 0;
325
326 if (kcb != 0) {
327 struct kctl *kctl;
328 if ((kctl = kcb->kctl) != 0) {
329 lck_mtx_lock(&ctl_mtx);
330 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
331 kctlstat.kcs_pcbcount--;
332 kctlstat.kcs_gencnt++;
333 lck_mtx_unlock(&ctl_mtx);
334 }
335 kcb_delete(kcb);
336 }
337 sofreelastref(so, 1);
338 return 0;
339 }
340
341 /*
342 * Use this function and ctl_kcb_require_clearing to serialize
343 * critical calls into the kctl subsystem
344 */
345 static void
ctl_kcb_increment_use_count(struct ctl_cb * kcb,lck_mtx_t * mutex_held)346 ctl_kcb_increment_use_count(struct ctl_cb *kcb, lck_mtx_t *mutex_held)
347 {
348 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
349 while (kcb->require_clearing_count > 0) {
350 msleep(&kcb->require_clearing_count, mutex_held, PSOCK | PCATCH, "kcb_require_clearing", NULL);
351 }
352 kcb->kcb_usecount++;
353 }
354
355 static void
ctl_kcb_require_clearing(struct ctl_cb * kcb,lck_mtx_t * mutex_held)356 ctl_kcb_require_clearing(struct ctl_cb *kcb, lck_mtx_t *mutex_held)
357 {
358 assert(kcb->kcb_usecount != 0);
359 kcb->require_clearing_count++;
360 kcb->kcb_usecount--;
361 while (kcb->kcb_usecount > 0) { // we need to wait until no one else is running
362 msleep(&kcb->kcb_usecount, mutex_held, PSOCK | PCATCH, "kcb_usecount", NULL);
363 }
364 kcb->kcb_usecount++;
365 }
366
367 static void
ctl_kcb_done_clearing(struct ctl_cb * kcb)368 ctl_kcb_done_clearing(struct ctl_cb *kcb)
369 {
370 assert(kcb->require_clearing_count != 0);
371 kcb->require_clearing_count--;
372 wakeup((caddr_t)&kcb->require_clearing_count);
373 }
374
375 static void
ctl_kcb_decrement_use_count(struct ctl_cb * kcb)376 ctl_kcb_decrement_use_count(struct ctl_cb *kcb)
377 {
378 assert(kcb->kcb_usecount != 0);
379 kcb->kcb_usecount--;
380 wakeup((caddr_t)&kcb->kcb_usecount);
381 }
382
383 static int
ctl_detach(struct socket * so)384 ctl_detach(struct socket *so)
385 {
386 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
387
388 if (kcb == 0) {
389 return 0;
390 }
391
392 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
393 ctl_kcb_increment_use_count(kcb, mtx_held);
394 ctl_kcb_require_clearing(kcb, mtx_held);
395
396 if (kcb->kctl != NULL && kcb->kctl->bind != NULL &&
397 kcb->userdata != NULL && !(so->so_state & SS_ISCONNECTED)) {
398 // The unit was bound, but not connected
399 // Invoke the disconnected call to cleanup
400 if (kcb->kctl->disconnect != NULL) {
401 socket_unlock(so, 0);
402 (*kcb->kctl->disconnect)(kcb->kctl->kctlref,
403 kcb->sac.sc_unit, kcb->userdata);
404 socket_lock(so, 0);
405 }
406 }
407
408 soisdisconnected(so);
409 #if DEVELOPMENT || DEBUG
410 kcb->status = KCTL_DISCONNECTED;
411 #endif /* DEVELOPMENT || DEBUG */
412 so->so_flags |= SOF_PCBCLEARING;
413 ctl_kcb_done_clearing(kcb);
414 ctl_kcb_decrement_use_count(kcb);
415 return 0;
416 }
417
418 static int
ctl_setup_kctl(struct socket * so,struct sockaddr * nam,struct proc * p)419 ctl_setup_kctl(struct socket *so, struct sockaddr *nam, struct proc *p)
420 {
421 struct kctl *kctl = NULL;
422 int error = 0;
423 struct sockaddr_ctl sa;
424 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
425 struct ctl_cb *kcb_next = NULL;
426 u_int32_t recvbufsize, sendbufsize;
427
428 if (kcb == 0) {
429 panic("ctl_setup_kctl so_pcb null");
430 }
431
432 if (kcb->kctl != NULL) {
433 // Already set up, skip
434 return 0;
435 }
436
437 if (nam->sa_len != sizeof(struct sockaddr_ctl)) {
438 return EINVAL;
439 }
440
441 bcopy(nam, &sa, sizeof(struct sockaddr_ctl));
442
443 lck_mtx_lock(&ctl_mtx);
444 kctl = ctl_find_by_id_unit(sa.sc_id, sa.sc_unit);
445 if (kctl == NULL) {
446 lck_mtx_unlock(&ctl_mtx);
447 return ENOENT;
448 }
449
450 if (((kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&
451 (so->so_type != SOCK_STREAM)) ||
452 (!(kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&
453 (so->so_type != SOCK_DGRAM))) {
454 lck_mtx_unlock(&ctl_mtx);
455 return EPROTOTYPE;
456 }
457
458 if (kctl->flags & CTL_FLAG_PRIVILEGED) {
459 if (p == 0) {
460 lck_mtx_unlock(&ctl_mtx);
461 return EINVAL;
462 }
463 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
464 lck_mtx_unlock(&ctl_mtx);
465 return EPERM;
466 }
467 }
468
469 if (kctl->setup != NULL) {
470 error = (*kctl->setup)(&sa.sc_unit, &kcb->userdata);
471 if (error != 0) {
472 lck_mtx_unlock(&ctl_mtx);
473 return error;
474 }
475 } else if ((kctl->flags & CTL_FLAG_REG_ID_UNIT) || sa.sc_unit != 0) {
476 if (kcb_find(kctl, sa.sc_unit) != NULL) {
477 lck_mtx_unlock(&ctl_mtx);
478 return EBUSY;
479 }
480 } else {
481 /* Find an unused ID, assumes control IDs are in order */
482 u_int32_t unit = 1;
483
484 TAILQ_FOREACH(kcb_next, &kctl->kcb_head, next) {
485 if (kcb_next->sac.sc_unit > unit) {
486 /* Found a gap, lets fill it in */
487 break;
488 }
489 unit = kcb_next->sac.sc_unit + 1;
490 if (unit == ctl_maxunit) {
491 break;
492 }
493 }
494
495 if (unit == ctl_maxunit) {
496 lck_mtx_unlock(&ctl_mtx);
497 return EBUSY;
498 }
499
500 sa.sc_unit = unit;
501 }
502
503 bcopy(&sa, &kcb->sac, sizeof(struct sockaddr_ctl));
504 kcb->kctl = kctl;
505 if (kcb_next != NULL) {
506 TAILQ_INSERT_BEFORE(kcb_next, kcb, next);
507 } else {
508 TAILQ_INSERT_TAIL(&kctl->kcb_head, kcb, next);
509 }
510 kctlstat.kcs_pcbcount++;
511 kctlstat.kcs_gencnt++;
512 kctlstat.kcs_connections++;
513 lck_mtx_unlock(&ctl_mtx);
514
515 /*
516 * rdar://15526688: Limit the send and receive sizes to sb_max
517 * by using the same scaling as sbreserve()
518 */
519 if (kctl->sendbufsize > sb_max_adj) {
520 sendbufsize = (u_int32_t)sb_max_adj;
521 } else {
522 sendbufsize = kctl->sendbufsize;
523 }
524
525 if (kctl->recvbufsize > sb_max_adj) {
526 recvbufsize = (u_int32_t)sb_max_adj;
527 } else {
528 recvbufsize = kctl->recvbufsize;
529 }
530
531 error = soreserve(so, sendbufsize, recvbufsize);
532 if (error) {
533 if (ctl_debug) {
534 printf("%s - soreserve(%llx, %u, %u) error %d\n",
535 __func__, (uint64_t)VM_KERNEL_ADDRPERM(so),
536 sendbufsize, recvbufsize, error);
537 }
538 goto done;
539 }
540
541 done:
542 if (error) {
543 soisdisconnected(so);
544 #if DEVELOPMENT || DEBUG
545 kcb->status = KCTL_DISCONNECTED;
546 #endif /* DEVELOPMENT || DEBUG */
547 lck_mtx_lock(&ctl_mtx);
548 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
549 kcb->kctl = NULL;
550 kcb->sac.sc_unit = 0;
551 kctlstat.kcs_pcbcount--;
552 kctlstat.kcs_gencnt++;
553 kctlstat.kcs_conn_fail++;
554 lck_mtx_unlock(&ctl_mtx);
555 }
556 return error;
557 }
558
559 static int
ctl_bind(struct socket * so,struct sockaddr * nam,struct proc * p)560 ctl_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
561 {
562 int error = 0;
563 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
564
565 if (kcb == NULL) {
566 panic("ctl_bind so_pcb null");
567 }
568
569 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
570 ctl_kcb_increment_use_count(kcb, mtx_held);
571 ctl_kcb_require_clearing(kcb, mtx_held);
572
573 error = ctl_setup_kctl(so, nam, p);
574 if (error) {
575 goto out;
576 }
577
578 if (kcb->kctl == NULL) {
579 panic("ctl_bind kctl null");
580 }
581
582 if (kcb->kctl->bind == NULL) {
583 error = EINVAL;
584 goto out;
585 }
586
587 socket_unlock(so, 0);
588 error = (*kcb->kctl->bind)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata);
589 socket_lock(so, 0);
590
591 out:
592 ctl_kcb_done_clearing(kcb);
593 ctl_kcb_decrement_use_count(kcb);
594 return error;
595 }
596
597 static int
ctl_connect(struct socket * so,struct sockaddr * nam,struct proc * p)598 ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
599 {
600 int error = 0;
601 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
602
603 if (kcb == NULL) {
604 panic("ctl_connect so_pcb null");
605 }
606
607 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
608 ctl_kcb_increment_use_count(kcb, mtx_held);
609 ctl_kcb_require_clearing(kcb, mtx_held);
610
611 #if DEVELOPMENT || DEBUG
612 if (kcb->status != KCTL_DISCONNECTED && ctl_panic_debug) {
613 panic("kctl already connecting/connected");
614 }
615 kcb->status = KCTL_CONNECTING;
616 #endif /* DEVELOPMENT || DEBUG */
617
618 error = ctl_setup_kctl(so, nam, p);
619 if (error) {
620 goto out;
621 }
622
623 if (kcb->kctl == NULL) {
624 panic("ctl_connect kctl null");
625 }
626
627 soisconnecting(so);
628 socket_unlock(so, 0);
629 error = (*kcb->kctl->connect)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata);
630 socket_lock(so, 0);
631 if (error) {
632 goto end;
633 }
634 soisconnected(so);
635 #if DEVELOPMENT || DEBUG
636 kcb->status = KCTL_CONNECTED;
637 #endif /* DEVELOPMENT || DEBUG */
638
639 end:
640 if (error && kcb->kctl->disconnect) {
641 /*
642 * XXX Make sure we Don't check the return value
643 * of disconnect here.
644 * ipsec/utun_ctl_disconnect will return error when
645 * disconnect gets called after connect failure.
646 * However if we decide to check for disconnect return
647 * value here. Please make sure to revisit
648 * ipsec/utun_ctl_disconnect.
649 */
650 socket_unlock(so, 0);
651 (*kcb->kctl->disconnect)(kcb->kctl->kctlref, kcb->sac.sc_unit, kcb->userdata);
652 socket_lock(so, 0);
653 }
654 if (error) {
655 soisdisconnected(so);
656 #if DEVELOPMENT || DEBUG
657 kcb->status = KCTL_DISCONNECTED;
658 #endif /* DEVELOPMENT || DEBUG */
659 lck_mtx_lock(&ctl_mtx);
660 TAILQ_REMOVE(&kcb->kctl->kcb_head, kcb, next);
661 kcb->kctl = NULL;
662 kcb->sac.sc_unit = 0;
663 kctlstat.kcs_pcbcount--;
664 kctlstat.kcs_gencnt++;
665 kctlstat.kcs_conn_fail++;
666 lck_mtx_unlock(&ctl_mtx);
667 }
668 out:
669 ctl_kcb_done_clearing(kcb);
670 ctl_kcb_decrement_use_count(kcb);
671 return error;
672 }
673
674 static int
ctl_disconnect(struct socket * so)675 ctl_disconnect(struct socket *so)
676 {
677 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
678
679 if ((kcb = (struct ctl_cb *)so->so_pcb)) {
680 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
681 ctl_kcb_increment_use_count(kcb, mtx_held);
682 ctl_kcb_require_clearing(kcb, mtx_held);
683 struct kctl *kctl = kcb->kctl;
684
685 if (kctl && kctl->disconnect) {
686 socket_unlock(so, 0);
687 (*kctl->disconnect)(kctl->kctlref, kcb->sac.sc_unit,
688 kcb->userdata);
689 socket_lock(so, 0);
690 }
691
692 soisdisconnected(so);
693 #if DEVELOPMENT || DEBUG
694 kcb->status = KCTL_DISCONNECTED;
695 #endif /* DEVELOPMENT || DEBUG */
696
697 socket_unlock(so, 0);
698 lck_mtx_lock(&ctl_mtx);
699 kcb->kctl = 0;
700 kcb->sac.sc_unit = 0;
701 while (kcb->usecount != 0) {
702 msleep(&kcb->usecount, &ctl_mtx, 0, "kcb->usecount", 0);
703 }
704 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
705 kctlstat.kcs_pcbcount--;
706 kctlstat.kcs_gencnt++;
707 lck_mtx_unlock(&ctl_mtx);
708 socket_lock(so, 0);
709 ctl_kcb_done_clearing(kcb);
710 ctl_kcb_decrement_use_count(kcb);
711 }
712 return 0;
713 }
714
715 static int
ctl_peeraddr(struct socket * so,struct sockaddr ** nam)716 ctl_peeraddr(struct socket *so, struct sockaddr **nam)
717 {
718 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
719 struct kctl *kctl;
720 struct sockaddr_ctl sc;
721
722 if (kcb == NULL) { /* sanity check */
723 return ENOTCONN;
724 }
725
726 if ((kctl = kcb->kctl) == NULL) {
727 return EINVAL;
728 }
729
730 bzero(&sc, sizeof(struct sockaddr_ctl));
731 sc.sc_len = sizeof(struct sockaddr_ctl);
732 sc.sc_family = AF_SYSTEM;
733 sc.ss_sysaddr = AF_SYS_CONTROL;
734 sc.sc_id = kctl->id;
735 sc.sc_unit = kcb->sac.sc_unit;
736
737 *nam = dup_sockaddr((struct sockaddr *)&sc, 1);
738
739 return 0;
740 }
741
742 static void
ctl_sbrcv_trim(struct socket * so)743 ctl_sbrcv_trim(struct socket *so)
744 {
745 struct sockbuf *sb = &so->so_rcv;
746
747 if (sb->sb_hiwat > sb->sb_idealsize) {
748 u_int32_t diff;
749 int32_t trim;
750
751 /*
752 * The difference between the ideal size and the
753 * current size is the upper bound of the trimage
754 */
755 diff = sb->sb_hiwat - sb->sb_idealsize;
756 /*
757 * We cannot trim below the outstanding data
758 */
759 trim = sb->sb_hiwat - sb->sb_cc;
760
761 trim = imin(trim, (int32_t)diff);
762
763 if (trim > 0) {
764 sbreserve(sb, (sb->sb_hiwat - trim));
765
766 if (ctl_debug) {
767 printf("%s - shrunk to %d\n",
768 __func__, sb->sb_hiwat);
769 }
770 }
771 }
772 }
773
774 static int
ctl_usr_rcvd(struct socket * so,int flags)775 ctl_usr_rcvd(struct socket *so, int flags)
776 {
777 int error = 0;
778 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
779 struct kctl *kctl;
780
781 if (kcb == NULL) {
782 return ENOTCONN;
783 }
784
785 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
786 ctl_kcb_increment_use_count(kcb, mtx_held);
787
788 if ((kctl = kcb->kctl) == NULL) {
789 error = EINVAL;
790 goto out;
791 }
792
793 if (kctl->rcvd) {
794 socket_unlock(so, 0);
795 (*kctl->rcvd)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata, flags);
796 socket_lock(so, 0);
797 }
798
799 ctl_sbrcv_trim(so);
800
801 out:
802 ctl_kcb_decrement_use_count(kcb);
803 return error;
804 }
805
806 static int
ctl_send(struct socket * so,int flags,struct mbuf * m,struct sockaddr * addr,struct mbuf * control,struct proc * p)807 ctl_send(struct socket *so, int flags, struct mbuf *m,
808 struct sockaddr *addr, struct mbuf *control,
809 struct proc *p)
810 {
811 #pragma unused(addr, p)
812 int error = 0;
813 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
814 struct kctl *kctl;
815
816 if (control) {
817 m_freem(control);
818 }
819
820 if (kcb == NULL) { /* sanity check */
821 m_freem(m);
822 return ENOTCONN;
823 }
824
825 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
826 ctl_kcb_increment_use_count(kcb, mtx_held);
827
828 if (error == 0 && (kctl = kcb->kctl) == NULL) {
829 error = EINVAL;
830 }
831
832 if (error == 0 && kctl->send) {
833 so_tc_update_stats(m, so, m_get_service_class(m));
834 socket_unlock(so, 0);
835 error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata,
836 m, flags);
837 socket_lock(so, 0);
838 } else {
839 m_freem(m);
840 if (error == 0) {
841 error = ENOTSUP;
842 }
843 }
844 if (error != 0) {
845 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_send_fail);
846 }
847 ctl_kcb_decrement_use_count(kcb);
848
849 return error;
850 }
851
852 static int
ctl_send_list(struct socket * so,struct mbuf * m,u_int * pktcnt,int flags)853 ctl_send_list(struct socket *so, struct mbuf *m, u_int *pktcnt, int flags)
854 {
855 int error = 0;
856 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
857 struct kctl *kctl;
858 const bool update_tc = SOCK_DOM(so) == PF_INET || SOCK_DOM(so) == PF_INET6;
859
860 if (kcb == NULL) { /* sanity check */
861 m_freem_list(m);
862 return ENOTCONN;
863 }
864
865 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
866 ctl_kcb_increment_use_count(kcb, mtx_held);
867
868 if ((kctl = kcb->kctl) == NULL) {
869 error = EINVAL;
870 goto done;
871 }
872
873 if (kctl->send_list != NULL && ctl_send_list_mode == 0) {
874 struct mbuf *nxt;
875
876 for (nxt = m; update_tc && nxt != NULL; nxt = nxt->m_nextpkt) {
877 so_tc_update_stats(nxt, so, m_get_service_class(nxt));
878 }
879
880 socket_unlock(so, 0);
881 error = (*kctl->send_list)(kctl->kctlref, kcb->sac.sc_unit,
882 kcb->userdata, m, flags);
883 socket_lock(so, 0);
884 } else {
885 *pktcnt = 0;
886 while (m != NULL && error == 0) {
887 struct mbuf *nextpkt = m->m_nextpkt;
888
889 m->m_nextpkt = NULL;
890
891 if (update_tc) {
892 so_tc_update_stats(m, so, m_get_service_class(m));
893 }
894 socket_unlock(so, 0);
895 error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit,
896 kcb->userdata, m, flags);
897 socket_lock(so, 0);
898 m = nextpkt;
899 if (error == 0) {
900 *pktcnt += 1;
901 }
902 }
903 if (m != NULL) {
904 m_freem_list(m);
905 }
906 }
907 done:
908 if (error != 0) {
909 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_send_list_fail);
910 }
911 ctl_kcb_decrement_use_count(kcb);
912
913 return error;
914 }
915
916 static errno_t
ctl_rcvbspace(struct socket * so,size_t datasize,u_int32_t kctlflags,u_int32_t flags)917 ctl_rcvbspace(struct socket *so, size_t datasize,
918 u_int32_t kctlflags, u_int32_t flags)
919 {
920 struct sockbuf *sb = &so->so_rcv;
921 u_int32_t space = sbspace(sb);
922 errno_t error;
923
924 if ((kctlflags & CTL_FLAG_REG_CRIT) == 0) {
925 if ((u_int32_t) space >= datasize) {
926 error = 0;
927 } else {
928 error = ENOBUFS;
929 }
930 } else if ((flags & CTL_DATA_CRIT) == 0) {
931 /*
932 * Reserve 25% for critical messages
933 */
934 if (space < (sb->sb_hiwat >> 2) ||
935 space < datasize) {
936 error = ENOBUFS;
937 } else {
938 error = 0;
939 }
940 } else {
941 size_t autorcvbuf_max;
942
943 /*
944 * Allow overcommit of 25%
945 */
946 autorcvbuf_max = min(sb->sb_idealsize + (sb->sb_idealsize >> 2),
947 ctl_autorcvbuf_max);
948
949 if ((u_int32_t) space >= datasize) {
950 error = 0;
951 } else if (tcp_cansbgrow(sb) &&
952 sb->sb_hiwat < autorcvbuf_max) {
953 /*
954 * Grow with a little bit of leeway
955 */
956 size_t grow = datasize - space + _MSIZE;
957 u_int32_t cc = (u_int32_t)MIN(MIN((sb->sb_hiwat + grow), autorcvbuf_max), UINT32_MAX);
958
959 if (sbreserve(sb, cc) == 1) {
960 if (sb->sb_hiwat > ctl_autorcvbuf_high) {
961 ctl_autorcvbuf_high = sb->sb_hiwat;
962 }
963
964 /*
965 * A final check
966 */
967 if ((u_int32_t) sbspace(sb) >= datasize) {
968 error = 0;
969 } else {
970 error = ENOBUFS;
971 }
972
973 if (ctl_debug) {
974 printf("%s - grown to %d error %d\n",
975 __func__, sb->sb_hiwat, error);
976 }
977 } else {
978 error = ENOBUFS;
979 }
980 } else {
981 error = ENOBUFS;
982 }
983 }
984 return error;
985 }
986
987 errno_t
ctl_enqueuembuf(kern_ctl_ref kctlref,u_int32_t unit,struct mbuf * m,u_int32_t flags)988 ctl_enqueuembuf(kern_ctl_ref kctlref, u_int32_t unit, struct mbuf *m,
989 u_int32_t flags)
990 {
991 struct socket *so;
992 errno_t error = 0;
993 int len = m->m_pkthdr.len;
994 u_int32_t kctlflags;
995
996 so = kcb_find_socket(kctlref, unit, &kctlflags);
997 if (so == NULL) {
998 return EINVAL;
999 }
1000
1001 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
1002 error = ENOBUFS;
1003 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1004 goto bye;
1005 }
1006 if ((flags & CTL_DATA_EOR)) {
1007 m->m_flags |= M_EOR;
1008 }
1009
1010 so_recv_data_stat(so, m, 0);
1011 if (sbappend_nodrop(&so->so_rcv, m) != 0) {
1012 if ((flags & CTL_DATA_NOWAKEUP) == 0) {
1013 sorwakeup(so);
1014 }
1015 } else {
1016 error = ENOBUFS;
1017 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1018 }
1019 bye:
1020 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1021 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1022 __func__, error, len,
1023 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1024 }
1025
1026 socket_unlock(so, 1);
1027 if (error != 0) {
1028 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1029 }
1030
1031 return error;
1032 }
1033
1034 /*
1035 * Compute space occupied by mbuf like sbappendrecord
1036 */
1037 static int
m_space(struct mbuf * m)1038 m_space(struct mbuf *m)
1039 {
1040 int space = 0;
1041 struct mbuf *nxt;
1042
1043 for (nxt = m; nxt != NULL; nxt = nxt->m_next) {
1044 space += nxt->m_len;
1045 }
1046
1047 return space;
1048 }
1049
1050 errno_t
ctl_enqueuembuf_list(void * kctlref,u_int32_t unit,struct mbuf * m_list,u_int32_t flags,struct mbuf ** m_remain)1051 ctl_enqueuembuf_list(void *kctlref, u_int32_t unit, struct mbuf *m_list,
1052 u_int32_t flags, struct mbuf **m_remain)
1053 {
1054 struct socket *so = NULL;
1055 errno_t error = 0;
1056 struct mbuf *m, *nextpkt;
1057 int needwakeup = 0;
1058 int len = 0;
1059 u_int32_t kctlflags;
1060
1061 /*
1062 * Need to point the beginning of the list in case of early exit
1063 */
1064 m = m_list;
1065
1066 /*
1067 * kcb_find_socket takes the socket lock with a reference
1068 */
1069 so = kcb_find_socket(kctlref, unit, &kctlflags);
1070 if (so == NULL) {
1071 error = EINVAL;
1072 goto done;
1073 }
1074
1075 if (kctlflags & CTL_FLAG_REG_SOCK_STREAM) {
1076 error = EOPNOTSUPP;
1077 goto done;
1078 }
1079 if (flags & CTL_DATA_EOR) {
1080 error = EINVAL;
1081 goto done;
1082 }
1083
1084 for (m = m_list; m != NULL; m = nextpkt) {
1085 nextpkt = m->m_nextpkt;
1086
1087 if (m->m_pkthdr.len == 0 && ctl_debug) {
1088 printf("%s: %llx m_pkthdr.len is 0",
1089 __func__, (uint64_t)VM_KERNEL_ADDRPERM(m));
1090 }
1091
1092 /*
1093 * The mbuf is either appended or freed by sbappendrecord()
1094 * so it's not reliable from a data standpoint
1095 */
1096 len = m_space(m);
1097 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
1098 error = ENOBUFS;
1099 OSIncrementAtomic64(
1100 (SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1101 break;
1102 } else {
1103 /*
1104 * Unlink from the list, m is on its own
1105 */
1106 m->m_nextpkt = NULL;
1107 so_recv_data_stat(so, m, 0);
1108 if (sbappendrecord_nodrop(&so->so_rcv, m) != 0) {
1109 needwakeup = 1;
1110 } else {
1111 /*
1112 * We free or return the remaining
1113 * mbufs in the list
1114 */
1115 m = nextpkt;
1116 error = ENOBUFS;
1117 OSIncrementAtomic64(
1118 (SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1119 break;
1120 }
1121 }
1122 }
1123 if (needwakeup && (flags & CTL_DATA_NOWAKEUP) == 0) {
1124 sorwakeup(so);
1125 }
1126
1127 done:
1128 if (so != NULL) {
1129 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1130 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1131 __func__, error, len,
1132 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1133 }
1134
1135 socket_unlock(so, 1);
1136 }
1137 if (m_remain) {
1138 *m_remain = m;
1139
1140 if (m != NULL && socket_debug && so != NULL &&
1141 (so->so_options & SO_DEBUG)) {
1142 struct mbuf *n;
1143
1144 printf("%s m_list %llx\n", __func__,
1145 (uint64_t) VM_KERNEL_ADDRPERM(m_list));
1146 for (n = m; n != NULL; n = n->m_nextpkt) {
1147 printf(" remain %llx m_next %llx\n",
1148 (uint64_t) VM_KERNEL_ADDRPERM(n),
1149 (uint64_t) VM_KERNEL_ADDRPERM(n->m_next));
1150 }
1151 }
1152 } else {
1153 if (m != NULL) {
1154 m_freem_list(m);
1155 }
1156 }
1157 if (error != 0) {
1158 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1159 }
1160 return error;
1161 }
1162
1163 errno_t
ctl_enqueuedata(void * kctlref,u_int32_t unit,void * data,size_t len,u_int32_t flags)1164 ctl_enqueuedata(void *kctlref, u_int32_t unit, void *data, size_t len,
1165 u_int32_t flags)
1166 {
1167 struct socket *so;
1168 struct mbuf *m;
1169 errno_t error = 0;
1170 unsigned int num_needed;
1171 struct mbuf *n;
1172 size_t curlen = 0;
1173 u_int32_t kctlflags;
1174
1175 so = kcb_find_socket(kctlref, unit, &kctlflags);
1176 if (so == NULL) {
1177 return EINVAL;
1178 }
1179
1180 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
1181 error = ENOBUFS;
1182 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1183 goto bye;
1184 }
1185
1186 num_needed = 1;
1187 m = m_allocpacket_internal(&num_needed, len, NULL, M_NOWAIT, 1, 0);
1188 if (m == NULL) {
1189 kctlstat.kcs_enqdata_mb_alloc_fail++;
1190 if (ctl_debug) {
1191 printf("%s: m_allocpacket_internal(%lu) failed\n",
1192 __func__, len);
1193 }
1194 error = ENOMEM;
1195 goto bye;
1196 }
1197
1198 for (n = m; n != NULL; n = n->m_next) {
1199 size_t mlen = mbuf_maxlen(n);
1200
1201 if (mlen + curlen > len) {
1202 mlen = len - curlen;
1203 }
1204 n->m_len = (int32_t)mlen;
1205 bcopy((char *)data + curlen, n->m_data, mlen);
1206 curlen += mlen;
1207 }
1208 mbuf_pkthdr_setlen(m, curlen);
1209
1210 if ((flags & CTL_DATA_EOR)) {
1211 m->m_flags |= M_EOR;
1212 }
1213 so_recv_data_stat(so, m, 0);
1214 /*
1215 * No need to call the "nodrop" variant of sbappend
1216 * because the mbuf is local to the scope of the function
1217 */
1218 if (sbappend(&so->so_rcv, m) != 0) {
1219 if ((flags & CTL_DATA_NOWAKEUP) == 0) {
1220 sorwakeup(so);
1221 }
1222 } else {
1223 kctlstat.kcs_enqdata_sbappend_fail++;
1224 error = ENOBUFS;
1225 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1226 }
1227
1228 bye:
1229 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1230 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1231 __func__, error, (int)len,
1232 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1233 }
1234
1235 socket_unlock(so, 1);
1236 if (error != 0) {
1237 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1238 }
1239 return error;
1240 }
1241
1242 errno_t
ctl_getenqueuepacketcount(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * pcnt)1243 ctl_getenqueuepacketcount(kern_ctl_ref kctlref, u_int32_t unit, u_int32_t *pcnt)
1244 {
1245 struct socket *so;
1246 u_int32_t cnt;
1247 struct mbuf *m1;
1248
1249 if (pcnt == NULL) {
1250 return EINVAL;
1251 }
1252
1253 so = kcb_find_socket(kctlref, unit, NULL);
1254 if (so == NULL) {
1255 return EINVAL;
1256 }
1257
1258 cnt = 0;
1259 m1 = so->so_rcv.sb_mb;
1260 while (m1 != NULL) {
1261 if (m1->m_type == MT_DATA ||
1262 m1->m_type == MT_HEADER ||
1263 m1->m_type == MT_OOBDATA) {
1264 cnt += 1;
1265 }
1266 m1 = m1->m_nextpkt;
1267 }
1268 *pcnt = cnt;
1269
1270 socket_unlock(so, 1);
1271
1272 return 0;
1273 }
1274
1275 errno_t
ctl_getenqueuespace(kern_ctl_ref kctlref,u_int32_t unit,size_t * space)1276 ctl_getenqueuespace(kern_ctl_ref kctlref, u_int32_t unit, size_t *space)
1277 {
1278 struct socket *so;
1279 long avail;
1280
1281 if (space == NULL) {
1282 return EINVAL;
1283 }
1284
1285 so = kcb_find_socket(kctlref, unit, NULL);
1286 if (so == NULL) {
1287 return EINVAL;
1288 }
1289
1290 avail = sbspace(&so->so_rcv);
1291 *space = (avail < 0) ? 0 : avail;
1292 socket_unlock(so, 1);
1293
1294 return 0;
1295 }
1296
1297 errno_t
ctl_getenqueuereadable(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * difference)1298 ctl_getenqueuereadable(kern_ctl_ref kctlref, u_int32_t unit,
1299 u_int32_t *difference)
1300 {
1301 struct socket *so;
1302
1303 if (difference == NULL) {
1304 return EINVAL;
1305 }
1306
1307 so = kcb_find_socket(kctlref, unit, NULL);
1308 if (so == NULL) {
1309 return EINVAL;
1310 }
1311
1312 if (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat) {
1313 *difference = 0;
1314 } else {
1315 *difference = (so->so_rcv.sb_lowat - so->so_rcv.sb_cc);
1316 }
1317 socket_unlock(so, 1);
1318
1319 return 0;
1320 }
1321
1322 static int
ctl_ctloutput(struct socket * so,struct sockopt * sopt)1323 ctl_ctloutput(struct socket *so, struct sockopt *sopt)
1324 {
1325 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
1326 struct kctl *kctl;
1327 int error = 0;
1328 void *data = NULL;
1329 size_t data_len = 0;
1330 size_t len;
1331
1332 if (sopt->sopt_level != SYSPROTO_CONTROL) {
1333 return EINVAL;
1334 }
1335
1336 if (kcb == NULL) { /* sanity check */
1337 return ENOTCONN;
1338 }
1339
1340 if ((kctl = kcb->kctl) == NULL) {
1341 return EINVAL;
1342 }
1343
1344 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
1345 ctl_kcb_increment_use_count(kcb, mtx_held);
1346
1347 switch (sopt->sopt_dir) {
1348 case SOPT_SET:
1349 if (kctl->setopt == NULL) {
1350 error = ENOTSUP;
1351 goto out;
1352 }
1353 if (sopt->sopt_valsize != 0) {
1354 data_len = sopt->sopt_valsize;
1355 data = kalloc_data(data_len, Z_WAITOK | Z_ZERO);
1356 if (data == NULL) {
1357 data_len = 0;
1358 error = ENOMEM;
1359 goto out;
1360 }
1361 error = sooptcopyin(sopt, data,
1362 sopt->sopt_valsize, sopt->sopt_valsize);
1363 }
1364 if (error == 0) {
1365 socket_unlock(so, 0);
1366 error = (*kctl->setopt)(kctl->kctlref,
1367 kcb->sac.sc_unit, kcb->userdata, sopt->sopt_name,
1368 data, sopt->sopt_valsize);
1369 socket_lock(so, 0);
1370 }
1371
1372 kfree_data(data, data_len);
1373 break;
1374
1375 case SOPT_GET:
1376 if (kctl->getopt == NULL) {
1377 error = ENOTSUP;
1378 goto out;
1379 }
1380
1381 if (sopt->sopt_valsize && sopt->sopt_val) {
1382 data_len = sopt->sopt_valsize;
1383 data = kalloc_data(data_len, Z_WAITOK | Z_ZERO);
1384 if (data == NULL) {
1385 data_len = 0;
1386 error = ENOMEM;
1387 goto out;
1388 }
1389 /*
1390 * 4108337 - copy user data in case the
1391 * kernel control needs it
1392 */
1393 error = sooptcopyin(sopt, data,
1394 sopt->sopt_valsize, sopt->sopt_valsize);
1395 }
1396
1397 if (error == 0) {
1398 len = sopt->sopt_valsize;
1399 socket_unlock(so, 0);
1400 error = (*kctl->getopt)(kctl->kctlref, kcb->sac.sc_unit,
1401 kcb->userdata, sopt->sopt_name,
1402 data, &len);
1403 if (data != NULL && len > sopt->sopt_valsize) {
1404 panic_plain("ctl_ctloutput: ctl %s returned "
1405 "len (%lu) > sopt_valsize (%lu)\n",
1406 kcb->kctl->name, len,
1407 sopt->sopt_valsize);
1408 }
1409 socket_lock(so, 0);
1410 if (error == 0) {
1411 if (data != NULL) {
1412 error = sooptcopyout(sopt, data, len);
1413 } else {
1414 sopt->sopt_valsize = len;
1415 }
1416 }
1417 }
1418
1419 kfree_data(data, data_len);
1420 break;
1421 }
1422
1423 out:
1424 ctl_kcb_decrement_use_count(kcb);
1425 return error;
1426 }
1427
1428 static int
ctl_ioctl(struct socket * so,u_long cmd,caddr_t data,struct ifnet * ifp,struct proc * p)1429 ctl_ioctl(struct socket *so, u_long cmd, caddr_t data,
1430 struct ifnet *ifp, struct proc *p)
1431 {
1432 #pragma unused(so, ifp, p)
1433 int error = ENOTSUP;
1434
1435 switch (cmd) {
1436 /* get the number of controllers */
1437 case CTLIOCGCOUNT: {
1438 struct kctl *kctl;
1439 u_int32_t n = 0;
1440
1441 lck_mtx_lock(&ctl_mtx);
1442 TAILQ_FOREACH(kctl, &ctl_head, next)
1443 n++;
1444 lck_mtx_unlock(&ctl_mtx);
1445
1446 bcopy(&n, data, sizeof(n));
1447 error = 0;
1448 break;
1449 }
1450 case CTLIOCGINFO: {
1451 struct ctl_info ctl_info;
1452 struct kctl *kctl = 0;
1453 size_t name_len;
1454
1455 bcopy(data, &ctl_info, sizeof(ctl_info));
1456 name_len = strnlen(ctl_info.ctl_name, MAX_KCTL_NAME);
1457
1458 if (name_len == 0 || name_len + 1 > MAX_KCTL_NAME) {
1459 error = EINVAL;
1460 break;
1461 }
1462 lck_mtx_lock(&ctl_mtx);
1463 kctl = ctl_find_by_name(ctl_info.ctl_name);
1464 lck_mtx_unlock(&ctl_mtx);
1465 if (kctl == 0) {
1466 error = ENOENT;
1467 break;
1468 }
1469 ctl_info.ctl_id = kctl->id;
1470 bcopy(&ctl_info, data, sizeof(ctl_info));
1471 error = 0;
1472 break;
1473 }
1474
1475 /* add controls to get list of NKEs */
1476 }
1477
1478 return error;
1479 }
1480
1481 static void
kctl_tbl_grow(void)1482 kctl_tbl_grow(void)
1483 {
1484 struct kctl **new_table;
1485 uintptr_t new_size;
1486
1487 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1488
1489 if (kctl_tbl_growing) {
1490 /* Another thread is allocating */
1491 kctl_tbl_growing_waiting++;
1492
1493 do {
1494 (void) msleep((caddr_t) &kctl_tbl_growing, &ctl_mtx,
1495 PSOCK | PCATCH, "kctl_tbl_growing", 0);
1496 } while (kctl_tbl_growing);
1497 kctl_tbl_growing_waiting--;
1498 }
1499 /* Another thread grew the table */
1500 if (kctl_table != NULL && kctl_tbl_count < kctl_tbl_size) {
1501 return;
1502 }
1503
1504 /* Verify we have a sane size */
1505 if (kctl_tbl_size + KCTL_TBL_INC >= UINT16_MAX) {
1506 kctlstat.kcs_tbl_size_too_big++;
1507 if (ctl_debug) {
1508 printf("%s kctl_tbl_size %lu too big\n",
1509 __func__, kctl_tbl_size);
1510 }
1511 return;
1512 }
1513 kctl_tbl_growing = 1;
1514
1515 new_size = kctl_tbl_size + KCTL_TBL_INC;
1516
1517 lck_mtx_unlock(&ctl_mtx);
1518 new_table = kalloc_type(struct kctl *, new_size, Z_WAITOK | Z_ZERO);
1519 lck_mtx_lock(&ctl_mtx);
1520
1521 if (new_table != NULL) {
1522 if (kctl_table != NULL) {
1523 bcopy(kctl_table, new_table,
1524 kctl_tbl_size * sizeof(struct kctl *));
1525
1526 kfree_type(struct kctl *, kctl_tbl_size, kctl_table);
1527 }
1528 kctl_table = new_table;
1529 kctl_tbl_size = new_size;
1530 }
1531
1532 kctl_tbl_growing = 0;
1533
1534 if (kctl_tbl_growing_waiting) {
1535 wakeup(&kctl_tbl_growing);
1536 }
1537 }
1538
1539 #define KCTLREF_INDEX_MASK 0x0000FFFF
1540 #define KCTLREF_GENCNT_MASK 0xFFFF0000
1541 #define KCTLREF_GENCNT_SHIFT 16
1542
1543 static kern_ctl_ref
kctl_make_ref(struct kctl * kctl)1544 kctl_make_ref(struct kctl *kctl)
1545 {
1546 uintptr_t i;
1547
1548 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1549
1550 if (kctl_tbl_count >= kctl_tbl_size) {
1551 kctl_tbl_grow();
1552 }
1553
1554 kctl->kctlref = NULL;
1555 for (i = 0; i < kctl_tbl_size; i++) {
1556 if (kctl_table[i] == NULL) {
1557 uintptr_t ref;
1558
1559 /*
1560 * Reference is index plus one
1561 */
1562 kctl_ref_gencnt += 1;
1563
1564 /*
1565 * Add generation count as salt to reference to prevent
1566 * use after deregister
1567 */
1568 ref = ((kctl_ref_gencnt << KCTLREF_GENCNT_SHIFT) &
1569 KCTLREF_GENCNT_MASK) +
1570 ((i + 1) & KCTLREF_INDEX_MASK);
1571
1572 kctl->kctlref = (void *)(ref);
1573 kctl_table[i] = kctl;
1574 kctl_tbl_count++;
1575 break;
1576 }
1577 }
1578
1579 if (kctl->kctlref == NULL) {
1580 panic("%s no space in table", __func__);
1581 }
1582
1583 if (ctl_debug > 0) {
1584 printf("%s %p for %p\n",
1585 __func__, kctl->kctlref, kctl);
1586 }
1587
1588 return kctl->kctlref;
1589 }
1590
1591 static void
kctl_delete_ref(kern_ctl_ref kctlref)1592 kctl_delete_ref(kern_ctl_ref kctlref)
1593 {
1594 /*
1595 * Reference is index plus one
1596 */
1597 uintptr_t i = (((uintptr_t)kctlref) & KCTLREF_INDEX_MASK) - 1;
1598
1599 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1600
1601 if (i < kctl_tbl_size) {
1602 struct kctl *kctl = kctl_table[i];
1603
1604 if (kctl->kctlref == kctlref) {
1605 kctl_table[i] = NULL;
1606 kctl_tbl_count--;
1607 } else {
1608 kctlstat.kcs_bad_kctlref++;
1609 }
1610 } else {
1611 kctlstat.kcs_bad_kctlref++;
1612 }
1613 }
1614
1615 static struct kctl *
kctl_from_ref(kern_ctl_ref kctlref)1616 kctl_from_ref(kern_ctl_ref kctlref)
1617 {
1618 /*
1619 * Reference is index plus one
1620 */
1621 uintptr_t i = (((uintptr_t)kctlref) & KCTLREF_INDEX_MASK) - 1;
1622 struct kctl *kctl = NULL;
1623
1624 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1625
1626 if (i >= kctl_tbl_size) {
1627 kctlstat.kcs_bad_kctlref++;
1628 return NULL;
1629 }
1630 kctl = kctl_table[i];
1631 if (kctl->kctlref != kctlref) {
1632 kctlstat.kcs_bad_kctlref++;
1633 return NULL;
1634 }
1635 return kctl;
1636 }
1637
1638 /*
1639 * Register/unregister a NKE
1640 */
1641 errno_t
ctl_register(struct kern_ctl_reg * userkctl,kern_ctl_ref * kctlref)1642 ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref)
1643 {
1644 struct kctl *kctl = NULL;
1645 struct kctl *kctl_next = NULL;
1646 u_int32_t id = 1;
1647 size_t name_len;
1648 int is_extended = 0;
1649 int is_setup = 0;
1650
1651 if (userkctl == NULL) { /* sanity check */
1652 return EINVAL;
1653 }
1654 if (userkctl->ctl_connect == NULL) {
1655 return EINVAL;
1656 }
1657 name_len = strlen(userkctl->ctl_name);
1658 if (name_len == 0 || name_len + 1 > MAX_KCTL_NAME) {
1659 return EINVAL;
1660 }
1661
1662 kctl = kalloc_type(struct kctl, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1663
1664 lck_mtx_lock(&ctl_mtx);
1665
1666 if (kctl_make_ref(kctl) == NULL) {
1667 lck_mtx_unlock(&ctl_mtx);
1668 kfree_type(struct kctl, kctl);
1669 return ENOMEM;
1670 }
1671
1672 /*
1673 * Kernel Control IDs
1674 *
1675 * CTL_FLAG_REG_ID_UNIT indicates the control ID and unit number are
1676 * static. If they do not exist, add them to the list in order. If the
1677 * flag is not set, we must find a new unique value. We assume the
1678 * list is in order. We find the last item in the list and add one. If
1679 * this leads to wrapping the id around, we start at the front of the
1680 * list and look for a gap.
1681 */
1682
1683 if ((userkctl->ctl_flags & CTL_FLAG_REG_ID_UNIT) == 0) {
1684 /* Must dynamically assign an unused ID */
1685
1686 /* Verify the same name isn't already registered */
1687 if (ctl_find_by_name(userkctl->ctl_name) != NULL) {
1688 kctl_delete_ref(kctl->kctlref);
1689 lck_mtx_unlock(&ctl_mtx);
1690 kfree_type(struct kctl, kctl);
1691 return EEXIST;
1692 }
1693
1694 /* Start with 1 in case the list is empty */
1695 id = 1;
1696 kctl_next = TAILQ_LAST(&ctl_head, kctl_list);
1697
1698 if (kctl_next != NULL) {
1699 /* List was not empty, add one to the last item */
1700 id = kctl_next->id + 1;
1701 kctl_next = NULL;
1702
1703 /*
1704 * If this wrapped the id number, start looking at
1705 * the front of the list for an unused id.
1706 */
1707 if (id == 0) {
1708 /* Find the next unused ID */
1709 id = 1;
1710
1711 TAILQ_FOREACH(kctl_next, &ctl_head, next) {
1712 if (kctl_next->id > id) {
1713 /* We found a gap */
1714 break;
1715 }
1716
1717 id = kctl_next->id + 1;
1718 }
1719 }
1720 }
1721
1722 userkctl->ctl_id = id;
1723 kctl->id = id;
1724 kctl->reg_unit = -1;
1725 } else {
1726 TAILQ_FOREACH(kctl_next, &ctl_head, next) {
1727 if (kctl_next->id > userkctl->ctl_id) {
1728 break;
1729 }
1730 }
1731
1732 if (ctl_find_by_id_unit(userkctl->ctl_id, userkctl->ctl_unit)) {
1733 kctl_delete_ref(kctl->kctlref);
1734 lck_mtx_unlock(&ctl_mtx);
1735 kfree_type(struct kctl, kctl);
1736 return EEXIST;
1737 }
1738 kctl->id = userkctl->ctl_id;
1739 kctl->reg_unit = userkctl->ctl_unit;
1740 }
1741
1742 is_extended = (userkctl->ctl_flags & CTL_FLAG_REG_EXTENDED);
1743 is_setup = (userkctl->ctl_flags & CTL_FLAG_REG_SETUP);
1744
1745 strlcpy(kctl->name, userkctl->ctl_name, MAX_KCTL_NAME);
1746 kctl->flags = userkctl->ctl_flags;
1747
1748 /*
1749 * Let the caller know the default send and receive sizes
1750 */
1751 if (userkctl->ctl_sendsize == 0) {
1752 kctl->sendbufsize = CTL_SENDSIZE;
1753 userkctl->ctl_sendsize = kctl->sendbufsize;
1754 } else {
1755 kctl->sendbufsize = userkctl->ctl_sendsize;
1756 }
1757 if (userkctl->ctl_recvsize == 0) {
1758 kctl->recvbufsize = CTL_RECVSIZE;
1759 userkctl->ctl_recvsize = kctl->recvbufsize;
1760 } else {
1761 kctl->recvbufsize = userkctl->ctl_recvsize;
1762 }
1763
1764 if (is_setup) {
1765 kctl->setup = userkctl->ctl_setup;
1766 }
1767 kctl->bind = userkctl->ctl_bind;
1768 kctl->connect = userkctl->ctl_connect;
1769 kctl->disconnect = userkctl->ctl_disconnect;
1770 kctl->send = userkctl->ctl_send;
1771 kctl->setopt = userkctl->ctl_setopt;
1772 kctl->getopt = userkctl->ctl_getopt;
1773 if (is_extended) {
1774 kctl->rcvd = userkctl->ctl_rcvd;
1775 kctl->send_list = userkctl->ctl_send_list;
1776 }
1777
1778 TAILQ_INIT(&kctl->kcb_head);
1779
1780 if (kctl_next) {
1781 TAILQ_INSERT_BEFORE(kctl_next, kctl, next);
1782 } else {
1783 TAILQ_INSERT_TAIL(&ctl_head, kctl, next);
1784 }
1785
1786 kctlstat.kcs_reg_count++;
1787 kctlstat.kcs_gencnt++;
1788
1789 lck_mtx_unlock(&ctl_mtx);
1790
1791 *kctlref = kctl->kctlref;
1792
1793 ctl_post_msg(KEV_CTL_REGISTERED, kctl->id);
1794 return 0;
1795 }
1796
1797 errno_t
ctl_deregister(void * kctlref)1798 ctl_deregister(void *kctlref)
1799 {
1800 struct kctl *kctl;
1801
1802 lck_mtx_lock(&ctl_mtx);
1803 if ((kctl = kctl_from_ref(kctlref)) == NULL) {
1804 kctlstat.kcs_bad_kctlref++;
1805 lck_mtx_unlock(&ctl_mtx);
1806 if (ctl_debug != 0) {
1807 printf("%s invalid kctlref %p\n",
1808 __func__, kctlref);
1809 }
1810 return EINVAL;
1811 }
1812
1813 if (!TAILQ_EMPTY(&kctl->kcb_head)) {
1814 lck_mtx_unlock(&ctl_mtx);
1815 return EBUSY;
1816 }
1817
1818 TAILQ_REMOVE(&ctl_head, kctl, next);
1819
1820 kctlstat.kcs_reg_count--;
1821 kctlstat.kcs_gencnt++;
1822
1823 kctl_delete_ref(kctl->kctlref);
1824 lck_mtx_unlock(&ctl_mtx);
1825
1826 ctl_post_msg(KEV_CTL_DEREGISTERED, kctl->id);
1827 kfree_type(struct kctl, kctl);
1828 return 0;
1829 }
1830
1831 /*
1832 * Must be called with global ctl_mtx lock taked
1833 */
1834 static struct kctl *
ctl_find_by_name(const char * name)1835 ctl_find_by_name(const char *name)
1836 {
1837 struct kctl *kctl;
1838
1839 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1840
1841 TAILQ_FOREACH(kctl, &ctl_head, next)
1842 if (strncmp(kctl->name, name, sizeof(kctl->name)) == 0) {
1843 return kctl;
1844 }
1845
1846 return NULL;
1847 }
1848
1849 u_int32_t
ctl_id_by_name(const char * name)1850 ctl_id_by_name(const char *name)
1851 {
1852 u_int32_t ctl_id = 0;
1853 struct kctl *kctl;
1854
1855 lck_mtx_lock(&ctl_mtx);
1856 kctl = ctl_find_by_name(name);
1857 if (kctl) {
1858 ctl_id = kctl->id;
1859 }
1860 lck_mtx_unlock(&ctl_mtx);
1861
1862 return ctl_id;
1863 }
1864
1865 errno_t
ctl_name_by_id(u_int32_t id,char * out_name,size_t maxsize)1866 ctl_name_by_id(u_int32_t id, char *out_name, size_t maxsize)
1867 {
1868 int found = 0;
1869 struct kctl *kctl;
1870
1871 lck_mtx_lock(&ctl_mtx);
1872 TAILQ_FOREACH(kctl, &ctl_head, next) {
1873 if (kctl->id == id) {
1874 break;
1875 }
1876 }
1877
1878 if (kctl) {
1879 if (maxsize > MAX_KCTL_NAME) {
1880 maxsize = MAX_KCTL_NAME;
1881 }
1882 strlcpy(out_name, kctl->name, maxsize);
1883 found = 1;
1884 }
1885 lck_mtx_unlock(&ctl_mtx);
1886
1887 return found ? 0 : ENOENT;
1888 }
1889
1890 /*
1891 * Must be called with global ctl_mtx lock taked
1892 *
1893 */
1894 static struct kctl *
ctl_find_by_id_unit(u_int32_t id,u_int32_t unit)1895 ctl_find_by_id_unit(u_int32_t id, u_int32_t unit)
1896 {
1897 struct kctl *kctl;
1898
1899 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1900
1901 TAILQ_FOREACH(kctl, &ctl_head, next) {
1902 if (kctl->id == id && (kctl->flags & CTL_FLAG_REG_ID_UNIT) == 0) {
1903 return kctl;
1904 } else if (kctl->id == id && kctl->reg_unit == unit) {
1905 return kctl;
1906 }
1907 }
1908 return NULL;
1909 }
1910
1911 /*
1912 * Must be called with kernel controller lock taken
1913 */
1914 static struct ctl_cb *
kcb_find(struct kctl * kctl,u_int32_t unit)1915 kcb_find(struct kctl *kctl, u_int32_t unit)
1916 {
1917 struct ctl_cb *kcb;
1918
1919 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1920
1921 TAILQ_FOREACH(kcb, &kctl->kcb_head, next)
1922 if (kcb->sac.sc_unit == unit) {
1923 return kcb;
1924 }
1925
1926 return NULL;
1927 }
1928
1929 static struct socket *
kcb_find_socket(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * kctlflags)1930 kcb_find_socket(kern_ctl_ref kctlref, u_int32_t unit, u_int32_t *kctlflags)
1931 {
1932 struct socket *so = NULL;
1933 struct ctl_cb *kcb;
1934 void *lr_saved;
1935 struct kctl *kctl;
1936 int i;
1937
1938 lr_saved = __builtin_return_address(0);
1939
1940 lck_mtx_lock(&ctl_mtx);
1941 /*
1942 * First validate the kctlref
1943 */
1944 if ((kctl = kctl_from_ref(kctlref)) == NULL) {
1945 kctlstat.kcs_bad_kctlref++;
1946 lck_mtx_unlock(&ctl_mtx);
1947 if (ctl_debug != 0) {
1948 printf("%s invalid kctlref %p\n",
1949 __func__, kctlref);
1950 }
1951 return NULL;
1952 }
1953
1954 kcb = kcb_find(kctl, unit);
1955 if (kcb == NULL || kcb->kctl != kctl || (so = kcb->so) == NULL) {
1956 lck_mtx_unlock(&ctl_mtx);
1957 return NULL;
1958 }
1959 /*
1960 * This prevents the socket from being closed
1961 */
1962 kcb->usecount++;
1963 /*
1964 * Respect lock ordering: socket before ctl_mtx
1965 */
1966 lck_mtx_unlock(&ctl_mtx);
1967
1968 socket_lock(so, 1);
1969 /*
1970 * The socket lock history is more useful if we store
1971 * the address of the caller.
1972 */
1973 i = (so->next_lock_lr + SO_LCKDBG_MAX - 1) % SO_LCKDBG_MAX;
1974 so->lock_lr[i] = lr_saved;
1975
1976 lck_mtx_lock(&ctl_mtx);
1977
1978 if ((kctl = kctl_from_ref(kctlref)) == NULL || kcb->kctl == NULL) {
1979 lck_mtx_unlock(&ctl_mtx);
1980 socket_unlock(so, 1);
1981 so = NULL;
1982 lck_mtx_lock(&ctl_mtx);
1983 } else if (kctlflags != NULL) {
1984 *kctlflags = kctl->flags;
1985 }
1986
1987 kcb->usecount--;
1988 if (kcb->usecount == 0) {
1989 wakeup((event_t)&kcb->usecount);
1990 }
1991
1992 lck_mtx_unlock(&ctl_mtx);
1993
1994 return so;
1995 }
1996
1997 static void
ctl_post_msg(u_int32_t event_code,u_int32_t id)1998 ctl_post_msg(u_int32_t event_code, u_int32_t id)
1999 {
2000 struct ctl_event_data ctl_ev_data;
2001 struct kev_msg ev_msg;
2002
2003 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_NOTOWNED);
2004
2005 bzero(&ev_msg, sizeof(struct kev_msg));
2006 ev_msg.vendor_code = KEV_VENDOR_APPLE;
2007
2008 ev_msg.kev_class = KEV_SYSTEM_CLASS;
2009 ev_msg.kev_subclass = KEV_CTL_SUBCLASS;
2010 ev_msg.event_code = event_code;
2011
2012 /* common nke subclass data */
2013 bzero(&ctl_ev_data, sizeof(ctl_ev_data));
2014 ctl_ev_data.ctl_id = id;
2015 ev_msg.dv[0].data_ptr = &ctl_ev_data;
2016 ev_msg.dv[0].data_length = sizeof(ctl_ev_data);
2017
2018 ev_msg.dv[1].data_length = 0;
2019
2020 kev_post_msg(&ev_msg);
2021 }
2022
2023 static int
ctl_lock(struct socket * so,int refcount,void * lr)2024 ctl_lock(struct socket *so, int refcount, void *lr)
2025 {
2026 void *lr_saved;
2027
2028 if (lr == NULL) {
2029 lr_saved = __builtin_return_address(0);
2030 } else {
2031 lr_saved = lr;
2032 }
2033
2034 if (so->so_pcb != NULL) {
2035 lck_mtx_lock(&((struct ctl_cb *)so->so_pcb)->mtx);
2036 } else {
2037 panic("ctl_lock: so=%p NO PCB! lr=%p lrh= %s",
2038 so, lr_saved, solockhistory_nr(so));
2039 /* NOTREACHED */
2040 }
2041
2042 if (so->so_usecount < 0) {
2043 panic("ctl_lock: so=%p so_pcb=%p lr=%p ref=%x lrh= %s",
2044 so, so->so_pcb, lr_saved, so->so_usecount,
2045 solockhistory_nr(so));
2046 /* NOTREACHED */
2047 }
2048
2049 if (refcount) {
2050 so->so_usecount++;
2051 }
2052
2053 so->lock_lr[so->next_lock_lr] = lr_saved;
2054 so->next_lock_lr = (so->next_lock_lr + 1) % SO_LCKDBG_MAX;
2055 return 0;
2056 }
2057
2058 static int
ctl_unlock(struct socket * so,int refcount,void * lr)2059 ctl_unlock(struct socket *so, int refcount, void *lr)
2060 {
2061 void *lr_saved;
2062 lck_mtx_t *mutex_held;
2063
2064 if (lr == NULL) {
2065 lr_saved = __builtin_return_address(0);
2066 } else {
2067 lr_saved = lr;
2068 }
2069
2070 #if (MORE_KCTLLOCK_DEBUG && (DEVELOPMENT || DEBUG))
2071 printf("ctl_unlock: so=%llx sopcb=%x lock=%llx ref=%u lr=%llx\n",
2072 (uint64_t)VM_KERNEL_ADDRPERM(so),
2073 (uint64_t)VM_KERNEL_ADDRPERM(so->so_pcb,
2074 (uint64_t)VM_KERNEL_ADDRPERM(&((struct ctl_cb *)so->so_pcb)->mtx),
2075 so->so_usecount, (uint64_t)VM_KERNEL_ADDRPERM(lr_saved));
2076 #endif /* (MORE_KCTLLOCK_DEBUG && (DEVELOPMENT || DEBUG)) */
2077 if (refcount) {
2078 so->so_usecount--;
2079 }
2080
2081 if (so->so_usecount < 0) {
2082 panic("ctl_unlock: so=%p usecount=%x lrh= %s",
2083 so, so->so_usecount, solockhistory_nr(so));
2084 /* NOTREACHED */
2085 }
2086 if (so->so_pcb == NULL) {
2087 panic("ctl_unlock: so=%p NO PCB usecount=%x lr=%p lrh= %s",
2088 so, so->so_usecount, (void *)lr_saved,
2089 solockhistory_nr(so));
2090 /* NOTREACHED */
2091 }
2092 mutex_held = &((struct ctl_cb *)so->so_pcb)->mtx;
2093
2094 lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
2095 so->unlock_lr[so->next_unlock_lr] = lr_saved;
2096 so->next_unlock_lr = (so->next_unlock_lr + 1) % SO_LCKDBG_MAX;
2097 lck_mtx_unlock(mutex_held);
2098
2099 if (so->so_usecount == 0) {
2100 ctl_sofreelastref(so);
2101 }
2102
2103 return 0;
2104 }
2105
2106 static lck_mtx_t *
2107 ctl_getlock(struct socket *so, int flags)
2108 {
2109 #pragma unused(flags)
2110 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
2111
2112 if (so->so_pcb) {
2113 if (so->so_usecount < 0) {
2114 panic("ctl_getlock: so=%p usecount=%x lrh= %s",
2115 so, so->so_usecount, solockhistory_nr(so));
2116 }
2117 return &kcb->mtx;
2118 } else {
2119 panic("ctl_getlock: so=%p NULL NO so_pcb %s",
2120 so, solockhistory_nr(so));
2121 return so->so_proto->pr_domain->dom_mtx;
2122 }
2123 }
2124
2125 __private_extern__ int
2126 kctl_reg_list SYSCTL_HANDLER_ARGS
2127 {
2128 #pragma unused(oidp, arg1, arg2)
2129 int error = 0;
2130 u_int64_t i, n;
2131 struct xsystmgen xsg;
2132 void *buf = NULL;
2133 struct kctl *kctl;
2134 size_t item_size = ROUNDUP64(sizeof(struct xkctl_reg));
2135
2136 buf = kalloc_data(item_size, Z_WAITOK | Z_ZERO | Z_NOFAIL);
2137
2138 lck_mtx_lock(&ctl_mtx);
2139
2140 n = kctlstat.kcs_reg_count;
2141
2142 if (req->oldptr == USER_ADDR_NULL) {
2143 req->oldidx = (size_t)(n + n / 8) * sizeof(struct xkctl_reg);
2144 goto done;
2145 }
2146 if (req->newptr != USER_ADDR_NULL) {
2147 error = EPERM;
2148 goto done;
2149 }
2150 bzero(&xsg, sizeof(xsg));
2151 xsg.xg_len = sizeof(xsg);
2152 xsg.xg_count = n;
2153 xsg.xg_gen = kctlstat.kcs_gencnt;
2154 xsg.xg_sogen = so_gencnt;
2155 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2156 if (error) {
2157 goto done;
2158 }
2159 /*
2160 * We are done if there is no pcb
2161 */
2162 if (n == 0) {
2163 goto done;
2164 }
2165
2166 for (i = 0, kctl = TAILQ_FIRST(&ctl_head);
2167 i < n && kctl != NULL;
2168 i++, kctl = TAILQ_NEXT(kctl, next)) {
2169 struct xkctl_reg *xkr = (struct xkctl_reg *)buf;
2170 struct ctl_cb *kcb;
2171 u_int32_t pcbcount = 0;
2172
2173 TAILQ_FOREACH(kcb, &kctl->kcb_head, next)
2174 pcbcount++;
2175
2176 bzero(buf, item_size);
2177
2178 xkr->xkr_len = sizeof(struct xkctl_reg);
2179 xkr->xkr_kind = XSO_KCREG;
2180 xkr->xkr_id = kctl->id;
2181 xkr->xkr_reg_unit = kctl->reg_unit;
2182 xkr->xkr_flags = kctl->flags;
2183 xkr->xkr_kctlref = (uint64_t)(kctl->kctlref);
2184 xkr->xkr_recvbufsize = kctl->recvbufsize;
2185 xkr->xkr_sendbufsize = kctl->sendbufsize;
2186 xkr->xkr_lastunit = kctl->lastunit;
2187 xkr->xkr_pcbcount = pcbcount;
2188 xkr->xkr_connect = (uint64_t)VM_KERNEL_UNSLIDE(kctl->connect);
2189 xkr->xkr_disconnect =
2190 (uint64_t)VM_KERNEL_UNSLIDE(kctl->disconnect);
2191 xkr->xkr_send = (uint64_t)VM_KERNEL_UNSLIDE(kctl->send);
2192 xkr->xkr_send_list =
2193 (uint64_t)VM_KERNEL_UNSLIDE(kctl->send_list);
2194 xkr->xkr_setopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->setopt);
2195 xkr->xkr_getopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->getopt);
2196 xkr->xkr_rcvd = (uint64_t)VM_KERNEL_UNSLIDE(kctl->rcvd);
2197 strlcpy(xkr->xkr_name, kctl->name, sizeof(xkr->xkr_name));
2198
2199 error = SYSCTL_OUT(req, buf, item_size);
2200 }
2201
2202 if (error == 0) {
2203 /*
2204 * Give the user an updated idea of our state.
2205 * If the generation differs from what we told
2206 * her before, she knows that something happened
2207 * while we were processing this request, and it
2208 * might be necessary to retry.
2209 */
2210 bzero(&xsg, sizeof(xsg));
2211 xsg.xg_len = sizeof(xsg);
2212 xsg.xg_count = n;
2213 xsg.xg_gen = kctlstat.kcs_gencnt;
2214 xsg.xg_sogen = so_gencnt;
2215 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2216 if (error) {
2217 goto done;
2218 }
2219 }
2220
2221 done:
2222 lck_mtx_unlock(&ctl_mtx);
2223
2224 kfree_data(buf, item_size);
2225
2226 return error;
2227 }
2228
2229 __private_extern__ int
2230 kctl_pcblist SYSCTL_HANDLER_ARGS
2231 {
2232 #pragma unused(oidp, arg1, arg2)
2233 int error = 0;
2234 u_int64_t n, i;
2235 struct xsystmgen xsg;
2236 void *buf = NULL;
2237 struct kctl *kctl;
2238 size_t item_size = ROUNDUP64(sizeof(struct xkctlpcb)) +
2239 ROUNDUP64(sizeof(struct xsocket_n)) +
2240 2 * ROUNDUP64(sizeof(struct xsockbuf_n)) +
2241 ROUNDUP64(sizeof(struct xsockstat_n));
2242
2243 buf = kalloc_data(item_size, Z_WAITOK | Z_ZERO | Z_NOFAIL);
2244
2245 lck_mtx_lock(&ctl_mtx);
2246
2247 n = kctlstat.kcs_pcbcount;
2248
2249 if (req->oldptr == USER_ADDR_NULL) {
2250 req->oldidx = (size_t)(n + n / 8) * item_size;
2251 goto done;
2252 }
2253 if (req->newptr != USER_ADDR_NULL) {
2254 error = EPERM;
2255 goto done;
2256 }
2257 bzero(&xsg, sizeof(xsg));
2258 xsg.xg_len = sizeof(xsg);
2259 xsg.xg_count = n;
2260 xsg.xg_gen = kctlstat.kcs_gencnt;
2261 xsg.xg_sogen = so_gencnt;
2262 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2263 if (error) {
2264 goto done;
2265 }
2266 /*
2267 * We are done if there is no pcb
2268 */
2269 if (n == 0) {
2270 goto done;
2271 }
2272
2273 for (i = 0, kctl = TAILQ_FIRST(&ctl_head);
2274 i < n && kctl != NULL;
2275 kctl = TAILQ_NEXT(kctl, next)) {
2276 struct ctl_cb *kcb;
2277
2278 for (kcb = TAILQ_FIRST(&kctl->kcb_head);
2279 i < n && kcb != NULL;
2280 i++, kcb = TAILQ_NEXT(kcb, next)) {
2281 struct xkctlpcb *xk = (struct xkctlpcb *)buf;
2282 struct xsocket_n *xso = (struct xsocket_n *)
2283 ADVANCE64(xk, sizeof(*xk));
2284 struct xsockbuf_n *xsbrcv = (struct xsockbuf_n *)
2285 ADVANCE64(xso, sizeof(*xso));
2286 struct xsockbuf_n *xsbsnd = (struct xsockbuf_n *)
2287 ADVANCE64(xsbrcv, sizeof(*xsbrcv));
2288 struct xsockstat_n *xsostats = (struct xsockstat_n *)
2289 ADVANCE64(xsbsnd, sizeof(*xsbsnd));
2290
2291 bzero(buf, item_size);
2292
2293 xk->xkp_len = sizeof(struct xkctlpcb);
2294 xk->xkp_kind = XSO_KCB;
2295 xk->xkp_unit = kcb->sac.sc_unit;
2296 xk->xkp_kctpcb = (uint64_t)VM_KERNEL_ADDRPERM(kcb);
2297 xk->xkp_kctlref = (uint64_t)VM_KERNEL_ADDRPERM(kctl);
2298 xk->xkp_kctlid = kctl->id;
2299 strlcpy(xk->xkp_kctlname, kctl->name,
2300 sizeof(xk->xkp_kctlname));
2301
2302 sotoxsocket_n(kcb->so, xso);
2303 sbtoxsockbuf_n(kcb->so ?
2304 &kcb->so->so_rcv : NULL, xsbrcv);
2305 sbtoxsockbuf_n(kcb->so ?
2306 &kcb->so->so_snd : NULL, xsbsnd);
2307 sbtoxsockstat_n(kcb->so, xsostats);
2308
2309 error = SYSCTL_OUT(req, buf, item_size);
2310 }
2311 }
2312
2313 if (error == 0) {
2314 /*
2315 * Give the user an updated idea of our state.
2316 * If the generation differs from what we told
2317 * her before, she knows that something happened
2318 * while we were processing this request, and it
2319 * might be necessary to retry.
2320 */
2321 bzero(&xsg, sizeof(xsg));
2322 xsg.xg_len = sizeof(xsg);
2323 xsg.xg_count = n;
2324 xsg.xg_gen = kctlstat.kcs_gencnt;
2325 xsg.xg_sogen = so_gencnt;
2326 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2327 if (error) {
2328 goto done;
2329 }
2330 }
2331
2332 done:
2333 lck_mtx_unlock(&ctl_mtx);
2334
2335 kfree_data(buf, item_size);
2336 return error;
2337 }
2338
2339 int
2340 kctl_getstat SYSCTL_HANDLER_ARGS
2341 {
2342 #pragma unused(oidp, arg1, arg2)
2343 int error = 0;
2344
2345 lck_mtx_lock(&ctl_mtx);
2346
2347 if (req->newptr != USER_ADDR_NULL) {
2348 error = EPERM;
2349 goto done;
2350 }
2351 if (req->oldptr == USER_ADDR_NULL) {
2352 req->oldidx = sizeof(struct kctlstat);
2353 goto done;
2354 }
2355
2356 error = SYSCTL_OUT(req, &kctlstat,
2357 MIN(sizeof(struct kctlstat), req->oldlen));
2358 done:
2359 lck_mtx_unlock(&ctl_mtx);
2360 return error;
2361 }
2362
2363 void
2364 kctl_fill_socketinfo(struct socket *so, struct socket_info *si)
2365 {
2366 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
2367 struct kern_ctl_info *kcsi =
2368 &si->soi_proto.pri_kern_ctl;
2369 struct kctl *kctl = kcb->kctl;
2370
2371 si->soi_kind = SOCKINFO_KERN_CTL;
2372
2373 if (kctl == 0) {
2374 return;
2375 }
2376
2377 kcsi->kcsi_id = kctl->id;
2378 kcsi->kcsi_reg_unit = kctl->reg_unit;
2379 kcsi->kcsi_flags = kctl->flags;
2380 kcsi->kcsi_recvbufsize = kctl->recvbufsize;
2381 kcsi->kcsi_sendbufsize = kctl->sendbufsize;
2382 kcsi->kcsi_unit = kcb->sac.sc_unit;
2383 strlcpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME);
2384 }
2385