1 /*
2 * Copyright (c) 1999-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Kernel Control domain - allows control connections to
31 * and to read/write data.
32 *
33 * Vincent Lubet, 040506
34 * Christophe Allie, 010928
35 * Justin C. Walker, 990319
36 */
37
38 #include <sys/types.h>
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/syslog.h>
42 #include <sys/socket.h>
43 #include <sys/socketvar.h>
44 #include <sys/protosw.h>
45 #include <sys/domain.h>
46 #include <sys/malloc.h>
47 #include <sys/mbuf.h>
48 #include <sys/sys_domain.h>
49 #include <sys/kern_event.h>
50 #include <sys/kern_control.h>
51 #include <sys/kauth.h>
52 #include <sys/sysctl.h>
53 #include <sys/proc_info.h>
54 #include <net/if_var.h>
55
56 #include <mach/vm_types.h>
57
58 #include <kern/thread.h>
59
60 struct kctl {
61 TAILQ_ENTRY(kctl) next; /* controller chain */
62 kern_ctl_ref kctlref;
63
64 /* controller information provided when registering */
65 char name[MAX_KCTL_NAME]; /* unique identifier */
66 u_int32_t id;
67 u_int32_t reg_unit;
68
69 /* misc communication information */
70 u_int32_t flags; /* support flags */
71 u_int32_t recvbufsize; /* request more than the default buffer size */
72 u_int32_t sendbufsize; /* request more than the default buffer size */
73
74 /* Dispatch functions */
75 ctl_setup_func setup; /* Setup contact */
76 ctl_bind_func bind; /* Prepare contact */
77 ctl_connect_func connect; /* Make contact */
78 ctl_disconnect_func disconnect; /* Break contact */
79 ctl_send_func send; /* Send data to nke */
80 ctl_send_list_func send_list; /* Send list of packets */
81 ctl_setopt_func setopt; /* set kctl configuration */
82 ctl_getopt_func getopt; /* get kctl configuration */
83 ctl_rcvd_func rcvd; /* Notify nke when client reads data */
84
85 TAILQ_HEAD(, ctl_cb) kcb_head;
86 u_int32_t lastunit;
87 };
88
89 #if DEVELOPMENT || DEBUG
90 enum ctl_status {
91 KCTL_DISCONNECTED = 0,
92 KCTL_CONNECTING = 1,
93 KCTL_CONNECTED = 2
94 };
95 #endif /* DEVELOPMENT || DEBUG */
96
97 struct ctl_cb {
98 TAILQ_ENTRY(ctl_cb) next; /* controller chain */
99 lck_mtx_t mtx;
100 struct socket *so; /* controlling socket */
101 struct kctl *kctl; /* back pointer to controller */
102 void *userdata;
103 struct sockaddr_ctl sac;
104 uint32_t usecount;
105 uint32_t kcb_usecount;
106 uint32_t require_clearing_count;
107 #if DEVELOPMENT || DEBUG
108 enum ctl_status status;
109 #endif /* DEVELOPMENT || DEBUG */
110 };
111
112 #ifndef ROUNDUP64
113 #define ROUNDUP64(x) P2ROUNDUP((x), sizeof (u_int64_t))
114 #endif
115
116 #ifndef ADVANCE64
117 #define ADVANCE64(p, n) (void*)((char *)(p) + ROUNDUP64(n))
118 #endif
119
120 /*
121 * Definitions and vars for we support
122 */
123
124 #define CTL_SENDSIZE (2 * 1024) /* default buffer size */
125 #define CTL_RECVSIZE (8 * 1024) /* default buffer size */
126
127 /*
128 * Definitions and vars for we support
129 */
130
131 const u_int32_t ctl_maxunit = 65536;
132 static LCK_ATTR_DECLARE(ctl_lck_attr, 0, 0);
133 static LCK_GRP_DECLARE(ctl_lck_grp, "Kernel Control Protocol");
134 static LCK_MTX_DECLARE_ATTR(ctl_mtx, &ctl_lck_grp, &ctl_lck_attr);
135
136 /* all the controllers are chained */
137 TAILQ_HEAD(kctl_list, kctl) ctl_head = TAILQ_HEAD_INITIALIZER(ctl_head);
138
139 static int ctl_attach(struct socket *, int, struct proc *);
140 static int ctl_detach(struct socket *);
141 static int ctl_sofreelastref(struct socket *so);
142 static int ctl_bind(struct socket *, struct sockaddr *, struct proc *);
143 static int ctl_connect(struct socket *, struct sockaddr *, struct proc *);
144 static int ctl_disconnect(struct socket *);
145 static int ctl_ioctl(struct socket *so, u_long cmd, caddr_t data,
146 struct ifnet *ifp, struct proc *p);
147 static int ctl_send(struct socket *, int, struct mbuf *,
148 struct sockaddr *, struct mbuf *, struct proc *);
149 static int ctl_send_list(struct socket *, struct mbuf *, u_int *, int);
150 static int ctl_ctloutput(struct socket *, struct sockopt *);
151 static int ctl_peeraddr(struct socket *so, struct sockaddr **nam);
152 static int ctl_usr_rcvd(struct socket *so, int flags);
153
154 static struct kctl *ctl_find_by_name(const char *);
155 static struct kctl *ctl_find_by_id_unit(u_int32_t id, u_int32_t unit);
156
157 static struct socket *kcb_find_socket(kern_ctl_ref kctlref, u_int32_t unit,
158 u_int32_t *);
159 static struct ctl_cb *kcb_find(struct kctl *, u_int32_t unit);
160 static void ctl_post_msg(u_int32_t event_code, u_int32_t id);
161
162 static int ctl_lock(struct socket *, int, void *);
163 static int ctl_unlock(struct socket *, int, void *);
164 static lck_mtx_t * ctl_getlock(struct socket *, int);
165
166 static struct pr_usrreqs ctl_usrreqs = {
167 .pru_attach = ctl_attach,
168 .pru_bind = ctl_bind,
169 .pru_connect = ctl_connect,
170 .pru_control = ctl_ioctl,
171 .pru_detach = ctl_detach,
172 .pru_disconnect = ctl_disconnect,
173 .pru_peeraddr = ctl_peeraddr,
174 .pru_rcvd = ctl_usr_rcvd,
175 .pru_send = ctl_send,
176 .pru_send_list = ctl_send_list,
177 .pru_sosend = sosend,
178 .pru_sosend_list = sosend_list,
179 .pru_soreceive = soreceive,
180 };
181
182 static struct protosw kctlsw[] = {
183 {
184 .pr_type = SOCK_DGRAM,
185 .pr_protocol = SYSPROTO_CONTROL,
186 .pr_flags = PR_ATOMIC | PR_CONNREQUIRED | PR_PCBLOCK | PR_WANTRCVD,
187 .pr_ctloutput = ctl_ctloutput,
188 .pr_usrreqs = &ctl_usrreqs,
189 .pr_lock = ctl_lock,
190 .pr_unlock = ctl_unlock,
191 .pr_getlock = ctl_getlock,
192 },
193 {
194 .pr_type = SOCK_STREAM,
195 .pr_protocol = SYSPROTO_CONTROL,
196 .pr_flags = PR_CONNREQUIRED | PR_PCBLOCK | PR_WANTRCVD,
197 .pr_ctloutput = ctl_ctloutput,
198 .pr_usrreqs = &ctl_usrreqs,
199 .pr_lock = ctl_lock,
200 .pr_unlock = ctl_unlock,
201 .pr_getlock = ctl_getlock,
202 }
203 };
204
205 __private_extern__ int kctl_reg_list SYSCTL_HANDLER_ARGS;
206 __private_extern__ int kctl_pcblist SYSCTL_HANDLER_ARGS;
207 __private_extern__ int kctl_getstat SYSCTL_HANDLER_ARGS;
208
209
210 SYSCTL_NODE(_net_systm, OID_AUTO, kctl,
211 CTLFLAG_RW | CTLFLAG_LOCKED, 0, "Kernel control family");
212
213 struct kctlstat kctlstat;
214 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, stats,
215 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
216 kctl_getstat, "S,kctlstat", "");
217
218 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, reg_list,
219 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
220 kctl_reg_list, "S,xkctl_reg", "");
221
222 SYSCTL_PROC(_net_systm_kctl, OID_AUTO, pcblist,
223 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0,
224 kctl_pcblist, "S,xkctlpcb", "");
225
226 u_int32_t ctl_autorcvbuf_max = 256 * 1024;
227 SYSCTL_INT(_net_systm_kctl, OID_AUTO, autorcvbufmax,
228 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_autorcvbuf_max, 0, "");
229
230 u_int32_t ctl_autorcvbuf_high = 0;
231 SYSCTL_INT(_net_systm_kctl, OID_AUTO, autorcvbufhigh,
232 CTLFLAG_RD | CTLFLAG_LOCKED, &ctl_autorcvbuf_high, 0, "");
233
234 u_int32_t ctl_debug = 0;
235 SYSCTL_INT(_net_systm_kctl, OID_AUTO, debug,
236 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_debug, 0, "");
237
238 #if DEVELOPMENT || DEBUG
239 u_int32_t ctl_panic_debug = 0;
240 SYSCTL_INT(_net_systm_kctl, OID_AUTO, panicdebug,
241 CTLFLAG_RW | CTLFLAG_LOCKED, &ctl_panic_debug, 0, "");
242 #endif /* DEVELOPMENT || DEBUG */
243
244 #define KCTL_TBL_INC 16
245
246 static uintptr_t kctl_tbl_size = 0;
247 static u_int32_t kctl_tbl_growing = 0;
248 static u_int32_t kctl_tbl_growing_waiting = 0;
249 static uintptr_t kctl_tbl_count = 0;
250 static struct kctl **kctl_table = NULL;
251 static uintptr_t kctl_ref_gencnt = 0;
252
253 static void kctl_tbl_grow(void);
254 static kern_ctl_ref kctl_make_ref(struct kctl *kctl);
255 static void kctl_delete_ref(kern_ctl_ref);
256 static struct kctl *kctl_from_ref(kern_ctl_ref);
257
258 /*
259 * Install the protosw's for the Kernel Control manager.
260 */
261 __private_extern__ void
kern_control_init(struct domain * dp)262 kern_control_init(struct domain *dp)
263 {
264 struct protosw *pr;
265 int i;
266 int kctl_proto_count = (sizeof(kctlsw) / sizeof(struct protosw));
267
268 VERIFY(!(dp->dom_flags & DOM_INITIALIZED));
269 VERIFY(dp == systemdomain);
270
271 for (i = 0, pr = &kctlsw[0]; i < kctl_proto_count; i++, pr++) {
272 net_add_proto(pr, dp, 1);
273 }
274 }
275
276 static void
kcb_delete(struct ctl_cb * kcb)277 kcb_delete(struct ctl_cb *kcb)
278 {
279 if (kcb != 0) {
280 lck_mtx_destroy(&kcb->mtx, &ctl_lck_grp);
281 kfree_type(struct ctl_cb, kcb);
282 }
283 }
284
285 /*
286 * Kernel Controller user-request functions
287 * attach function must exist and succeed
288 * detach not necessary
289 * we need a pcb for the per socket mutex
290 */
291 static int
ctl_attach(struct socket * so,int proto,struct proc * p)292 ctl_attach(struct socket *so, int proto, struct proc *p)
293 {
294 #pragma unused(proto, p)
295 struct ctl_cb *kcb = 0;
296
297 kcb = kalloc_type(struct ctl_cb, Z_WAITOK | Z_ZERO | Z_NOFAIL);
298
299 lck_mtx_init(&kcb->mtx, &ctl_lck_grp, &ctl_lck_attr);
300 kcb->so = so;
301 so->so_pcb = (caddr_t)kcb;
302
303 /*
304 * For datagram, use character count for sbspace as its value
305 * may be use for packetization and we do not want to
306 * drop packets based on the sbspace hint that was just provided
307 */
308 if (SOCK_CHECK_TYPE(so, SOCK_DGRAM)) {
309 so->so_rcv.sb_flags |= SB_KCTL;
310 so->so_snd.sb_flags |= SB_KCTL;
311 }
312 return 0;
313 }
314
315 static int
ctl_sofreelastref(struct socket * so)316 ctl_sofreelastref(struct socket *so)
317 {
318 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
319
320 so->so_pcb = 0;
321
322 if (kcb != 0) {
323 struct kctl *kctl;
324 if ((kctl = kcb->kctl) != 0) {
325 lck_mtx_lock(&ctl_mtx);
326 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
327 kctlstat.kcs_pcbcount--;
328 kctlstat.kcs_gencnt++;
329 lck_mtx_unlock(&ctl_mtx);
330 }
331 kcb_delete(kcb);
332 }
333 sofreelastref(so, 1);
334 return 0;
335 }
336
337 /*
338 * Use this function and ctl_kcb_require_clearing to serialize
339 * critical calls into the kctl subsystem
340 */
341 static void
ctl_kcb_increment_use_count(struct ctl_cb * kcb,lck_mtx_t * mutex_held)342 ctl_kcb_increment_use_count(struct ctl_cb *kcb, lck_mtx_t *mutex_held)
343 {
344 LCK_MTX_ASSERT(mutex_held, LCK_MTX_ASSERT_OWNED);
345 while (kcb->require_clearing_count > 0) {
346 msleep(&kcb->require_clearing_count, mutex_held, PSOCK | PCATCH, "kcb_require_clearing", NULL);
347 }
348 kcb->kcb_usecount++;
349 }
350
351 static void
ctl_kcb_require_clearing(struct ctl_cb * kcb,lck_mtx_t * mutex_held)352 ctl_kcb_require_clearing(struct ctl_cb *kcb, lck_mtx_t *mutex_held)
353 {
354 assert(kcb->kcb_usecount != 0);
355 kcb->require_clearing_count++;
356 kcb->kcb_usecount--;
357 while (kcb->kcb_usecount > 0) { // we need to wait until no one else is running
358 msleep(&kcb->kcb_usecount, mutex_held, PSOCK | PCATCH, "kcb_usecount", NULL);
359 }
360 kcb->kcb_usecount++;
361 }
362
363 static void
ctl_kcb_done_clearing(struct ctl_cb * kcb)364 ctl_kcb_done_clearing(struct ctl_cb *kcb)
365 {
366 assert(kcb->require_clearing_count != 0);
367 kcb->require_clearing_count--;
368 wakeup((caddr_t)&kcb->require_clearing_count);
369 }
370
371 static void
ctl_kcb_decrement_use_count(struct ctl_cb * kcb)372 ctl_kcb_decrement_use_count(struct ctl_cb *kcb)
373 {
374 assert(kcb->kcb_usecount != 0);
375 kcb->kcb_usecount--;
376 if (kcb->require_clearing_count != 0) {
377 wakeup((caddr_t)&kcb->kcb_usecount);
378 }
379 }
380
381 static int
ctl_detach(struct socket * so)382 ctl_detach(struct socket *so)
383 {
384 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
385
386 if (kcb == 0) {
387 return 0;
388 }
389
390 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
391 ctl_kcb_increment_use_count(kcb, mtx_held);
392 ctl_kcb_require_clearing(kcb, mtx_held);
393
394 if (kcb->kctl != NULL && kcb->kctl->bind != NULL &&
395 kcb->userdata != NULL && !(so->so_state & SS_ISCONNECTED)) {
396 // The unit was bound, but not connected
397 // Invoke the disconnected call to cleanup
398 if (kcb->kctl->disconnect != NULL) {
399 socket_unlock(so, 0);
400 (*kcb->kctl->disconnect)(kcb->kctl->kctlref,
401 kcb->sac.sc_unit, kcb->userdata);
402 socket_lock(so, 0);
403 }
404 }
405
406 soisdisconnected(so);
407 #if DEVELOPMENT || DEBUG
408 kcb->status = KCTL_DISCONNECTED;
409 #endif /* DEVELOPMENT || DEBUG */
410 so->so_flags |= SOF_PCBCLEARING;
411 ctl_kcb_done_clearing(kcb);
412 ctl_kcb_decrement_use_count(kcb);
413 return 0;
414 }
415
416 static int
ctl_setup_kctl(struct socket * so,struct sockaddr * nam,struct proc * p)417 ctl_setup_kctl(struct socket *so, struct sockaddr *nam, struct proc *p)
418 {
419 struct kctl *kctl = NULL;
420 int error = 0;
421 struct sockaddr_ctl sa;
422 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
423 struct ctl_cb *kcb_next = NULL;
424
425 if (kcb == 0) {
426 panic("ctl_setup_kctl so_pcb null");
427 }
428
429 if (kcb->kctl != NULL) {
430 // Already set up, skip
431 return 0;
432 }
433
434 if (nam->sa_len != sizeof(struct sockaddr_ctl)) {
435 return EINVAL;
436 }
437
438 bcopy(nam, &sa, sizeof(struct sockaddr_ctl));
439
440 lck_mtx_lock(&ctl_mtx);
441 kctl = ctl_find_by_id_unit(sa.sc_id, sa.sc_unit);
442 if (kctl == NULL) {
443 lck_mtx_unlock(&ctl_mtx);
444 return ENOENT;
445 }
446
447 if (((kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&
448 (so->so_type != SOCK_STREAM)) ||
449 (!(kctl->flags & CTL_FLAG_REG_SOCK_STREAM) &&
450 (so->so_type != SOCK_DGRAM))) {
451 lck_mtx_unlock(&ctl_mtx);
452 return EPROTOTYPE;
453 }
454
455 if (kctl->flags & CTL_FLAG_PRIVILEGED) {
456 if (p == 0) {
457 lck_mtx_unlock(&ctl_mtx);
458 return EINVAL;
459 }
460 if (kauth_cred_issuser(kauth_cred_get()) == 0) {
461 lck_mtx_unlock(&ctl_mtx);
462 return EPERM;
463 }
464 }
465
466 if (kctl->setup != NULL) {
467 error = (*kctl->setup)(&sa.sc_unit, &kcb->userdata);
468 if (error != 0) {
469 lck_mtx_unlock(&ctl_mtx);
470 return error;
471 }
472 } else if ((kctl->flags & CTL_FLAG_REG_ID_UNIT) || sa.sc_unit != 0) {
473 if (kcb_find(kctl, sa.sc_unit) != NULL) {
474 lck_mtx_unlock(&ctl_mtx);
475 return EBUSY;
476 }
477 } else {
478 /* Find an unused ID, assumes control IDs are in order */
479 u_int32_t unit = 1;
480
481 TAILQ_FOREACH(kcb_next, &kctl->kcb_head, next) {
482 if (kcb_next->sac.sc_unit > unit) {
483 /* Found a gap, lets fill it in */
484 break;
485 }
486 unit = kcb_next->sac.sc_unit + 1;
487 if (unit == ctl_maxunit) {
488 break;
489 }
490 }
491
492 if (unit == ctl_maxunit) {
493 lck_mtx_unlock(&ctl_mtx);
494 return EBUSY;
495 }
496
497 sa.sc_unit = unit;
498 }
499
500 bcopy(&sa, &kcb->sac, sizeof(struct sockaddr_ctl));
501 kcb->kctl = kctl;
502 if (kcb_next != NULL) {
503 TAILQ_INSERT_BEFORE(kcb_next, kcb, next);
504 } else {
505 TAILQ_INSERT_TAIL(&kctl->kcb_head, kcb, next);
506 }
507 kctlstat.kcs_pcbcount++;
508 kctlstat.kcs_gencnt++;
509 kctlstat.kcs_connections++;
510 lck_mtx_unlock(&ctl_mtx);
511
512 error = soreserve(so, kctl->sendbufsize, kctl->recvbufsize);
513 if (error) {
514 if (ctl_debug) {
515 printf("%s - soreserve(%llx, %u, %u) error %d\n",
516 __func__, (uint64_t)VM_KERNEL_ADDRPERM(so),
517 kctl->sendbufsize, kctl->recvbufsize, error);
518 }
519 goto done;
520 }
521
522 done:
523 if (error) {
524 soisdisconnected(so);
525 #if DEVELOPMENT || DEBUG
526 kcb->status = KCTL_DISCONNECTED;
527 #endif /* DEVELOPMENT || DEBUG */
528 lck_mtx_lock(&ctl_mtx);
529 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
530 kcb->kctl = NULL;
531 kcb->sac.sc_unit = 0;
532 kctlstat.kcs_pcbcount--;
533 kctlstat.kcs_gencnt++;
534 kctlstat.kcs_conn_fail++;
535 lck_mtx_unlock(&ctl_mtx);
536 }
537 return error;
538 }
539
540 static int
ctl_bind(struct socket * so,struct sockaddr * nam,struct proc * p)541 ctl_bind(struct socket *so, struct sockaddr *nam, struct proc *p)
542 {
543 int error = 0;
544 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
545
546 if (kcb == NULL) {
547 panic("ctl_bind so_pcb null");
548 }
549
550 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
551 ctl_kcb_increment_use_count(kcb, mtx_held);
552 ctl_kcb_require_clearing(kcb, mtx_held);
553
554 error = ctl_setup_kctl(so, nam, p);
555 if (error) {
556 goto out;
557 }
558
559 if (kcb->kctl == NULL) {
560 panic("ctl_bind kctl null");
561 }
562
563 if (kcb->kctl->bind == NULL) {
564 error = EINVAL;
565 goto out;
566 }
567
568 socket_unlock(so, 0);
569 error = (*kcb->kctl->bind)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata);
570 socket_lock(so, 0);
571
572 out:
573 ctl_kcb_done_clearing(kcb);
574 ctl_kcb_decrement_use_count(kcb);
575 return error;
576 }
577
578 static int
ctl_connect(struct socket * so,struct sockaddr * nam,struct proc * p)579 ctl_connect(struct socket *so, struct sockaddr *nam, struct proc *p)
580 {
581 int error = 0;
582 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
583
584 if (kcb == NULL) {
585 panic("ctl_connect so_pcb null");
586 }
587
588 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
589 ctl_kcb_increment_use_count(kcb, mtx_held);
590 ctl_kcb_require_clearing(kcb, mtx_held);
591
592 #if DEVELOPMENT || DEBUG
593 if (kcb->status != KCTL_DISCONNECTED && ctl_panic_debug) {
594 panic("kctl already connecting/connected");
595 }
596 kcb->status = KCTL_CONNECTING;
597 #endif /* DEVELOPMENT || DEBUG */
598
599 error = ctl_setup_kctl(so, nam, p);
600 if (error) {
601 goto out;
602 }
603
604 if (kcb->kctl == NULL) {
605 panic("ctl_connect kctl null");
606 }
607
608 soisconnecting(so);
609 socket_unlock(so, 0);
610 error = (*kcb->kctl->connect)(kcb->kctl->kctlref, &kcb->sac, &kcb->userdata);
611 socket_lock(so, 0);
612 if (error) {
613 goto end;
614 }
615 soisconnected(so);
616 #if DEVELOPMENT || DEBUG
617 kcb->status = KCTL_CONNECTED;
618 #endif /* DEVELOPMENT || DEBUG */
619
620 end:
621 if (error && kcb->kctl->disconnect) {
622 /*
623 * XXX Make sure we Don't check the return value
624 * of disconnect here.
625 * ipsec/utun_ctl_disconnect will return error when
626 * disconnect gets called after connect failure.
627 * However if we decide to check for disconnect return
628 * value here. Please make sure to revisit
629 * ipsec/utun_ctl_disconnect.
630 */
631 socket_unlock(so, 0);
632 (*kcb->kctl->disconnect)(kcb->kctl->kctlref, kcb->sac.sc_unit, kcb->userdata);
633 socket_lock(so, 0);
634 }
635 if (error) {
636 soisdisconnected(so);
637 #if DEVELOPMENT || DEBUG
638 kcb->status = KCTL_DISCONNECTED;
639 #endif /* DEVELOPMENT || DEBUG */
640 lck_mtx_lock(&ctl_mtx);
641 TAILQ_REMOVE(&kcb->kctl->kcb_head, kcb, next);
642 kcb->kctl = NULL;
643 kcb->sac.sc_unit = 0;
644 kctlstat.kcs_pcbcount--;
645 kctlstat.kcs_gencnt++;
646 kctlstat.kcs_conn_fail++;
647 lck_mtx_unlock(&ctl_mtx);
648 }
649 out:
650 ctl_kcb_done_clearing(kcb);
651 ctl_kcb_decrement_use_count(kcb);
652 return error;
653 }
654
655 static int
ctl_disconnect(struct socket * so)656 ctl_disconnect(struct socket *so)
657 {
658 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
659
660 if ((kcb = (struct ctl_cb *)so->so_pcb)) {
661 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
662 ctl_kcb_increment_use_count(kcb, mtx_held);
663 ctl_kcb_require_clearing(kcb, mtx_held);
664 struct kctl *kctl = kcb->kctl;
665
666 if (kctl && kctl->disconnect) {
667 socket_unlock(so, 0);
668 (*kctl->disconnect)(kctl->kctlref, kcb->sac.sc_unit,
669 kcb->userdata);
670 socket_lock(so, 0);
671 }
672
673 soisdisconnected(so);
674 #if DEVELOPMENT || DEBUG
675 kcb->status = KCTL_DISCONNECTED;
676 #endif /* DEVELOPMENT || DEBUG */
677
678 socket_unlock(so, 0);
679 lck_mtx_lock(&ctl_mtx);
680 kcb->kctl = 0;
681 kcb->sac.sc_unit = 0;
682 while (kcb->usecount != 0) {
683 msleep(&kcb->usecount, &ctl_mtx, 0, "kcb->usecount", 0);
684 }
685 TAILQ_REMOVE(&kctl->kcb_head, kcb, next);
686 kctlstat.kcs_pcbcount--;
687 kctlstat.kcs_gencnt++;
688 lck_mtx_unlock(&ctl_mtx);
689 socket_lock(so, 0);
690 ctl_kcb_done_clearing(kcb);
691 ctl_kcb_decrement_use_count(kcb);
692 }
693 return 0;
694 }
695
696 static int
ctl_peeraddr(struct socket * so,struct sockaddr ** nam)697 ctl_peeraddr(struct socket *so, struct sockaddr **nam)
698 {
699 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
700 struct kctl *kctl;
701 struct sockaddr_ctl sc;
702
703 if (kcb == NULL) { /* sanity check */
704 return ENOTCONN;
705 }
706
707 if ((kctl = kcb->kctl) == NULL) {
708 return EINVAL;
709 }
710
711 bzero(&sc, sizeof(struct sockaddr_ctl));
712 sc.sc_len = sizeof(struct sockaddr_ctl);
713 sc.sc_family = AF_SYSTEM;
714 sc.ss_sysaddr = AF_SYS_CONTROL;
715 sc.sc_id = kctl->id;
716 sc.sc_unit = kcb->sac.sc_unit;
717
718 *nam = dup_sockaddr((struct sockaddr *)&sc, 1);
719
720 return 0;
721 }
722
723 static void
ctl_sbrcv_trim(struct socket * so)724 ctl_sbrcv_trim(struct socket *so)
725 {
726 struct sockbuf *sb = &so->so_rcv;
727
728 if (sb->sb_hiwat > sb->sb_idealsize) {
729 u_int32_t diff;
730 int32_t trim;
731
732 /*
733 * The difference between the ideal size and the
734 * current size is the upper bound of the trimage
735 */
736 diff = sb->sb_hiwat - sb->sb_idealsize;
737 /*
738 * We cannot trim below the outstanding data
739 */
740 trim = sb->sb_hiwat - sb->sb_cc;
741
742 trim = imin(trim, (int32_t)diff);
743
744 if (trim > 0) {
745 sbreserve(sb, (sb->sb_hiwat - trim));
746
747 if (ctl_debug) {
748 printf("%s - shrunk to %d\n",
749 __func__, sb->sb_hiwat);
750 }
751 }
752 }
753 }
754
755 static int
ctl_usr_rcvd(struct socket * so,int flags)756 ctl_usr_rcvd(struct socket *so, int flags)
757 {
758 int error = 0;
759 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
760 struct kctl *kctl;
761
762 if (kcb == NULL) {
763 return ENOTCONN;
764 }
765
766 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
767 ctl_kcb_increment_use_count(kcb, mtx_held);
768
769 if ((kctl = kcb->kctl) == NULL) {
770 error = EINVAL;
771 goto out;
772 }
773
774 if (kctl->rcvd) {
775 socket_unlock(so, 0);
776 (*kctl->rcvd)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata, flags);
777 socket_lock(so, 0);
778 }
779
780 ctl_sbrcv_trim(so);
781
782 out:
783 ctl_kcb_decrement_use_count(kcb);
784 return error;
785 }
786
787 static int
ctl_send(struct socket * so,int flags,struct mbuf * m,struct sockaddr * addr,struct mbuf * control,struct proc * p)788 ctl_send(struct socket *so, int flags, struct mbuf *m,
789 struct sockaddr *addr, struct mbuf *control,
790 struct proc *p)
791 {
792 #pragma unused(addr, p)
793 int error = 0;
794 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
795 struct kctl *kctl;
796
797 if (control) {
798 m_freem(control);
799 }
800
801 if (kcb == NULL) { /* sanity check */
802 m_freem(m);
803 return ENOTCONN;
804 }
805
806 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
807 ctl_kcb_increment_use_count(kcb, mtx_held);
808
809 if (error == 0 && (kctl = kcb->kctl) == NULL) {
810 error = EINVAL;
811 }
812
813 if (error == 0 && kctl->send) {
814 so_tc_update_stats(m, so, m_get_service_class(m));
815 socket_unlock(so, 0);
816 error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit, kcb->userdata,
817 m, flags);
818 socket_lock(so, 0);
819 } else {
820 m_freem(m);
821 if (error == 0) {
822 error = ENOTSUP;
823 }
824 }
825 if (error != 0) {
826 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_send_fail);
827 }
828 ctl_kcb_decrement_use_count(kcb);
829
830 return error;
831 }
832
833 static int
ctl_send_list(struct socket * so,struct mbuf * m,u_int * pktcnt,int flags)834 ctl_send_list(struct socket *so, struct mbuf *m, u_int *pktcnt, int flags)
835 {
836 int error = 0;
837 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
838 struct kctl *kctl;
839 const bool update_tc = SOCK_DOM(so) == PF_INET || SOCK_DOM(so) == PF_INET6;
840
841 if (kcb == NULL) { /* sanity check */
842 m_freem_list(m);
843 return ENOTCONN;
844 }
845
846 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
847 ctl_kcb_increment_use_count(kcb, mtx_held);
848
849 if ((kctl = kcb->kctl) == NULL) {
850 error = EINVAL;
851 goto done;
852 }
853
854 if (kctl->send_list != NULL) {
855 struct mbuf *nxt;
856
857 for (nxt = m; update_tc && nxt != NULL; nxt = nxt->m_nextpkt) {
858 so_tc_update_stats(nxt, so, m_get_service_class(nxt));
859 }
860
861 socket_unlock(so, 0);
862 error = (*kctl->send_list)(kctl->kctlref, kcb->sac.sc_unit,
863 kcb->userdata, m, flags);
864 socket_lock(so, 0);
865 } else {
866 *pktcnt = 0;
867 while (m != NULL && error == 0) {
868 struct mbuf *nextpkt = m->m_nextpkt;
869
870 m->m_nextpkt = NULL;
871
872 if (update_tc) {
873 so_tc_update_stats(m, so, m_get_service_class(m));
874 }
875 socket_unlock(so, 0);
876 error = (*kctl->send)(kctl->kctlref, kcb->sac.sc_unit,
877 kcb->userdata, m, flags);
878 socket_lock(so, 0);
879 m = nextpkt;
880 if (error == 0) {
881 *pktcnt += 1;
882 }
883 }
884 if (m != NULL) {
885 m_freem_list(m);
886 }
887 }
888 done:
889 if (error != 0) {
890 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_send_list_fail);
891 }
892 ctl_kcb_decrement_use_count(kcb);
893
894 return error;
895 }
896
897 static errno_t
ctl_rcvbspace(struct socket * so,size_t datasize,u_int32_t kctlflags,u_int32_t flags)898 ctl_rcvbspace(struct socket *so, size_t datasize,
899 u_int32_t kctlflags, u_int32_t flags)
900 {
901 struct sockbuf *sb = &so->so_rcv;
902 u_int32_t space = sbspace(sb);
903 errno_t error;
904
905 if ((kctlflags & CTL_FLAG_REG_CRIT) == 0) {
906 if ((u_int32_t) space >= datasize) {
907 error = 0;
908 } else {
909 error = ENOBUFS;
910 }
911 } else if ((flags & CTL_DATA_CRIT) == 0) {
912 /*
913 * Reserve 25% for critical messages
914 */
915 if (space < (sb->sb_hiwat >> 2) ||
916 space < datasize) {
917 error = ENOBUFS;
918 } else {
919 error = 0;
920 }
921 } else {
922 size_t autorcvbuf_max;
923
924 /*
925 * Allow overcommit of 25%
926 */
927 autorcvbuf_max = min(sb->sb_idealsize + (sb->sb_idealsize >> 2),
928 ctl_autorcvbuf_max);
929
930 if ((u_int32_t) space >= datasize) {
931 error = 0;
932 } else if (sb->sb_hiwat < autorcvbuf_max) {
933 /*
934 * Grow with a little bit of leeway
935 */
936 size_t grow = datasize - space + _MSIZE;
937 u_int32_t cc = (u_int32_t)MIN(MIN((sb->sb_hiwat + grow), autorcvbuf_max), UINT32_MAX);
938
939 if (sbreserve(sb, cc) == 1) {
940 if (sb->sb_hiwat > ctl_autorcvbuf_high) {
941 ctl_autorcvbuf_high = sb->sb_hiwat;
942 }
943
944 /*
945 * A final check
946 */
947 if ((u_int32_t) sbspace(sb) >= datasize) {
948 error = 0;
949 } else {
950 error = ENOBUFS;
951 }
952
953 if (ctl_debug) {
954 printf("%s - grown to %d error %d\n",
955 __func__, sb->sb_hiwat, error);
956 }
957 } else {
958 error = ENOBUFS;
959 }
960 } else {
961 error = ENOBUFS;
962 }
963 }
964 return error;
965 }
966
967 errno_t
ctl_enqueuembuf(kern_ctl_ref kctlref,u_int32_t unit,struct mbuf * m,u_int32_t flags)968 ctl_enqueuembuf(kern_ctl_ref kctlref, u_int32_t unit, struct mbuf *m,
969 u_int32_t flags)
970 {
971 struct socket *so;
972 errno_t error = 0;
973 int len = m->m_pkthdr.len;
974 u_int32_t kctlflags;
975
976 so = kcb_find_socket(kctlref, unit, &kctlflags);
977 if (so == NULL) {
978 return EINVAL;
979 }
980
981 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
982 error = ENOBUFS;
983 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
984 goto bye;
985 }
986 if ((flags & CTL_DATA_EOR)) {
987 m->m_flags |= M_EOR;
988 }
989
990 so_recv_data_stat(so, m, 0);
991 if (sbappend_nodrop(&so->so_rcv, m) != 0) {
992 if ((flags & CTL_DATA_NOWAKEUP) == 0) {
993 sorwakeup(so);
994 }
995 } else {
996 error = ENOBUFS;
997 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
998 }
999 bye:
1000 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1001 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1002 __func__, error, len,
1003 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1004 }
1005
1006 socket_unlock(so, 1);
1007 if (error != 0) {
1008 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1009 }
1010
1011 return error;
1012 }
1013
1014 /*
1015 * Compute space occupied by mbuf like sbappendrecord
1016 */
1017 static int
m_space(struct mbuf * m)1018 m_space(struct mbuf *m)
1019 {
1020 int space = 0;
1021 struct mbuf *nxt;
1022
1023 for (nxt = m; nxt != NULL; nxt = nxt->m_next) {
1024 space += nxt->m_len;
1025 }
1026
1027 return space;
1028 }
1029
1030 errno_t
ctl_enqueuembuf_list(void * kctlref,u_int32_t unit,struct mbuf * m_list,u_int32_t flags,struct mbuf ** m_remain)1031 ctl_enqueuembuf_list(void *kctlref, u_int32_t unit, struct mbuf *m_list,
1032 u_int32_t flags, struct mbuf **m_remain)
1033 {
1034 struct socket *so = NULL;
1035 errno_t error = 0;
1036 struct mbuf *m, *nextpkt;
1037 int needwakeup = 0;
1038 int len = 0;
1039 u_int32_t kctlflags;
1040
1041 /*
1042 * Need to point the beginning of the list in case of early exit
1043 */
1044 m = m_list;
1045
1046 /*
1047 * kcb_find_socket takes the socket lock with a reference
1048 */
1049 so = kcb_find_socket(kctlref, unit, &kctlflags);
1050 if (so == NULL) {
1051 error = EINVAL;
1052 goto done;
1053 }
1054
1055 if (kctlflags & CTL_FLAG_REG_SOCK_STREAM) {
1056 error = EOPNOTSUPP;
1057 goto done;
1058 }
1059 if (flags & CTL_DATA_EOR) {
1060 error = EINVAL;
1061 goto done;
1062 }
1063
1064 for (m = m_list; m != NULL; m = nextpkt) {
1065 nextpkt = m->m_nextpkt;
1066
1067 if (m->m_pkthdr.len == 0 && ctl_debug) {
1068 printf("%s: %llx m_pkthdr.len is 0",
1069 __func__, (uint64_t)VM_KERNEL_ADDRPERM(m));
1070 }
1071
1072 /*
1073 * The mbuf is either appended or freed by sbappendrecord()
1074 * so it's not reliable from a data standpoint
1075 */
1076 len = m_space(m);
1077 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
1078 error = ENOBUFS;
1079 OSIncrementAtomic64(
1080 (SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1081 break;
1082 } else {
1083 /*
1084 * Unlink from the list, m is on its own
1085 */
1086 m->m_nextpkt = NULL;
1087 so_recv_data_stat(so, m, 0);
1088 if (sbappendrecord_nodrop(&so->so_rcv, m) != 0) {
1089 needwakeup = 1;
1090 } else {
1091 /*
1092 * We free or return the remaining
1093 * mbufs in the list
1094 */
1095 m = nextpkt;
1096 error = ENOBUFS;
1097 OSIncrementAtomic64(
1098 (SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1099 break;
1100 }
1101 }
1102 }
1103 if (needwakeup && (flags & CTL_DATA_NOWAKEUP) == 0) {
1104 sorwakeup(so);
1105 }
1106
1107 done:
1108 if (so != NULL) {
1109 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1110 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1111 __func__, error, len,
1112 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1113 }
1114
1115 socket_unlock(so, 1);
1116 }
1117 if (m_remain) {
1118 *m_remain = m;
1119
1120 if (m != NULL && socket_debug && so != NULL &&
1121 (so->so_options & SO_DEBUG)) {
1122 struct mbuf *n;
1123
1124 printf("%s m_list %llx\n", __func__,
1125 (uint64_t) VM_KERNEL_ADDRPERM(m_list));
1126 for (n = m; n != NULL; n = n->m_nextpkt) {
1127 printf(" remain %llx m_next %llx\n",
1128 (uint64_t) VM_KERNEL_ADDRPERM(n),
1129 (uint64_t) VM_KERNEL_ADDRPERM(n->m_next));
1130 }
1131 }
1132 } else {
1133 if (m != NULL) {
1134 m_freem_list(m);
1135 }
1136 }
1137 if (error != 0) {
1138 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1139 }
1140 return error;
1141 }
1142
1143 errno_t
ctl_enqueuedata(void * kctlref,u_int32_t unit,void * data,size_t len,u_int32_t flags)1144 ctl_enqueuedata(void *kctlref, u_int32_t unit, void *data, size_t len,
1145 u_int32_t flags)
1146 {
1147 struct socket *so;
1148 struct mbuf *m;
1149 errno_t error = 0;
1150 unsigned int num_needed;
1151 struct mbuf *n;
1152 size_t curlen = 0;
1153 u_int32_t kctlflags;
1154
1155 so = kcb_find_socket(kctlref, unit, &kctlflags);
1156 if (so == NULL) {
1157 return EINVAL;
1158 }
1159
1160 if (ctl_rcvbspace(so, len, kctlflags, flags) != 0) {
1161 error = ENOBUFS;
1162 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1163 goto bye;
1164 }
1165
1166 num_needed = 1;
1167 m = m_allocpacket_internal(&num_needed, len, NULL, M_NOWAIT, 1, 0);
1168 if (m == NULL) {
1169 kctlstat.kcs_enqdata_mb_alloc_fail++;
1170 if (ctl_debug) {
1171 printf("%s: m_allocpacket_internal(%lu) failed\n",
1172 __func__, len);
1173 }
1174 error = ENOMEM;
1175 goto bye;
1176 }
1177
1178 for (n = m; n != NULL; n = n->m_next) {
1179 size_t mlen = mbuf_maxlen(n);
1180
1181 if (mlen + curlen > len) {
1182 mlen = len - curlen;
1183 }
1184 n->m_len = (int32_t)mlen;
1185 bcopy((char *)data + curlen, m_mtod_current(n), mlen);
1186 curlen += mlen;
1187 }
1188 mbuf_pkthdr_setlen(m, curlen);
1189
1190 if ((flags & CTL_DATA_EOR)) {
1191 m->m_flags |= M_EOR;
1192 }
1193 so_recv_data_stat(so, m, 0);
1194 /*
1195 * No need to call the "nodrop" variant of sbappend
1196 * because the mbuf is local to the scope of the function
1197 */
1198 if (sbappend(&so->so_rcv, m) != 0) {
1199 if ((flags & CTL_DATA_NOWAKEUP) == 0) {
1200 sorwakeup(so);
1201 }
1202 } else {
1203 kctlstat.kcs_enqdata_sbappend_fail++;
1204 error = ENOBUFS;
1205 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fullsock);
1206 }
1207
1208 bye:
1209 if (ctl_debug && error != 0 && (flags & CTL_DATA_CRIT)) {
1210 printf("%s - crit data err %d len %d hiwat %d cc: %d\n",
1211 __func__, error, (int)len,
1212 so->so_rcv.sb_hiwat, so->so_rcv.sb_cc);
1213 }
1214
1215 socket_unlock(so, 1);
1216 if (error != 0) {
1217 OSIncrementAtomic64((SInt64 *)&kctlstat.kcs_enqueue_fail);
1218 }
1219 return error;
1220 }
1221
1222 errno_t
ctl_getenqueuepacketcount(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * pcnt)1223 ctl_getenqueuepacketcount(kern_ctl_ref kctlref, u_int32_t unit, u_int32_t *pcnt)
1224 {
1225 struct socket *so;
1226 u_int32_t cnt;
1227 struct mbuf *m1;
1228
1229 if (pcnt == NULL) {
1230 return EINVAL;
1231 }
1232
1233 so = kcb_find_socket(kctlref, unit, NULL);
1234 if (so == NULL) {
1235 return EINVAL;
1236 }
1237
1238 cnt = 0;
1239 m1 = so->so_rcv.sb_mb;
1240 while (m1 != NULL) {
1241 if (m_has_mtype(m1, MTF_DATA | MTF_HEADER | MTF_OOBDATA)) {
1242 cnt += 1;
1243 }
1244 m1 = m1->m_nextpkt;
1245 }
1246 *pcnt = cnt;
1247
1248 socket_unlock(so, 1);
1249
1250 return 0;
1251 }
1252
1253 errno_t
ctl_getenqueuespace(kern_ctl_ref kctlref,u_int32_t unit,size_t * space)1254 ctl_getenqueuespace(kern_ctl_ref kctlref, u_int32_t unit, size_t *space)
1255 {
1256 struct socket *so;
1257 long avail;
1258
1259 if (space == NULL) {
1260 return EINVAL;
1261 }
1262
1263 so = kcb_find_socket(kctlref, unit, NULL);
1264 if (so == NULL) {
1265 return EINVAL;
1266 }
1267
1268 avail = sbspace(&so->so_rcv);
1269 *space = (avail < 0) ? 0 : avail;
1270 socket_unlock(so, 1);
1271
1272 return 0;
1273 }
1274
1275 errno_t
ctl_getenqueuereadable(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * difference)1276 ctl_getenqueuereadable(kern_ctl_ref kctlref, u_int32_t unit,
1277 u_int32_t *difference)
1278 {
1279 struct socket *so;
1280
1281 if (difference == NULL) {
1282 return EINVAL;
1283 }
1284
1285 so = kcb_find_socket(kctlref, unit, NULL);
1286 if (so == NULL) {
1287 return EINVAL;
1288 }
1289
1290 if (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat) {
1291 *difference = 0;
1292 } else {
1293 *difference = (so->so_rcv.sb_lowat - so->so_rcv.sb_cc);
1294 }
1295 socket_unlock(so, 1);
1296
1297 return 0;
1298 }
1299
1300 static int
ctl_ctloutput(struct socket * so,struct sockopt * sopt)1301 ctl_ctloutput(struct socket *so, struct sockopt *sopt)
1302 {
1303 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
1304 struct kctl *kctl;
1305 int error = 0;
1306 void *data = NULL;
1307 size_t data_len = 0;
1308 size_t len;
1309
1310 if (sopt->sopt_level != SYSPROTO_CONTROL) {
1311 return EINVAL;
1312 }
1313
1314 if (kcb == NULL) { /* sanity check */
1315 return ENOTCONN;
1316 }
1317
1318 if ((kctl = kcb->kctl) == NULL) {
1319 return EINVAL;
1320 }
1321
1322 lck_mtx_t *mtx_held = socket_getlock(so, PR_F_WILLUNLOCK);
1323 ctl_kcb_increment_use_count(kcb, mtx_held);
1324
1325 switch (sopt->sopt_dir) {
1326 case SOPT_SET:
1327 if (kctl->setopt == NULL) {
1328 error = ENOTSUP;
1329 goto out;
1330 }
1331 if (sopt->sopt_valsize != 0) {
1332 data_len = sopt->sopt_valsize;
1333 data = kalloc_data(data_len, Z_WAITOK | Z_ZERO);
1334 if (data == NULL) {
1335 data_len = 0;
1336 error = ENOMEM;
1337 goto out;
1338 }
1339 error = sooptcopyin(sopt, data,
1340 sopt->sopt_valsize, sopt->sopt_valsize);
1341 }
1342 if (error == 0) {
1343 socket_unlock(so, 0);
1344 error = (*kctl->setopt)(kctl->kctlref,
1345 kcb->sac.sc_unit, kcb->userdata, sopt->sopt_name,
1346 data, sopt->sopt_valsize);
1347 socket_lock(so, 0);
1348 }
1349
1350 kfree_data(data, data_len);
1351 break;
1352
1353 case SOPT_GET:
1354 if (kctl->getopt == NULL) {
1355 error = ENOTSUP;
1356 goto out;
1357 }
1358
1359 if (sopt->sopt_valsize && sopt->sopt_val) {
1360 data_len = sopt->sopt_valsize;
1361 data = kalloc_data(data_len, Z_WAITOK | Z_ZERO);
1362 if (data == NULL) {
1363 data_len = 0;
1364 error = ENOMEM;
1365 goto out;
1366 }
1367 /*
1368 * 4108337 - copy user data in case the
1369 * kernel control needs it
1370 */
1371 error = sooptcopyin(sopt, data,
1372 sopt->sopt_valsize, sopt->sopt_valsize);
1373 }
1374
1375 if (error == 0) {
1376 len = sopt->sopt_valsize;
1377 socket_unlock(so, 0);
1378 error = (*kctl->getopt)(kctl->kctlref, kcb->sac.sc_unit,
1379 kcb->userdata, sopt->sopt_name,
1380 data, &len);
1381 if (data != NULL && len > sopt->sopt_valsize) {
1382 panic_plain("ctl_ctloutput: ctl %s returned "
1383 "len (%lu) > sopt_valsize (%lu)\n",
1384 kcb->kctl->name, len,
1385 sopt->sopt_valsize);
1386 }
1387 socket_lock(so, 0);
1388 if (error == 0) {
1389 if (data != NULL) {
1390 error = sooptcopyout(sopt, data, len);
1391 } else {
1392 sopt->sopt_valsize = len;
1393 }
1394 }
1395 }
1396
1397 kfree_data(data, data_len);
1398 break;
1399 }
1400
1401 out:
1402 ctl_kcb_decrement_use_count(kcb);
1403 return error;
1404 }
1405
1406 static int
ctl_ioctl(struct socket * so,u_long cmd,caddr_t data,struct ifnet * ifp,struct proc * p)1407 ctl_ioctl(struct socket *so, u_long cmd, caddr_t data,
1408 struct ifnet *ifp, struct proc *p)
1409 {
1410 #pragma unused(so, ifp, p)
1411 int error = ENOTSUP;
1412
1413 switch (cmd) {
1414 /* get the number of controllers */
1415 case CTLIOCGCOUNT: {
1416 struct kctl *kctl;
1417 u_int32_t n = 0;
1418
1419 lck_mtx_lock(&ctl_mtx);
1420 TAILQ_FOREACH(kctl, &ctl_head, next)
1421 n++;
1422 lck_mtx_unlock(&ctl_mtx);
1423
1424 bcopy(&n, data, sizeof(n));
1425 error = 0;
1426 break;
1427 }
1428 case CTLIOCGINFO: {
1429 struct ctl_info ctl_info;
1430 struct kctl *kctl = 0;
1431 size_t name_len;
1432
1433 bcopy(data, &ctl_info, sizeof(ctl_info));
1434 name_len = strnlen(ctl_info.ctl_name, MAX_KCTL_NAME);
1435
1436 if (name_len == 0 || name_len + 1 > MAX_KCTL_NAME) {
1437 error = EINVAL;
1438 break;
1439 }
1440 lck_mtx_lock(&ctl_mtx);
1441 kctl = ctl_find_by_name(ctl_info.ctl_name);
1442 lck_mtx_unlock(&ctl_mtx);
1443 if (kctl == 0) {
1444 error = ENOENT;
1445 break;
1446 }
1447 ctl_info.ctl_id = kctl->id;
1448 bcopy(&ctl_info, data, sizeof(ctl_info));
1449 error = 0;
1450 break;
1451 }
1452
1453 /* add controls to get list of NKEs */
1454 }
1455
1456 return error;
1457 }
1458
1459 static void
kctl_tbl_grow(void)1460 kctl_tbl_grow(void)
1461 {
1462 struct kctl **new_table;
1463 uintptr_t new_size;
1464
1465 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1466
1467 if (kctl_tbl_growing) {
1468 /* Another thread is allocating */
1469 kctl_tbl_growing_waiting++;
1470
1471 do {
1472 (void) msleep((caddr_t) &kctl_tbl_growing, &ctl_mtx,
1473 PSOCK | PCATCH, "kctl_tbl_growing", 0);
1474 } while (kctl_tbl_growing);
1475 kctl_tbl_growing_waiting--;
1476 }
1477 /* Another thread grew the table */
1478 if (kctl_table != NULL && kctl_tbl_count < kctl_tbl_size) {
1479 return;
1480 }
1481
1482 /* Verify we have a sane size */
1483 if (kctl_tbl_size + KCTL_TBL_INC >= UINT16_MAX) {
1484 kctlstat.kcs_tbl_size_too_big++;
1485 if (ctl_debug) {
1486 printf("%s kctl_tbl_size %lu too big\n",
1487 __func__, kctl_tbl_size);
1488 }
1489 return;
1490 }
1491 kctl_tbl_growing = 1;
1492
1493 new_size = kctl_tbl_size + KCTL_TBL_INC;
1494
1495 lck_mtx_unlock(&ctl_mtx);
1496 new_table = kalloc_type(struct kctl *, new_size, Z_WAITOK | Z_ZERO);
1497 lck_mtx_lock(&ctl_mtx);
1498
1499 if (new_table != NULL) {
1500 if (kctl_table != NULL) {
1501 bcopy(kctl_table, new_table,
1502 kctl_tbl_size * sizeof(struct kctl *));
1503
1504 kfree_type(struct kctl *, kctl_tbl_size, kctl_table);
1505 }
1506 kctl_table = new_table;
1507 kctl_tbl_size = new_size;
1508 }
1509
1510 kctl_tbl_growing = 0;
1511
1512 if (kctl_tbl_growing_waiting) {
1513 wakeup(&kctl_tbl_growing);
1514 }
1515 }
1516
1517 #define KCTLREF_INDEX_MASK 0x0000FFFF
1518 #define KCTLREF_GENCNT_MASK 0xFFFF0000
1519 #define KCTLREF_GENCNT_SHIFT 16
1520
1521 static kern_ctl_ref
kctl_make_ref(struct kctl * kctl)1522 kctl_make_ref(struct kctl *kctl)
1523 {
1524 uintptr_t i;
1525
1526 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1527
1528 if (kctl_tbl_count >= kctl_tbl_size) {
1529 kctl_tbl_grow();
1530 }
1531
1532 kctl->kctlref = NULL;
1533 for (i = 0; i < kctl_tbl_size; i++) {
1534 if (kctl_table[i] == NULL) {
1535 uintptr_t ref;
1536
1537 /*
1538 * Reference is index plus one
1539 */
1540 kctl_ref_gencnt += 1;
1541
1542 /*
1543 * Add generation count as salt to reference to prevent
1544 * use after deregister
1545 */
1546 ref = ((kctl_ref_gencnt << KCTLREF_GENCNT_SHIFT) &
1547 KCTLREF_GENCNT_MASK) +
1548 ((i + 1) & KCTLREF_INDEX_MASK);
1549
1550 kctl->kctlref = (void *)(ref);
1551 kctl_table[i] = kctl;
1552 kctl_tbl_count++;
1553 break;
1554 }
1555 }
1556
1557 if (kctl->kctlref == NULL) {
1558 panic("%s no space in table", __func__);
1559 }
1560
1561 if (ctl_debug > 0) {
1562 printf("%s %p for %p\n",
1563 __func__, kctl->kctlref, kctl);
1564 }
1565
1566 return kctl->kctlref;
1567 }
1568
1569 static void
kctl_delete_ref(kern_ctl_ref kctlref)1570 kctl_delete_ref(kern_ctl_ref kctlref)
1571 {
1572 /*
1573 * Reference is index plus one
1574 */
1575 uintptr_t i = (((uintptr_t)kctlref) & KCTLREF_INDEX_MASK) - 1;
1576
1577 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1578
1579 if (i < kctl_tbl_size) {
1580 struct kctl *kctl = kctl_table[i];
1581
1582 if (kctl->kctlref == kctlref) {
1583 kctl_table[i] = NULL;
1584 kctl_tbl_count--;
1585 } else {
1586 kctlstat.kcs_bad_kctlref++;
1587 }
1588 } else {
1589 kctlstat.kcs_bad_kctlref++;
1590 }
1591 }
1592
1593 static struct kctl *
kctl_from_ref(kern_ctl_ref kctlref)1594 kctl_from_ref(kern_ctl_ref kctlref)
1595 {
1596 /*
1597 * Reference is index plus one
1598 */
1599 uintptr_t i = (((uintptr_t)kctlref) & KCTLREF_INDEX_MASK) - 1;
1600 struct kctl *kctl = NULL;
1601
1602 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1603
1604 if (i >= kctl_tbl_size) {
1605 kctlstat.kcs_bad_kctlref++;
1606 return NULL;
1607 }
1608 kctl = kctl_table[i];
1609 if (kctl->kctlref != kctlref) {
1610 kctlstat.kcs_bad_kctlref++;
1611 return NULL;
1612 }
1613 return kctl;
1614 }
1615
1616 /*
1617 * Register/unregister a NKE
1618 */
1619 errno_t
ctl_register(struct kern_ctl_reg * userkctl,kern_ctl_ref * kctlref)1620 ctl_register(struct kern_ctl_reg *userkctl, kern_ctl_ref *kctlref)
1621 {
1622 struct kctl *kctl = NULL;
1623 struct kctl *kctl_next = NULL;
1624 u_int32_t id = 1;
1625 size_t name_len;
1626 int is_extended = 0;
1627 int is_setup = 0;
1628
1629 if (userkctl == NULL) { /* sanity check */
1630 return EINVAL;
1631 }
1632 if (userkctl->ctl_connect == NULL) {
1633 return EINVAL;
1634 }
1635 name_len = strlen(userkctl->ctl_name);
1636 if (name_len == 0 || name_len + 1 > MAX_KCTL_NAME) {
1637 return EINVAL;
1638 }
1639
1640 kctl = kalloc_type(struct kctl, Z_WAITOK | Z_ZERO | Z_NOFAIL);
1641
1642 lck_mtx_lock(&ctl_mtx);
1643
1644 if (kctl_make_ref(kctl) == NULL) {
1645 lck_mtx_unlock(&ctl_mtx);
1646 kfree_type(struct kctl, kctl);
1647 return ENOMEM;
1648 }
1649
1650 /*
1651 * Kernel Control IDs
1652 *
1653 * CTL_FLAG_REG_ID_UNIT indicates the control ID and unit number are
1654 * static. If they do not exist, add them to the list in order. If the
1655 * flag is not set, we must find a new unique value. We assume the
1656 * list is in order. We find the last item in the list and add one. If
1657 * this leads to wrapping the id around, we start at the front of the
1658 * list and look for a gap.
1659 */
1660
1661 if ((userkctl->ctl_flags & CTL_FLAG_REG_ID_UNIT) == 0) {
1662 /* Must dynamically assign an unused ID */
1663
1664 /* Verify the same name isn't already registered */
1665 if (ctl_find_by_name(userkctl->ctl_name) != NULL) {
1666 kctl_delete_ref(kctl->kctlref);
1667 lck_mtx_unlock(&ctl_mtx);
1668 kfree_type(struct kctl, kctl);
1669 return EEXIST;
1670 }
1671
1672 /* Start with 1 in case the list is empty */
1673 id = 1;
1674 kctl_next = TAILQ_LAST(&ctl_head, kctl_list);
1675
1676 if (kctl_next != NULL) {
1677 /* List was not empty, add one to the last item */
1678 id = kctl_next->id + 1;
1679 kctl_next = NULL;
1680
1681 /*
1682 * If this wrapped the id number, start looking at
1683 * the front of the list for an unused id.
1684 */
1685 if (id == 0) {
1686 /* Find the next unused ID */
1687 id = 1;
1688
1689 TAILQ_FOREACH(kctl_next, &ctl_head, next) {
1690 if (kctl_next->id > id) {
1691 /* We found a gap */
1692 break;
1693 }
1694
1695 id = kctl_next->id + 1;
1696 }
1697 }
1698 }
1699
1700 userkctl->ctl_id = id;
1701 kctl->id = id;
1702 kctl->reg_unit = -1;
1703 } else {
1704 TAILQ_FOREACH(kctl_next, &ctl_head, next) {
1705 if (kctl_next->id > userkctl->ctl_id) {
1706 break;
1707 }
1708 }
1709
1710 if (ctl_find_by_id_unit(userkctl->ctl_id, userkctl->ctl_unit)) {
1711 kctl_delete_ref(kctl->kctlref);
1712 lck_mtx_unlock(&ctl_mtx);
1713 kfree_type(struct kctl, kctl);
1714 return EEXIST;
1715 }
1716 kctl->id = userkctl->ctl_id;
1717 kctl->reg_unit = userkctl->ctl_unit;
1718 }
1719
1720 is_extended = (userkctl->ctl_flags & CTL_FLAG_REG_EXTENDED);
1721 is_setup = (userkctl->ctl_flags & CTL_FLAG_REG_SETUP);
1722
1723 strlcpy(kctl->name, userkctl->ctl_name, MAX_KCTL_NAME);
1724 kctl->flags = userkctl->ctl_flags;
1725
1726 /*
1727 * Let the caller know the default send and receive sizes
1728 */
1729 if (userkctl->ctl_sendsize == 0) {
1730 kctl->sendbufsize = CTL_SENDSIZE;
1731 userkctl->ctl_sendsize = kctl->sendbufsize;
1732 } else {
1733 kctl->sendbufsize = userkctl->ctl_sendsize;
1734 }
1735 if (userkctl->ctl_recvsize == 0) {
1736 kctl->recvbufsize = CTL_RECVSIZE;
1737 userkctl->ctl_recvsize = kctl->recvbufsize;
1738 } else {
1739 kctl->recvbufsize = userkctl->ctl_recvsize;
1740 }
1741
1742 if (is_setup) {
1743 kctl->setup = userkctl->ctl_setup;
1744 }
1745 kctl->bind = userkctl->ctl_bind;
1746 kctl->connect = userkctl->ctl_connect;
1747 kctl->disconnect = userkctl->ctl_disconnect;
1748 kctl->send = userkctl->ctl_send;
1749 kctl->setopt = userkctl->ctl_setopt;
1750 kctl->getopt = userkctl->ctl_getopt;
1751 if (is_extended) {
1752 kctl->rcvd = userkctl->ctl_rcvd;
1753 kctl->send_list = userkctl->ctl_send_list;
1754 }
1755
1756 TAILQ_INIT(&kctl->kcb_head);
1757
1758 if (kctl_next) {
1759 TAILQ_INSERT_BEFORE(kctl_next, kctl, next);
1760 } else {
1761 TAILQ_INSERT_TAIL(&ctl_head, kctl, next);
1762 }
1763
1764 kctlstat.kcs_reg_count++;
1765 kctlstat.kcs_gencnt++;
1766
1767 lck_mtx_unlock(&ctl_mtx);
1768
1769 *kctlref = kctl->kctlref;
1770
1771 ctl_post_msg(KEV_CTL_REGISTERED, kctl->id);
1772 return 0;
1773 }
1774
1775 errno_t
ctl_deregister(void * kctlref)1776 ctl_deregister(void *kctlref)
1777 {
1778 struct kctl *kctl;
1779
1780 lck_mtx_lock(&ctl_mtx);
1781 if ((kctl = kctl_from_ref(kctlref)) == NULL) {
1782 kctlstat.kcs_bad_kctlref++;
1783 lck_mtx_unlock(&ctl_mtx);
1784 if (ctl_debug != 0) {
1785 printf("%s invalid kctlref %p\n",
1786 __func__, kctlref);
1787 }
1788 return EINVAL;
1789 }
1790
1791 if (!TAILQ_EMPTY(&kctl->kcb_head)) {
1792 lck_mtx_unlock(&ctl_mtx);
1793 return EBUSY;
1794 }
1795
1796 TAILQ_REMOVE(&ctl_head, kctl, next);
1797
1798 kctlstat.kcs_reg_count--;
1799 kctlstat.kcs_gencnt++;
1800
1801 kctl_delete_ref(kctl->kctlref);
1802 lck_mtx_unlock(&ctl_mtx);
1803
1804 ctl_post_msg(KEV_CTL_DEREGISTERED, kctl->id);
1805 kfree_type(struct kctl, kctl);
1806 return 0;
1807 }
1808
1809 /*
1810 * Must be called with global ctl_mtx lock taked
1811 */
1812 static struct kctl *
ctl_find_by_name(const char * name)1813 ctl_find_by_name(const char *name)
1814 {
1815 struct kctl *kctl;
1816
1817 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1818
1819 TAILQ_FOREACH(kctl, &ctl_head, next)
1820 if (strncmp(kctl->name, name, sizeof(kctl->name)) == 0) {
1821 return kctl;
1822 }
1823
1824 return NULL;
1825 }
1826
1827 u_int32_t
ctl_id_by_name(const char * name)1828 ctl_id_by_name(const char *name)
1829 {
1830 u_int32_t ctl_id = 0;
1831 struct kctl *kctl;
1832
1833 lck_mtx_lock(&ctl_mtx);
1834 kctl = ctl_find_by_name(name);
1835 if (kctl) {
1836 ctl_id = kctl->id;
1837 }
1838 lck_mtx_unlock(&ctl_mtx);
1839
1840 return ctl_id;
1841 }
1842
1843 errno_t
ctl_name_by_id(u_int32_t id,char * out_name,size_t maxsize)1844 ctl_name_by_id(u_int32_t id, char *out_name, size_t maxsize)
1845 {
1846 int found = 0;
1847 struct kctl *kctl;
1848
1849 lck_mtx_lock(&ctl_mtx);
1850 TAILQ_FOREACH(kctl, &ctl_head, next) {
1851 if (kctl->id == id) {
1852 break;
1853 }
1854 }
1855
1856 if (kctl) {
1857 if (maxsize > MAX_KCTL_NAME) {
1858 maxsize = MAX_KCTL_NAME;
1859 }
1860 strlcpy(out_name, kctl->name, maxsize);
1861 found = 1;
1862 }
1863 lck_mtx_unlock(&ctl_mtx);
1864
1865 return found ? 0 : ENOENT;
1866 }
1867
1868 /*
1869 * Must be called with global ctl_mtx lock taked
1870 *
1871 */
1872 static struct kctl *
ctl_find_by_id_unit(u_int32_t id,u_int32_t unit)1873 ctl_find_by_id_unit(u_int32_t id, u_int32_t unit)
1874 {
1875 struct kctl *kctl;
1876
1877 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1878
1879 TAILQ_FOREACH(kctl, &ctl_head, next) {
1880 if (kctl->id == id && (kctl->flags & CTL_FLAG_REG_ID_UNIT) == 0) {
1881 return kctl;
1882 } else if (kctl->id == id && kctl->reg_unit == unit) {
1883 return kctl;
1884 }
1885 }
1886 return NULL;
1887 }
1888
1889 /*
1890 * Must be called with kernel controller lock taken
1891 */
1892 static struct ctl_cb *
kcb_find(struct kctl * kctl,u_int32_t unit)1893 kcb_find(struct kctl *kctl, u_int32_t unit)
1894 {
1895 struct ctl_cb *kcb;
1896
1897 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_OWNED);
1898
1899 TAILQ_FOREACH(kcb, &kctl->kcb_head, next)
1900 if (kcb->sac.sc_unit == unit) {
1901 return kcb;
1902 }
1903
1904 return NULL;
1905 }
1906
1907 static struct socket *
kcb_find_socket(kern_ctl_ref kctlref,u_int32_t unit,u_int32_t * kctlflags)1908 kcb_find_socket(kern_ctl_ref kctlref, u_int32_t unit, u_int32_t *kctlflags)
1909 {
1910 struct socket *so = NULL;
1911 struct ctl_cb *kcb;
1912 void *lr_saved;
1913 struct kctl *kctl;
1914 int i;
1915
1916 lr_saved = __builtin_return_address(0);
1917
1918 lck_mtx_lock(&ctl_mtx);
1919 /*
1920 * First validate the kctlref
1921 */
1922 if ((kctl = kctl_from_ref(kctlref)) == NULL) {
1923 kctlstat.kcs_bad_kctlref++;
1924 lck_mtx_unlock(&ctl_mtx);
1925 if (ctl_debug != 0) {
1926 printf("%s invalid kctlref %p\n",
1927 __func__, kctlref);
1928 }
1929 return NULL;
1930 }
1931
1932 kcb = kcb_find(kctl, unit);
1933 if (kcb == NULL || kcb->kctl != kctl || (so = kcb->so) == NULL) {
1934 lck_mtx_unlock(&ctl_mtx);
1935 return NULL;
1936 }
1937 /*
1938 * This prevents the socket from being closed
1939 */
1940 kcb->usecount++;
1941 /*
1942 * Respect lock ordering: socket before ctl_mtx
1943 */
1944 lck_mtx_unlock(&ctl_mtx);
1945
1946 socket_lock(so, 1);
1947 /*
1948 * The socket lock history is more useful if we store
1949 * the address of the caller.
1950 */
1951 i = (so->next_lock_lr + SO_LCKDBG_MAX - 1) % SO_LCKDBG_MAX;
1952 so->lock_lr[i] = lr_saved;
1953
1954 lck_mtx_lock(&ctl_mtx);
1955
1956 if ((kctl = kctl_from_ref(kctlref)) == NULL || kcb->kctl == NULL) {
1957 lck_mtx_unlock(&ctl_mtx);
1958 socket_unlock(so, 1);
1959 so = NULL;
1960 lck_mtx_lock(&ctl_mtx);
1961 } else if (kctlflags != NULL) {
1962 *kctlflags = kctl->flags;
1963 }
1964
1965 kcb->usecount--;
1966 if (kcb->usecount == 0 && kcb->require_clearing_count != 0) {
1967 wakeup((event_t)&kcb->usecount);
1968 }
1969
1970 lck_mtx_unlock(&ctl_mtx);
1971
1972 return so;
1973 }
1974
1975 static void
ctl_post_msg(u_int32_t event_code,u_int32_t id)1976 ctl_post_msg(u_int32_t event_code, u_int32_t id)
1977 {
1978 struct ctl_event_data ctl_ev_data;
1979 struct kev_msg ev_msg;
1980
1981 lck_mtx_assert(&ctl_mtx, LCK_MTX_ASSERT_NOTOWNED);
1982
1983 bzero(&ev_msg, sizeof(struct kev_msg));
1984 ev_msg.vendor_code = KEV_VENDOR_APPLE;
1985
1986 ev_msg.kev_class = KEV_SYSTEM_CLASS;
1987 ev_msg.kev_subclass = KEV_CTL_SUBCLASS;
1988 ev_msg.event_code = event_code;
1989
1990 /* common nke subclass data */
1991 bzero(&ctl_ev_data, sizeof(ctl_ev_data));
1992 ctl_ev_data.ctl_id = id;
1993 ev_msg.dv[0].data_ptr = &ctl_ev_data;
1994 ev_msg.dv[0].data_length = sizeof(ctl_ev_data);
1995
1996 ev_msg.dv[1].data_length = 0;
1997
1998 kev_post_msg(&ev_msg);
1999 }
2000
2001 static int
ctl_lock(struct socket * so,int refcount,void * lr)2002 ctl_lock(struct socket *so, int refcount, void *lr)
2003 {
2004 void *lr_saved;
2005
2006 if (lr == NULL) {
2007 lr_saved = __builtin_return_address(0);
2008 } else {
2009 lr_saved = lr;
2010 }
2011
2012 if (so->so_pcb != NULL) {
2013 lck_mtx_lock(&((struct ctl_cb *)so->so_pcb)->mtx);
2014 } else {
2015 panic("ctl_lock: so=%p NO PCB! lr=%p lrh= %s",
2016 so, lr_saved, solockhistory_nr(so));
2017 /* NOTREACHED */
2018 }
2019
2020 if (so->so_usecount < 0) {
2021 panic("ctl_lock: so=%p so_pcb=%p lr=%p ref=%x lrh= %s",
2022 so, so->so_pcb, lr_saved, so->so_usecount,
2023 solockhistory_nr(so));
2024 /* NOTREACHED */
2025 }
2026
2027 if (refcount) {
2028 so->so_usecount++;
2029 }
2030
2031 so->lock_lr[so->next_lock_lr] = lr_saved;
2032 so->next_lock_lr = (so->next_lock_lr + 1) % SO_LCKDBG_MAX;
2033 return 0;
2034 }
2035
2036 static int
ctl_unlock(struct socket * so,int refcount,void * lr)2037 ctl_unlock(struct socket *so, int refcount, void *lr)
2038 {
2039 void *lr_saved;
2040 lck_mtx_t *mutex_held;
2041
2042 if (lr == NULL) {
2043 lr_saved = __builtin_return_address(0);
2044 } else {
2045 lr_saved = lr;
2046 }
2047
2048 #if (MORE_KCTLLOCK_DEBUG && (DEVELOPMENT || DEBUG))
2049 printf("ctl_unlock: so=%llx sopcb=%x lock=%llx ref=%u lr=%llx\n",
2050 (uint64_t)VM_KERNEL_ADDRPERM(so),
2051 (uint64_t)VM_KERNEL_ADDRPERM(so->so_pcb,
2052 (uint64_t)VM_KERNEL_ADDRPERM(&((struct ctl_cb *)so->so_pcb)->mtx),
2053 so->so_usecount, (uint64_t)VM_KERNEL_ADDRPERM(lr_saved));
2054 #endif /* (MORE_KCTLLOCK_DEBUG && (DEVELOPMENT || DEBUG)) */
2055 if (refcount) {
2056 so->so_usecount--;
2057 }
2058
2059 if (so->so_usecount < 0) {
2060 panic("ctl_unlock: so=%p usecount=%x lrh= %s",
2061 so, so->so_usecount, solockhistory_nr(so));
2062 /* NOTREACHED */
2063 }
2064 if (so->so_pcb == NULL) {
2065 panic("ctl_unlock: so=%p NO PCB usecount=%x lr=%p lrh= %s",
2066 so, so->so_usecount, (void *)lr_saved,
2067 solockhistory_nr(so));
2068 /* NOTREACHED */
2069 }
2070 mutex_held = &((struct ctl_cb *)so->so_pcb)->mtx;
2071
2072 lck_mtx_assert(mutex_held, LCK_MTX_ASSERT_OWNED);
2073 so->unlock_lr[so->next_unlock_lr] = lr_saved;
2074 so->next_unlock_lr = (so->next_unlock_lr + 1) % SO_LCKDBG_MAX;
2075 lck_mtx_unlock(mutex_held);
2076
2077 if (so->so_usecount == 0) {
2078 ctl_sofreelastref(so);
2079 }
2080
2081 return 0;
2082 }
2083
2084 static lck_mtx_t *
2085 ctl_getlock(struct socket *so, int flags)
2086 {
2087 #pragma unused(flags)
2088 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
2089
2090 if (so->so_pcb) {
2091 if (so->so_usecount < 0) {
2092 panic("ctl_getlock: so=%p usecount=%x lrh= %s",
2093 so, so->so_usecount, solockhistory_nr(so));
2094 }
2095 return &kcb->mtx;
2096 } else {
2097 panic("ctl_getlock: so=%p NULL NO so_pcb %s",
2098 so, solockhistory_nr(so));
2099 return so->so_proto->pr_domain->dom_mtx;
2100 }
2101 }
2102
2103 __private_extern__ int
2104 kctl_reg_list SYSCTL_HANDLER_ARGS
2105 {
2106 #pragma unused(oidp, arg1, arg2)
2107 int error = 0;
2108 u_int64_t i, n;
2109 struct xsystmgen xsg;
2110 void *buf = NULL;
2111 struct kctl *kctl;
2112 size_t item_size = ROUNDUP64(sizeof(struct xkctl_reg));
2113
2114 buf = kalloc_data(item_size, Z_WAITOK | Z_ZERO | Z_NOFAIL);
2115
2116 lck_mtx_lock(&ctl_mtx);
2117
2118 n = kctlstat.kcs_reg_count;
2119
2120 if (req->oldptr == USER_ADDR_NULL) {
2121 req->oldidx = (size_t)(n + n / 8) * sizeof(struct xkctl_reg);
2122 goto done;
2123 }
2124 if (req->newptr != USER_ADDR_NULL) {
2125 error = EPERM;
2126 goto done;
2127 }
2128 bzero(&xsg, sizeof(xsg));
2129 xsg.xg_len = sizeof(xsg);
2130 xsg.xg_count = n;
2131 xsg.xg_gen = kctlstat.kcs_gencnt;
2132 xsg.xg_sogen = so_gencnt;
2133 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2134 if (error) {
2135 goto done;
2136 }
2137 /*
2138 * We are done if there is no pcb
2139 */
2140 if (n == 0) {
2141 goto done;
2142 }
2143
2144 for (i = 0, kctl = TAILQ_FIRST(&ctl_head);
2145 i < n && kctl != NULL;
2146 i++, kctl = TAILQ_NEXT(kctl, next)) {
2147 struct xkctl_reg *xkr = (struct xkctl_reg *)buf;
2148 struct ctl_cb *kcb;
2149 u_int32_t pcbcount = 0;
2150
2151 TAILQ_FOREACH(kcb, &kctl->kcb_head, next)
2152 pcbcount++;
2153
2154 bzero(buf, item_size);
2155
2156 xkr->xkr_len = sizeof(struct xkctl_reg);
2157 xkr->xkr_kind = XSO_KCREG;
2158 xkr->xkr_id = kctl->id;
2159 xkr->xkr_reg_unit = kctl->reg_unit;
2160 xkr->xkr_flags = kctl->flags;
2161 xkr->xkr_kctlref = (uint64_t)(kctl->kctlref);
2162 xkr->xkr_recvbufsize = kctl->recvbufsize;
2163 xkr->xkr_sendbufsize = kctl->sendbufsize;
2164 xkr->xkr_lastunit = kctl->lastunit;
2165 xkr->xkr_pcbcount = pcbcount;
2166 xkr->xkr_connect = (uint64_t)VM_KERNEL_UNSLIDE(kctl->connect);
2167 xkr->xkr_disconnect =
2168 (uint64_t)VM_KERNEL_UNSLIDE(kctl->disconnect);
2169 xkr->xkr_send = (uint64_t)VM_KERNEL_UNSLIDE(kctl->send);
2170 xkr->xkr_send_list =
2171 (uint64_t)VM_KERNEL_UNSLIDE(kctl->send_list);
2172 xkr->xkr_setopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->setopt);
2173 xkr->xkr_getopt = (uint64_t)VM_KERNEL_UNSLIDE(kctl->getopt);
2174 xkr->xkr_rcvd = (uint64_t)VM_KERNEL_UNSLIDE(kctl->rcvd);
2175 strlcpy(xkr->xkr_name, kctl->name, sizeof(xkr->xkr_name));
2176
2177 error = SYSCTL_OUT(req, buf, item_size);
2178 }
2179
2180 if (error == 0) {
2181 /*
2182 * Give the user an updated idea of our state.
2183 * If the generation differs from what we told
2184 * her before, she knows that something happened
2185 * while we were processing this request, and it
2186 * might be necessary to retry.
2187 */
2188 bzero(&xsg, sizeof(xsg));
2189 xsg.xg_len = sizeof(xsg);
2190 xsg.xg_count = n;
2191 xsg.xg_gen = kctlstat.kcs_gencnt;
2192 xsg.xg_sogen = so_gencnt;
2193 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2194 if (error) {
2195 goto done;
2196 }
2197 }
2198
2199 done:
2200 lck_mtx_unlock(&ctl_mtx);
2201
2202 kfree_data(buf, item_size);
2203
2204 return error;
2205 }
2206
2207 __private_extern__ int
2208 kctl_pcblist SYSCTL_HANDLER_ARGS
2209 {
2210 #pragma unused(oidp, arg1, arg2)
2211 int error = 0;
2212 u_int64_t n, i;
2213 struct xsystmgen xsg;
2214 void *buf = NULL;
2215 struct kctl *kctl;
2216 size_t item_size = ROUNDUP64(sizeof(struct xkctlpcb)) +
2217 ROUNDUP64(sizeof(struct xsocket_n)) +
2218 2 * ROUNDUP64(sizeof(struct xsockbuf_n)) +
2219 ROUNDUP64(sizeof(struct xsockstat_n));
2220
2221 buf = kalloc_data(item_size, Z_WAITOK | Z_ZERO | Z_NOFAIL);
2222
2223 lck_mtx_lock(&ctl_mtx);
2224
2225 n = kctlstat.kcs_pcbcount;
2226
2227 if (req->oldptr == USER_ADDR_NULL) {
2228 req->oldidx = (size_t)(n + n / 8) * item_size;
2229 goto done;
2230 }
2231 if (req->newptr != USER_ADDR_NULL) {
2232 error = EPERM;
2233 goto done;
2234 }
2235 bzero(&xsg, sizeof(xsg));
2236 xsg.xg_len = sizeof(xsg);
2237 xsg.xg_count = n;
2238 xsg.xg_gen = kctlstat.kcs_gencnt;
2239 xsg.xg_sogen = so_gencnt;
2240 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2241 if (error) {
2242 goto done;
2243 }
2244 /*
2245 * We are done if there is no pcb
2246 */
2247 if (n == 0) {
2248 goto done;
2249 }
2250
2251 for (i = 0, kctl = TAILQ_FIRST(&ctl_head);
2252 i < n && kctl != NULL;
2253 kctl = TAILQ_NEXT(kctl, next)) {
2254 struct ctl_cb *kcb;
2255
2256 for (kcb = TAILQ_FIRST(&kctl->kcb_head);
2257 i < n && kcb != NULL;
2258 i++, kcb = TAILQ_NEXT(kcb, next)) {
2259 struct xkctlpcb *xk = (struct xkctlpcb *)buf;
2260 struct xsocket_n *xso = (struct xsocket_n *)
2261 ADVANCE64(xk, sizeof(*xk));
2262 struct xsockbuf_n *xsbrcv = (struct xsockbuf_n *)
2263 ADVANCE64(xso, sizeof(*xso));
2264 struct xsockbuf_n *xsbsnd = (struct xsockbuf_n *)
2265 ADVANCE64(xsbrcv, sizeof(*xsbrcv));
2266 struct xsockstat_n *xsostats = (struct xsockstat_n *)
2267 ADVANCE64(xsbsnd, sizeof(*xsbsnd));
2268
2269 bzero(buf, item_size);
2270
2271 xk->xkp_len = sizeof(struct xkctlpcb);
2272 xk->xkp_kind = XSO_KCB;
2273 xk->xkp_unit = kcb->sac.sc_unit;
2274 xk->xkp_kctpcb = (uint64_t)VM_KERNEL_ADDRPERM(kcb);
2275 xk->xkp_kctlref = (uint64_t)VM_KERNEL_ADDRPERM(kctl);
2276 xk->xkp_kctlid = kctl->id;
2277 strlcpy(xk->xkp_kctlname, kctl->name,
2278 sizeof(xk->xkp_kctlname));
2279
2280 sotoxsocket_n(kcb->so, xso);
2281 sbtoxsockbuf_n(kcb->so ?
2282 &kcb->so->so_rcv : NULL, xsbrcv);
2283 sbtoxsockbuf_n(kcb->so ?
2284 &kcb->so->so_snd : NULL, xsbsnd);
2285 sbtoxsockstat_n(kcb->so, xsostats);
2286
2287 error = SYSCTL_OUT(req, buf, item_size);
2288 }
2289 }
2290
2291 if (error == 0) {
2292 /*
2293 * Give the user an updated idea of our state.
2294 * If the generation differs from what we told
2295 * her before, she knows that something happened
2296 * while we were processing this request, and it
2297 * might be necessary to retry.
2298 */
2299 bzero(&xsg, sizeof(xsg));
2300 xsg.xg_len = sizeof(xsg);
2301 xsg.xg_count = n;
2302 xsg.xg_gen = kctlstat.kcs_gencnt;
2303 xsg.xg_sogen = so_gencnt;
2304 error = SYSCTL_OUT(req, &xsg, sizeof(xsg));
2305 if (error) {
2306 goto done;
2307 }
2308 }
2309
2310 done:
2311 lck_mtx_unlock(&ctl_mtx);
2312
2313 kfree_data(buf, item_size);
2314 return error;
2315 }
2316
2317 int
2318 kctl_getstat SYSCTL_HANDLER_ARGS
2319 {
2320 #pragma unused(oidp, arg1, arg2)
2321 int error = 0;
2322
2323 lck_mtx_lock(&ctl_mtx);
2324
2325 if (req->newptr != USER_ADDR_NULL) {
2326 error = EPERM;
2327 goto done;
2328 }
2329 if (req->oldptr == USER_ADDR_NULL) {
2330 req->oldidx = sizeof(struct kctlstat);
2331 goto done;
2332 }
2333
2334 error = SYSCTL_OUT(req, &kctlstat,
2335 MIN(sizeof(struct kctlstat), req->oldlen));
2336 done:
2337 lck_mtx_unlock(&ctl_mtx);
2338 return error;
2339 }
2340
2341 void
2342 kctl_fill_socketinfo(struct socket *so, struct socket_info *si)
2343 {
2344 struct ctl_cb *kcb = (struct ctl_cb *)so->so_pcb;
2345 struct kern_ctl_info *kcsi =
2346 &si->soi_proto.pri_kern_ctl;
2347 struct kctl *kctl = kcb->kctl;
2348
2349 si->soi_kind = SOCKINFO_KERN_CTL;
2350
2351 if (kctl == 0) {
2352 return;
2353 }
2354
2355 kcsi->kcsi_id = kctl->id;
2356 kcsi->kcsi_reg_unit = kctl->reg_unit;
2357 kcsi->kcsi_flags = kctl->flags;
2358 kcsi->kcsi_recvbufsize = kctl->recvbufsize;
2359 kcsi->kcsi_sendbufsize = kctl->sendbufsize;
2360 kcsi->kcsi_unit = kcb->sac.sc_unit;
2361 strlcpy(kcsi->kcsi_name, kctl->name, MAX_KCTL_NAME);
2362 }
2363