1 /*
2 * Copyright (c) 2019-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk.h>
30 #include <skywalk/os_skywalk_private.h>
31 #include <skywalk/namespace/protons.h>
32
33 #include <kern/bits.h>
34 #include <netinet/in_var.h>
35 #include <netinet6/in6_var.h>
36 #include <sys/domain.h>
37
38 static int __protons_inited = 0;
39
40 decl_lck_mtx_data(static, protons_lock);
41 static LCK_GRP_DECLARE(protons_lock_group, "protons_lock");
42 static LCK_MTX_DECLARE(protons_lock, &protons_lock_group);
43
44 #define PROTONS_LOCK() \
45 lck_mtx_lock(&protons_lock)
46 #define PROTONS_UNLOCK() \
47 lck_mtx_unlock(&protons_lock)
48 #define PROTONS_LOCK_ASSERT_HELD() \
49 LCK_MTX_ASSERT(&protons_lock, LCK_MTX_ASSERT_OWNED)
50 #define PROTONS_LOCK_ASSERT_NOTHELD() \
51 LCK_MTX_ASSERT(&protons_lock, LCK_MTX_ASSERT_NOTOWNED)
52
53 os_refgrp_decl(static, protons_token_refgrp, "protons_token", NULL);
54
55 struct protons_token {
56 RB_ENTRY(protons_token) pt_link;
57 os_refcnt_t pt_refcnt;
58 pid_t pt_pid;
59 pid_t pt_epid;
60 uint8_t pt_protocol;
61 uint8_t pt_flags;
62 };
63
64 enum {
65 PROTONSF_VALID = (((uint8_t)1) << 0),
66 };
67
68 __attribute__((always_inline))
69 static inline int
pt_cmp(const struct protons_token * pt1,const struct protons_token * pt2)70 pt_cmp(const struct protons_token *pt1, const struct protons_token *pt2)
71 {
72 return (int)pt1->pt_protocol - (int)pt2->pt_protocol;
73 }
74 RB_HEAD(protons_token_tree, protons_token);
75 RB_PROTOTYPE_PREV(protons_token_tree, protons_token, pt_link, pt_cmp);
76 RB_GENERATE_PREV(protons_token_tree, protons_token, pt_link, pt_cmp);
77 static struct protons_token_tree protons_tokens;
78
79 static ZONE_DEFINE(protons_token_zone, SKMEM_ZONE_PREFIX ".protons.token",
80 sizeof(struct protons_token), ZC_NONE);
81
82 static struct protons_token *
protons_token_alloc(bool can_block)83 protons_token_alloc(bool can_block)
84 {
85 PROTONS_LOCK_ASSERT_HELD();
86
87 struct protons_token *pt = NULL;
88 pt = can_block ? zalloc(protons_token_zone) :
89 zalloc_noblock(protons_token_zone);
90 if (pt == NULL) {
91 return NULL;
92 }
93
94 memset(pt, 0, sizeof(*pt));
95 os_ref_init(&pt->pt_refcnt, &protons_token_refgrp);
96
97 SK_DF(SK_VERB_NS_PROTO, "token %p alloc", (void *)SK_KVA(pt));
98
99 return pt;
100 }
101
102 static void
protons_token_free(struct protons_token * pt)103 protons_token_free(struct protons_token *pt)
104 {
105 PROTONS_LOCK_ASSERT_HELD();
106
107 SK_DF(SK_VERB_NS_PROTO, "token %p free", (void *)SK_KVA(pt));
108 ASSERT(os_ref_get_count(&pt->pt_refcnt) == 0);
109 zfree(protons_token_zone, pt);
110 }
111
112 bool
protons_token_is_valid(struct protons_token * pt)113 protons_token_is_valid(struct protons_token *pt)
114 {
115 if (__improbable(pt == NULL)) {
116 return false;
117 }
118 return pt->pt_flags & PROTONSF_VALID;
119 }
120
121 bool
protons_token_has_matching_pid(struct protons_token * pt,pid_t pid,pid_t epid)122 protons_token_has_matching_pid(struct protons_token *pt, pid_t pid, pid_t epid)
123 {
124 ASSERT(pt != NULL);
125 return pt->pt_pid == pid && pt->pt_epid == epid;
126 }
127
128 static struct protons_token *
protons_find_token_with_protocol(uint8_t proto)129 protons_find_token_with_protocol(uint8_t proto)
130 {
131 struct protons_token find = { .pt_protocol = proto };
132
133 PROTONS_LOCK_ASSERT_HELD();
134 struct protons_token *pt = NULL;
135 pt = RB_FIND(protons_token_tree, &protons_tokens, &find);
136 if (pt) {
137 os_ref_retain(&pt->pt_refcnt);
138 }
139 return pt;
140 }
141
142 int
protons_token_get_use_count(struct protons_token * pt)143 protons_token_get_use_count(struct protons_token *pt)
144 {
145 /* minus one refcnt in RB tree*/
146 return os_ref_get_count(&pt->pt_refcnt) - 1;
147 }
148
149 static void
protons_token_release(struct protons_token * pt)150 protons_token_release(struct protons_token *pt)
151 {
152 os_ref_count_t refcnt = os_ref_release(&pt->pt_refcnt);
153
154 SK_DF(SK_VERB_NS_PROTO,
155 "token %p proto %u released by pid %d epid %d, curr use %u",
156 (void *)SK_KVA(pt), pt->pt_protocol, pt->pt_pid, pt->pt_epid,
157 protons_token_get_use_count(pt));
158
159 if (refcnt == 1) {
160 RB_REMOVE(protons_token_tree, &protons_tokens, pt);
161 (void) os_ref_release(&pt->pt_refcnt);
162 pt->pt_flags &= ~PROTONSF_VALID;
163 pt->pt_protocol = 0;
164 pt->pt_pid = 0;
165 pt->pt_epid = 0;
166 protons_token_free(pt);
167 }
168 }
169
170 static int
protons_reserve_locked(struct protons_token ** ptp,pid_t pid,pid_t epid,uint8_t proto)171 protons_reserve_locked(struct protons_token **ptp, pid_t pid, pid_t epid,
172 uint8_t proto)
173 {
174 struct protons_token *pt = NULL, *dup = NULL;
175 *ptp = NULL;
176
177 pt = protons_find_token_with_protocol(proto);
178 if (pt != NULL) {
179 /* use previously reserved token with same process */
180 ASSERT(pt->pt_flags & PROTONSF_VALID);
181 if (pt->pt_pid != pid || pt->pt_epid != epid) {
182 SK_ERR("proto %u existed with pid %d epid %d",
183 proto, pt->pt_pid, pt->pt_epid);
184 (void) os_ref_release(&pt->pt_refcnt);
185 pt = NULL;
186 return EEXIST;
187 }
188 } else {
189 /* start with new token */
190 pt = protons_token_alloc(true);
191 if (pt == NULL) {
192 return ENOMEM;
193 }
194
195 os_ref_retain(&pt->pt_refcnt);
196 pt->pt_flags |= PROTONSF_VALID;
197 pt->pt_pid = pid;
198 pt->pt_epid = (epid != -1) ? epid : pid;
199 pt->pt_protocol = proto;
200 dup = RB_INSERT(protons_token_tree, &protons_tokens, pt);
201 ASSERT(dup == NULL);
202 }
203
204 SK_DF(SK_VERB_NS_PROTO,
205 "token %p proto %u reserved by pid %d epid %d, curr use %u",
206 (void *)SK_KVA(pt), proto, pid, epid, protons_token_get_use_count(pt));
207 *ptp = pt;
208
209 return 0;
210 }
211
212 int
protons_reserve(struct protons_token ** ptp,pid_t pid,pid_t epid,uint8_t proto)213 protons_reserve(struct protons_token **ptp, pid_t pid, pid_t epid,
214 uint8_t proto)
215 {
216 int err = 0;
217 PROTONS_LOCK();
218 err = protons_reserve_locked(ptp, pid, epid, proto);
219 PROTONS_UNLOCK();
220 return err;
221 }
222
223 void
protons_release(struct protons_token ** ptp)224 protons_release(struct protons_token **ptp)
225 {
226 struct protons_token *pt = *ptp;
227 ASSERT(pt != NULL);
228
229 PROTONS_LOCK();
230 protons_token_release(pt);
231 PROTONS_UNLOCK();
232 *ptp = NULL;
233 }
234
235 /* Reserved all protocol used by BSD stack. */
236 static void
protons_init_netinet_protocol(void)237 protons_init_netinet_protocol(void)
238 {
239 PROTONS_LOCK();
240
241 uint8_t proto = 0;
242 struct protons_token *pt = NULL;
243 int error = 0;
244 struct protosw *pp = NULL;
245 TAILQ_FOREACH(pp, &inetdomain->dom_protosw, pr_entry) {
246 pt = NULL;
247 proto = (uint8_t)pp->pr_protocol;
248 error = protons_reserve_locked(&pt, 0, 0, proto);
249 VERIFY(error == 0 || error == EEXIST);
250 VERIFY(pt != NULL);
251 }
252
253 TAILQ_FOREACH(pp, &inet6domain->dom_protosw, pr_entry) {
254 pt = NULL;
255 proto = (uint8_t)pp->pr_protocol;
256 error = protons_reserve_locked(&pt, 0, 0, proto);
257 VERIFY(error == 0 || error == EEXIST);
258 VERIFY(pt != NULL);
259 }
260
261 PROTONS_UNLOCK();
262 }
263
264 int
protons_init(void)265 protons_init(void)
266 {
267 VERIFY(__protons_inited == 0);
268
269 RB_INIT(&protons_tokens);
270
271 protons_init_netinet_protocol();
272
273 __protons_inited = 1;
274 sk_features |= SK_FEATURE_PROTONS;
275
276 SK_D("initialized protons");
277
278 return 0;
279 }
280
281 void
protons_fini(void)282 protons_fini(void)
283 {
284 if (__protons_inited == 1) {
285 __protons_inited = 0;
286 sk_features &= ~SK_FEATURE_PROTONS;
287 }
288 }
289
290 static int protons_stats_sysctl SYSCTL_HANDLER_ARGS;
291 SYSCTL_PROC(_kern_skywalk_stats, OID_AUTO, protons,
292 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
293 0, 0, protons_stats_sysctl, "", "");
294
295 static int
296 protons_stats_sysctl SYSCTL_HANDLER_ARGS
297 {
298 #pragma unused(arg1, arg2, oidp)
299 int error = 0;
300 size_t actual_space;
301 caddr_t buffer = NULL;
302 size_t buffer_space;
303 size_t allocated_space;
304 int out_error;
305 caddr_t scan;
306
307 if (!kauth_cred_issuser(kauth_cred_get())) {
308 return EPERM;
309 }
310
311 net_update_uptime();
312 buffer_space = req->oldlen;
313 if (req->oldptr != USER_ADDR_NULL && buffer_space != 0) {
314 if (buffer_space > SK_SYSCTL_ALLOC_MAX) {
315 buffer_space = SK_SYSCTL_ALLOC_MAX;
316 }
317 allocated_space = buffer_space;
318 buffer = sk_alloc_data(allocated_space, Z_WAITOK, skmem_tag_sysctl_buf);
319 if (__improbable(buffer == NULL)) {
320 return ENOBUFS;
321 }
322 } else if (req->oldptr == USER_ADDR_NULL) {
323 buffer_space = 0;
324 }
325 actual_space = 0;
326 scan = buffer;
327
328 struct sk_stats_protons_token *spt = (void *)scan;
329 size_t spt_size = sizeof(*spt);
330 struct protons_token *pt = NULL;
331 PROTONS_LOCK();
332 RB_FOREACH(pt, protons_token_tree, &protons_tokens) {
333 if (scan != NULL) {
334 if (buffer_space < spt_size) {
335 /* supplied buffer too small, stop copying */
336 error = ENOMEM;
337 break;
338 }
339 spt->spt_protocol = pt->pt_protocol;
340 spt->spt_refcnt = protons_token_get_use_count(pt);
341 spt->spt_pid = pt->pt_pid;
342 spt->spt_epid = pt->pt_epid;
343 spt++;
344 buffer_space -= spt_size;
345 }
346 actual_space += spt_size;
347 }
348 PROTONS_UNLOCK();
349
350 if (actual_space != 0) {
351 out_error = SYSCTL_OUT(req, buffer, actual_space);
352 if (out_error != 0) {
353 error = out_error;
354 }
355 }
356 if (buffer != NULL) {
357 sk_free_data(buffer, allocated_space);
358 }
359 return error;
360 }
361