1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 #include <sys/cdefs.h>
25 #include <sys/systm.h>
26 #include <sys/param.h>
27 #include <sys/types.h>
28 #include <sys/mcache.h>
29 #include <kern/kern_types.h>
30 #include <net/pf_pbuf.h>
31 #include <net/pfvar.h>
32 #include <netinet/in.h>
33
34 void
pbuf_init_mbuf(pbuf_t * pbuf,struct mbuf * m,struct ifnet * ifp)35 pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp)
36 {
37 VERIFY((m->m_flags & M_PKTHDR) != 0);
38
39 pbuf->pb_type = PBUF_TYPE_MBUF;
40 pbuf->pb_mbuf = m;
41 pbuf->pb_ifp = ifp;
42 pbuf->pb_next = NULL;
43 pbuf_sync(pbuf);
44 }
45
46 void
pbuf_init_memory(pbuf_t * pbuf,const struct pbuf_memory * mp,struct ifnet * ifp)47 pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp)
48 {
49 pbuf->pb_type = PBUF_TYPE_MEMORY;
50 pbuf->pb_memory = *mp;
51 pbuf->pb_ifp = ifp;
52 pbuf->pb_next = NULL;
53 pbuf_sync(pbuf);
54 }
55
56 void
pbuf_destroy(pbuf_t * pbuf)57 pbuf_destroy(pbuf_t *pbuf)
58 {
59 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
60 if (pbuf->pb_mbuf) {
61 m_freem(pbuf->pb_mbuf);
62 pbuf->pb_mbuf = NULL;
63 }
64 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
65 VERIFY(pbuf->pb_memory.pm_buffer != NULL);
66 (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
67 PBUF_ACTION_DESTROY);
68 } else {
69 VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
70 }
71
72 memset(pbuf, 0, sizeof(*pbuf));
73 }
74
75 void
pbuf_sync(pbuf_t * pbuf)76 pbuf_sync(pbuf_t *pbuf)
77 {
78 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
79 struct mbuf *m = pbuf->pb_mbuf;
80
81 VERIFY(m != NULL);
82 VERIFY(m->m_flags & M_PKTHDR);
83
84 pbuf->pb_data = mtod(m, void *);
85 pbuf->pb_packet_len = m->m_pkthdr.len;
86 pbuf->pb_contig_len = m->m_len;
87 pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
88 pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
89 pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
90 pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
91 pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
92 pbuf->pb_flow_gencnt = &m->m_pkthdr.comp_gencnt;
93 pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
94 pbuf->pb_pftag = m_pftag(m);
95 pbuf->pb_pf_fragtag = pf_find_fragment_tag(m);
96 ASSERT((pbuf->pb_pf_fragtag == NULL) ||
97 (pbuf->pb_pftag->pftag_flags & PF_TAG_REASSEMBLED));
98 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
99 struct pbuf_memory *nm = &pbuf->pb_memory;
100
101 VERIFY(nm->pm_buffer != NULL);
102 VERIFY(nm->pm_buffer_len != 0);
103 VERIFY(nm->pm_len != 0);
104 VERIFY(nm->pm_len <= nm->pm_buffer_len);
105 VERIFY(nm->pm_offset < nm->pm_len);
106
107 pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
108 pbuf->pb_packet_len = nm->pm_len;
109 pbuf->pb_contig_len = nm->pm_len;
110 pbuf->pb_csum_flags = &nm->pm_csum_flags;
111 pbuf->pb_csum_data = &nm->pm_csum_data;
112 pbuf->pb_proto = &nm->pm_proto;
113 pbuf->pb_flowsrc = &nm->pm_flowsrc;
114 pbuf->pb_flowid = &nm->pm_flowid;
115 pbuf->pb_flow_gencnt = &nm->pm_flow_gencnt;
116 pbuf->pb_flags = &nm->pm_flags;
117 pbuf->pb_pftag = &nm->pm_pftag;
118 pbuf->pb_pf_fragtag = &nm->pm_pf_fragtag;
119 } else {
120 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
121 }
122 }
123
124 struct mbuf *
pbuf_to_mbuf(pbuf_t * pbuf,boolean_t release_ptr)125 pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
126 {
127 struct mbuf *m = NULL;
128
129 pbuf_sync(pbuf);
130
131 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
132 m = pbuf->pb_mbuf;
133 if (release_ptr) {
134 pbuf->pb_mbuf = NULL;
135 }
136 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
137 boolean_t fragtag = FALSE;
138
139 if (pbuf->pb_packet_len > (u_int)MHLEN) {
140 if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
141 printf("%s: packet too big for cluster (%u)\n",
142 __func__, pbuf->pb_packet_len);
143 return NULL;
144 }
145 m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
146 } else {
147 m = m_gethdr(M_DONTWAIT, MT_DATA);
148 }
149 if (m == NULL) {
150 goto done;
151 }
152
153 m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
154 m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
155 m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
156 m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
157 m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
158 m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
159 m->m_pkthdr.comp_gencnt = *pbuf->pb_flow_gencnt;
160 m->m_pkthdr.pkt_flags = *pbuf->pb_flags;
161
162 if (pbuf->pb_pftag != NULL) {
163 struct pf_mtag *pftag = m_pftag(m);
164
165 ASSERT(pftag != NULL);
166 *pftag = *pbuf->pb_pftag;
167 fragtag =
168 ((pftag->pftag_flags & PF_TAG_REASSEMBLED) != 0);
169 }
170
171 if (fragtag && pbuf->pb_pf_fragtag != NULL) {
172 if (pf_copy_fragment_tag(m, pbuf->pb_pf_fragtag,
173 M_NOWAIT) == NULL) {
174 m_freem(m);
175 m = NULL;
176 goto done;
177 }
178 }
179 }
180
181 done:
182 if (release_ptr) {
183 pbuf_destroy(pbuf);
184 }
185 return m;
186 }
187
188 struct mbuf *
pbuf_clone_to_mbuf(pbuf_t * pbuf)189 pbuf_clone_to_mbuf(pbuf_t *pbuf)
190 {
191 struct mbuf *m = NULL;
192
193 pbuf_sync(pbuf);
194
195 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
196 m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
197 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
198 m = pbuf_to_mbuf(pbuf, FALSE);
199 } else {
200 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
201 }
202
203 return m;
204 }
205
206 void *
pbuf_ensure_writable(pbuf_t * pbuf,size_t len)207 pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
208 {
209 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
210 struct mbuf *m = pbuf->pb_mbuf;
211
212 if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) {
213 return NULL;
214 }
215
216 if (pbuf->pb_mbuf == NULL) {
217 pbuf_destroy(pbuf);
218 return NULL;
219 }
220
221 if (m != pbuf->pb_mbuf) {
222 pbuf_sync(pbuf);
223 }
224 } else if (pbuf->pb_type != PBUF_TYPE_MEMORY) {
225 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
226 }
227
228 return pbuf->pb_data;
229 }
230
231 void *
232 __attribute__((warn_unused_result))
pbuf_resize_segment(pbuf_t * pbuf,int off,int olen,int nlen)233 pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
234 {
235 void *rv = NULL;
236
237 VERIFY(off >= 0);
238
239 /*
240 * Gracefully handle the case where `pbuf'
241 * does not have sufficient data
242 * for the requested `off'/`olen' combination.
243 */
244 if ((u_int)(off + olen) > pbuf->pb_packet_len) {
245 return NULL;
246 }
247
248 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
249 struct mbuf *m, *n;
250
251 VERIFY(pbuf->pb_mbuf != NULL);
252
253 m = pbuf->pb_mbuf;
254
255 if (off > 0) {
256 /* Split the mbuf chain at the specified boundary */
257 if ((n = m_split(m, off, M_DONTWAIT)) == NULL) {
258 return NULL;
259 }
260 } else {
261 n = m;
262 }
263
264 /* Trim old length */
265 m_adj(n, olen);
266
267 /* Prepend new length */
268 if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) {
269 /* mbuf is freed by M_PREPEND in this case */
270 pbuf->pb_mbuf = NULL;
271 pbuf_destroy(pbuf);
272 return NULL;
273 }
274
275 if (off > 0) {
276 /* Merge the two chains */
277 int mlen;
278 struct mbuf *new_n;
279 int new_off = 0;
280
281 mlen = n->m_pkthdr.len;
282 m_cat(m, n);
283 m->m_pkthdr.len += mlen;
284
285 new_n = m_getptr(m, off, &new_off);
286 rv = mtod(new_n, uint8_t *) + new_off;
287 } else {
288 /* The new mbuf becomes the packet header */
289 pbuf->pb_mbuf = n;
290
291 rv = mtod(n, void *);
292 }
293
294 pbuf_sync(pbuf);
295 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
296 struct pbuf_memory *nm = &pbuf->pb_memory;
297 u_int true_offset, move_len;
298 int delta_len;
299 uint8_t *psrc, *pdst;
300
301 delta_len = nlen - olen;
302 VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
303 nm->pm_buffer_len);
304
305 true_offset = (u_int)off + nm->pm_offset;
306 rv = &nm->pm_buffer[true_offset];
307 psrc = &nm->pm_buffer[true_offset + (u_int)olen];
308 pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
309 move_len = pbuf->pb_packet_len - ((u_int)off + olen);
310 memmove(pdst, psrc, move_len);
311
312 nm->pm_len += delta_len;
313
314 VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);
315
316 pbuf_sync(pbuf);
317 } else {
318 panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
319 }
320 return rv;
321 }
322
323 void *
324 __attribute__((warn_unused_result))
pbuf_contig_segment(pbuf_t * pbuf,int off,int len)325 pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
326 {
327 void *__single rv = NULL;
328
329 VERIFY(off >= 0);
330 VERIFY(len >= 0);
331
332 /*
333 * Gracefully handle the case where `pbuf'
334 * does not have sufficient data
335 * for the requested `off'/`len' combination.
336 */
337 if ((u_int)(off + len) > pbuf->pb_packet_len) {
338 return NULL;
339 }
340
341 /*
342 * Note: If this fails, then the pbuf is destroyed. This is a
343 * side-effect of m_pulldown().
344 *
345 * PF expects this behaviour so it's not a real problem.
346 */
347 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
348 struct mbuf *__single n;
349 int moff;
350
351 n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
352 if (n == NULL) {
353 /* mbuf is freed by m_pulldown() in this case */
354 pbuf->pb_mbuf = NULL;
355 pbuf_destroy(pbuf);
356 return NULL;
357 }
358
359 pbuf_sync(pbuf);
360
361 rv = (void *__single)(mtod(n, uint8_t *) + moff);
362 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
363 /*
364 * This always succeeds since memory pbufs are fully contig.
365 */
366 rv = (void *__single)(((uint8_t *)pbuf->pb_data) + off);
367 } else {
368 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
369 }
370
371 return rv;
372 }
373
374 void
pbuf_copy_back(pbuf_t * pbuf,int off,int len,void * __sized_by (src_buflen)src,size_t src_buflen)375 pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *__sized_by(src_buflen)src, size_t src_buflen)
376 {
377 VERIFY(off >= 0);
378 VERIFY(len >= 0);
379 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
380 VERIFY((size_t)len <= src_buflen);
381
382 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
383 m_copyback(pbuf->pb_mbuf, off, len, src);
384 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
385 if (len) {
386 memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
387 }
388 } else {
389 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
390 }
391 }
392
393 void
pbuf_copy_data(pbuf_t * pbuf,int off,int len,void * __sized_by (dst_buflen)dst,size_t dst_buflen)394 pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *__sized_by(dst_buflen)dst, size_t dst_buflen)
395 {
396 VERIFY(off >= 0);
397 VERIFY(len >= 0);
398 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
399 VERIFY((size_t)len <= dst_buflen);
400
401 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
402 m_copydata(pbuf->pb_mbuf, off, len, dst);
403 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
404 if (len) {
405 memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
406 }
407 } else {
408 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
409 }
410 }
411
412 uint16_t
pbuf_inet_cksum(const pbuf_t * pbuf,uint32_t nxt,uint32_t off,uint32_t len)413 pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
414 {
415 uint16_t sum = 0;
416
417 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
418 sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
419 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
420 sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
421 } else {
422 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
423 }
424
425 return sum;
426 }
427
428 uint16_t
pbuf_inet6_cksum(const pbuf_t * pbuf,uint32_t nxt,uint32_t off,uint32_t len)429 pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
430 {
431 uint16_t sum = 0;
432
433 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
434 sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
435 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
436 sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len,
437 pbuf->pb_contig_len);
438 } else {
439 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
440 }
441
442 return sum;
443 }
444
445 mbuf_svc_class_t
pbuf_get_service_class(const pbuf_t * pbuf)446 pbuf_get_service_class(const pbuf_t *pbuf)
447 {
448 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
449 return m_get_service_class(pbuf->pb_mbuf);
450 }
451
452 VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);
453
454 return MBUF_SC_BE;
455 }
456
457 void *
pbuf_get_packet_buffer_address(const pbuf_t * pbuf)458 pbuf_get_packet_buffer_address(const pbuf_t *pbuf)
459 {
460 VERIFY(pbuf != NULL);
461
462 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
463 return pbuf->pb_mbuf;
464 } else {
465 return pbuf->pb_memory.pm_buffer;
466 }
467 }
468