1 /*
2 * Copyright (c) 2016-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 #include <sys/cdefs.h>
25 #include <sys/systm.h>
26 #include <sys/param.h>
27 #include <sys/types.h>
28 #include <sys/mcache.h>
29 #include <kern/kern_types.h>
30 #include <net/pf_pbuf.h>
31 #include <net/pfvar.h>
32 #include <netinet/in.h>
33
34 void
pbuf_init_mbuf(pbuf_t * pbuf,struct mbuf * m,struct ifnet * ifp)35 pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp)
36 {
37 VERIFY((m->m_flags & M_PKTHDR) != 0);
38
39 pbuf->pb_type = PBUF_TYPE_MBUF;
40 pbuf->pb_mbuf = m;
41 pbuf->pb_ifp = ifp;
42 pbuf->pb_next = NULL;
43 pbuf_sync(pbuf);
44 }
45
46 void
pbuf_init_memory(pbuf_t * pbuf,const struct pbuf_memory * mp,struct ifnet * ifp)47 pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp)
48 {
49 pbuf->pb_type = PBUF_TYPE_MEMORY;
50 pbuf->pb_memory = *mp;
51 pbuf->pb_ifp = ifp;
52 pbuf->pb_next = NULL;
53 pbuf_sync(pbuf);
54 }
55
56 void
pbuf_destroy(pbuf_t * pbuf)57 pbuf_destroy(pbuf_t *pbuf)
58 {
59 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
60 if (pbuf->pb_mbuf) {
61 m_freem(pbuf->pb_mbuf);
62 pbuf->pb_mbuf = NULL;
63 }
64 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
65 VERIFY(pbuf->pb_memory.pm_buffer != NULL);
66 (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
67 PBUF_ACTION_DESTROY);
68 } else {
69 VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
70 }
71
72 memset(pbuf, 0, sizeof(*pbuf));
73 }
74
75 void
pbuf_sync(pbuf_t * pbuf)76 pbuf_sync(pbuf_t *pbuf)
77 {
78 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
79 struct mbuf *m = pbuf->pb_mbuf;
80
81 VERIFY(m != NULL);
82 VERIFY(m->m_flags & M_PKTHDR);
83
84 pbuf->pb_data = mtod(m, void *);
85 pbuf->pb_packet_len = m->m_pkthdr.len;
86 pbuf->pb_contig_len = m->m_len;
87 pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
88 pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
89 pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
90 pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
91 pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
92 pbuf->pb_flow_gencnt = &m->m_pkthdr.comp_gencnt;
93 pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
94 pbuf->pb_pftag = m_pftag(m);
95 pbuf->pb_pf_fragtag = pf_find_fragment_tag(m);
96 ASSERT((pbuf->pb_pf_fragtag == NULL) ||
97 (pbuf->pb_pftag->pftag_flags & PF_TAG_REASSEMBLED));
98 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
99 struct pbuf_memory *nm = &pbuf->pb_memory;
100
101 VERIFY(nm->pm_buffer != NULL);
102 VERIFY(nm->pm_buffer_len != 0);
103 VERIFY(nm->pm_len != 0);
104 VERIFY(nm->pm_len <= nm->pm_buffer_len);
105 VERIFY(nm->pm_offset < nm->pm_len);
106
107 pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
108 pbuf->pb_packet_len = nm->pm_len;
109 pbuf->pb_contig_len = nm->pm_len;
110 pbuf->pb_csum_flags = &nm->pm_csum_flags;
111 pbuf->pb_csum_data = &nm->pm_csum_data;
112 pbuf->pb_proto = &nm->pm_proto;
113 pbuf->pb_flowsrc = &nm->pm_flowsrc;
114 pbuf->pb_flowid = &nm->pm_flowid;
115 pbuf->pb_flow_gencnt = &nm->pm_flow_gencnt;
116 pbuf->pb_flags = &nm->pm_flags;
117 pbuf->pb_pftag = &nm->pm_pftag;
118 pbuf->pb_pf_fragtag = &nm->pm_pf_fragtag;
119 } else {
120 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
121 }
122 }
123
124 struct mbuf *
pbuf_to_mbuf(pbuf_t * pbuf,boolean_t release_ptr)125 pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
126 {
127 struct mbuf *m = NULL;
128
129 pbuf_sync(pbuf);
130
131 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
132 m = pbuf->pb_mbuf;
133 if (release_ptr) {
134 pbuf->pb_mbuf = NULL;
135 }
136 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
137 boolean_t fragtag = FALSE;
138
139 if (pbuf->pb_packet_len > (u_int)MHLEN) {
140 if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
141 printf("%s: packet too big for cluster (%u)\n",
142 __func__, pbuf->pb_packet_len);
143 return NULL;
144 }
145 m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
146 } else {
147 m = m_gethdr(M_DONTWAIT, MT_DATA);
148 }
149 if (m == NULL) {
150 goto done;
151 }
152
153 m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
154 m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
155 m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
156 m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
157 m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
158 m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
159 m->m_pkthdr.comp_gencnt = *pbuf->pb_flow_gencnt;
160 m->m_pkthdr.pkt_flags = *pbuf->pb_flags;
161
162 if (pbuf->pb_pftag != NULL) {
163 struct pf_mtag *pftag = m_pftag(m);
164
165 ASSERT(pftag != NULL);
166 *pftag = *pbuf->pb_pftag;
167 fragtag =
168 ((pftag->pftag_flags & PF_TAG_REASSEMBLED) != 0);
169 }
170
171 if (fragtag && pbuf->pb_pf_fragtag != NULL) {
172 if (pf_copy_fragment_tag(m, pbuf->pb_pf_fragtag,
173 M_NOWAIT) == NULL) {
174 m_freem(m);
175 m = NULL;
176 goto done;
177 }
178 }
179 }
180
181 done:
182 if (release_ptr) {
183 pbuf_destroy(pbuf);
184 }
185 return m;
186 }
187
188 struct mbuf *
pbuf_clone_to_mbuf(pbuf_t * pbuf)189 pbuf_clone_to_mbuf(pbuf_t *pbuf)
190 {
191 struct mbuf *m = NULL;
192
193 pbuf_sync(pbuf);
194
195 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
196 m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
197 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
198 m = pbuf_to_mbuf(pbuf, FALSE);
199 } else {
200 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
201 }
202
203 return m;
204 }
205
206 void *
pbuf_ensure_writable(pbuf_t * pbuf,size_t len)207 pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
208 {
209 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
210 struct mbuf *m = pbuf->pb_mbuf;
211
212 if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) {
213 return NULL;
214 }
215
216 if (pbuf->pb_mbuf == NULL) {
217 pbuf_destroy(pbuf);
218 return NULL;
219 }
220
221 if (m != pbuf->pb_mbuf) {
222 pbuf_sync(pbuf);
223 }
224 } else if (pbuf->pb_type != PBUF_TYPE_MEMORY) {
225 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
226 }
227
228 return pbuf->pb_data;
229 }
230
231 void *
232 __attribute__((warn_unused_result))
pbuf_resize_segment(pbuf_t * pbuf,int off,int olen,int nlen)233 pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
234 {
235 void *rv = NULL;
236
237 VERIFY(off >= 0);
238
239 /*
240 * Gracefully handle the case where `pbuf'
241 * does not have sufficient data
242 * for the requested `off'/`olen' combination.
243 */
244 if ((u_int)(off + olen) > pbuf->pb_packet_len) {
245 return NULL;
246 }
247
248 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
249 struct mbuf *m, *n;
250
251 VERIFY(pbuf->pb_mbuf != NULL);
252
253 m = pbuf->pb_mbuf;
254
255 if (off > 0) {
256 /* Split the mbuf chain at the specified boundary */
257 if ((n = m_split(m, off, M_DONTWAIT)) == NULL) {
258 return NULL;
259 }
260 } else {
261 n = m;
262 }
263
264 /* Trim old length */
265 m_adj(n, olen);
266
267 /* Prepend new length */
268 if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) {
269 /* mbuf is freed by M_PREPEND in this case */
270 pbuf->pb_mbuf = NULL;
271 pbuf_destroy(pbuf);
272 return NULL;
273 }
274
275 rv = mtod(n, void *);
276
277 if (off > 0) {
278 /* Merge the two chains */
279 int mlen;
280
281 mlen = n->m_pkthdr.len;
282 m_cat(m, n);
283 m->m_pkthdr.len += mlen;
284 } else {
285 /* The new mbuf becomes the packet header */
286 pbuf->pb_mbuf = n;
287 }
288
289 pbuf_sync(pbuf);
290 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
291 struct pbuf_memory *nm = &pbuf->pb_memory;
292 u_int true_offset, move_len;
293 int delta_len;
294 uint8_t *psrc, *pdst;
295
296 delta_len = nlen - olen;
297 VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
298 nm->pm_buffer_len);
299
300 true_offset = (u_int)off + nm->pm_offset;
301 rv = &nm->pm_buffer[true_offset];
302 psrc = &nm->pm_buffer[true_offset + (u_int)olen];
303 pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
304 move_len = pbuf->pb_packet_len - ((u_int)off + olen);
305 memmove(pdst, psrc, move_len);
306
307 nm->pm_len += delta_len;
308
309 VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);
310
311 pbuf_sync(pbuf);
312 } else {
313 panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
314 }
315 return rv;
316 }
317
318 void *
319 __attribute__((warn_unused_result))
pbuf_contig_segment(pbuf_t * pbuf,int off,int len)320 pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
321 {
322 void *__single rv = NULL;
323
324 VERIFY(off >= 0);
325 VERIFY(len >= 0);
326
327 /*
328 * Gracefully handle the case where `pbuf'
329 * does not have sufficient data
330 * for the requested `off'/`len' combination.
331 */
332 if ((u_int)(off + len) > pbuf->pb_packet_len) {
333 return NULL;
334 }
335
336 /*
337 * Note: If this fails, then the pbuf is destroyed. This is a
338 * side-effect of m_pulldown().
339 *
340 * PF expects this behaviour so it's not a real problem.
341 */
342 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
343 struct mbuf *__single n;
344 int moff;
345
346 n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
347 if (n == NULL) {
348 /* mbuf is freed by m_pulldown() in this case */
349 pbuf->pb_mbuf = NULL;
350 pbuf_destroy(pbuf);
351 return NULL;
352 }
353
354 pbuf_sync(pbuf);
355
356 rv = (void *__single)(mtod(n, uint8_t *) + moff);
357 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
358 /*
359 * This always succeeds since memory pbufs are fully contig.
360 */
361 rv = (void *__single)(((uint8_t *)pbuf->pb_data) + off);
362 } else {
363 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
364 }
365
366 return rv;
367 }
368
369 void
pbuf_copy_back(pbuf_t * pbuf,int off,int len,void * __sized_by (src_buflen)src,size_t src_buflen)370 pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *__sized_by(src_buflen)src, size_t src_buflen)
371 {
372 VERIFY(off >= 0);
373 VERIFY(len >= 0);
374 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
375 VERIFY((size_t)len <= src_buflen);
376
377 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
378 m_copyback(pbuf->pb_mbuf, off, len, src);
379 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
380 if (len) {
381 memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
382 }
383 } else {
384 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
385 }
386 }
387
388 void
pbuf_copy_data(pbuf_t * pbuf,int off,int len,void * __sized_by (dst_buflen)dst,size_t dst_buflen)389 pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *__sized_by(dst_buflen)dst, size_t dst_buflen)
390 {
391 VERIFY(off >= 0);
392 VERIFY(len >= 0);
393 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
394 VERIFY((size_t)len <= dst_buflen);
395
396 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
397 m_copydata(pbuf->pb_mbuf, off, len, dst);
398 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
399 if (len) {
400 memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
401 }
402 } else {
403 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
404 }
405 }
406
407 uint16_t
pbuf_inet_cksum(const pbuf_t * pbuf,uint32_t nxt,uint32_t off,uint32_t len)408 pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
409 {
410 uint16_t sum = 0;
411
412 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
413 sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
414 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
415 sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
416 } else {
417 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
418 }
419
420 return sum;
421 }
422
423 uint16_t
pbuf_inet6_cksum(const pbuf_t * pbuf,uint32_t nxt,uint32_t off,uint32_t len)424 pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
425 {
426 uint16_t sum = 0;
427
428 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
429 sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
430 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
431 sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len,
432 pbuf->pb_contig_len);
433 } else {
434 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
435 }
436
437 return sum;
438 }
439
440 mbuf_svc_class_t
pbuf_get_service_class(const pbuf_t * pbuf)441 pbuf_get_service_class(const pbuf_t *pbuf)
442 {
443 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
444 return m_get_service_class(pbuf->pb_mbuf);
445 }
446
447 VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);
448
449 return MBUF_SC_BE;
450 }
451
452 void *
pbuf_get_packet_buffer_address(const pbuf_t * pbuf)453 pbuf_get_packet_buffer_address(const pbuf_t *pbuf)
454 {
455 VERIFY(pbuf != NULL);
456
457 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
458 return pbuf->pb_mbuf;
459 } else {
460 return pbuf->pb_memory.pm_buffer;
461 }
462 }
463