1 /*
2 * Copyright (c) 2016-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 #include <sys/cdefs.h>
25 #include <sys/systm.h>
26 #include <sys/param.h>
27 #include <sys/types.h>
28 #include <sys/mcache.h>
29 #include <kern/kern_types.h>
30 #include <net/pf_pbuf.h>
31 #include <net/pfvar.h>
32 #include <netinet/in.h>
33
34 void
pbuf_init_mbuf(pbuf_t * pbuf,struct mbuf * m,struct ifnet * ifp)35 pbuf_init_mbuf(pbuf_t *pbuf, struct mbuf *m, struct ifnet *ifp)
36 {
37 VERIFY((m->m_flags & M_PKTHDR) != 0);
38
39 pbuf->pb_type = PBUF_TYPE_MBUF;
40 pbuf->pb_mbuf = m;
41 pbuf->pb_ifp = ifp;
42 pbuf->pb_next = NULL;
43 pbuf_sync(pbuf);
44 }
45
46 void
pbuf_init_memory(pbuf_t * pbuf,const struct pbuf_memory * mp,struct ifnet * ifp)47 pbuf_init_memory(pbuf_t *pbuf, const struct pbuf_memory *mp, struct ifnet *ifp)
48 {
49 pbuf->pb_type = PBUF_TYPE_MEMORY;
50 pbuf->pb_memory = *mp;
51 pbuf->pb_ifp = ifp;
52 pbuf->pb_next = NULL;
53 pbuf_sync(pbuf);
54 }
55
56 void
pbuf_destroy(pbuf_t * pbuf)57 pbuf_destroy(pbuf_t *pbuf)
58 {
59 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
60 if (pbuf->pb_mbuf) {
61 m_freem(pbuf->pb_mbuf);
62 pbuf->pb_mbuf = NULL;
63 }
64 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
65 VERIFY(pbuf->pb_memory.pm_buffer != NULL);
66 (void) (pbuf->pb_memory.pm_action)(&pbuf->pb_memory,
67 PBUF_ACTION_DESTROY);
68 } else {
69 VERIFY(pbuf->pb_type == PBUF_TYPE_ZOMBIE);
70 }
71
72 memset(pbuf, 0, sizeof(*pbuf));
73 }
74
75 void
pbuf_sync(pbuf_t * pbuf)76 pbuf_sync(pbuf_t *pbuf)
77 {
78 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
79 struct mbuf *m = pbuf->pb_mbuf;
80
81 VERIFY(m != NULL);
82 VERIFY(m->m_flags & M_PKTHDR);
83
84 pbuf->pb_data = mtod(m, void *);
85 pbuf->pb_packet_len = m->m_pkthdr.len;
86 pbuf->pb_contig_len = m->m_len;
87 pbuf->pb_csum_flags = &m->m_pkthdr.csum_flags;
88 pbuf->pb_csum_data = &m->m_pkthdr.csum_data;
89 pbuf->pb_proto = &m->m_pkthdr.pkt_proto;
90 pbuf->pb_flowsrc = &m->m_pkthdr.pkt_flowsrc;
91 pbuf->pb_flowid = &m->m_pkthdr.pkt_flowid;
92 pbuf->pb_flags = &m->m_pkthdr.pkt_flags;
93 pbuf->pb_pftag = m_pftag(m);
94 pbuf->pb_pf_fragtag = pf_find_fragment_tag(m);
95 ASSERT((pbuf->pb_pf_fragtag == NULL) ||
96 (pbuf->pb_pftag->pftag_flags & PF_TAG_REASSEMBLED));
97 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
98 struct pbuf_memory *nm = &pbuf->pb_memory;
99
100 VERIFY(nm->pm_buffer != NULL);
101 VERIFY(nm->pm_buffer_len != 0);
102 VERIFY(nm->pm_len != 0);
103 VERIFY(nm->pm_len <= nm->pm_buffer_len);
104 VERIFY(nm->pm_offset < nm->pm_len);
105
106 pbuf->pb_data = &nm->pm_buffer[nm->pm_offset];
107 pbuf->pb_packet_len = nm->pm_len;
108 pbuf->pb_contig_len = nm->pm_len;
109 pbuf->pb_csum_flags = &nm->pm_csum_flags;
110 pbuf->pb_csum_data = &nm->pm_csum_data;
111 pbuf->pb_proto = &nm->pm_proto;
112 pbuf->pb_flowsrc = &nm->pm_flowsrc;
113 pbuf->pb_flowid = &nm->pm_flowid;
114 pbuf->pb_flags = &nm->pm_flags;
115 pbuf->pb_pftag = &nm->pm_pftag;
116 pbuf->pb_pf_fragtag = &nm->pm_pf_fragtag;
117 } else {
118 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
119 }
120 }
121
122 struct mbuf *
pbuf_to_mbuf(pbuf_t * pbuf,boolean_t release_ptr)123 pbuf_to_mbuf(pbuf_t *pbuf, boolean_t release_ptr)
124 {
125 struct mbuf *m = NULL;
126
127 pbuf_sync(pbuf);
128
129 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
130 m = pbuf->pb_mbuf;
131 if (release_ptr) {
132 pbuf->pb_mbuf = NULL;
133 }
134 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
135 boolean_t fragtag = FALSE;
136
137 if (pbuf->pb_packet_len > (u_int)MHLEN) {
138 if (pbuf->pb_packet_len > (u_int)MCLBYTES) {
139 printf("%s: packet too big for cluster (%u)\n",
140 __func__, pbuf->pb_packet_len);
141 return NULL;
142 }
143 m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
144 } else {
145 m = m_gethdr(M_DONTWAIT, MT_DATA);
146 }
147 if (m == NULL) {
148 goto done;
149 }
150
151 m_copyback(m, 0, pbuf->pb_packet_len, pbuf->pb_data);
152 m->m_pkthdr.csum_flags = *pbuf->pb_csum_flags;
153 m->m_pkthdr.csum_data = *pbuf->pb_csum_data;
154 m->m_pkthdr.pkt_proto = *pbuf->pb_proto;
155 m->m_pkthdr.pkt_flowsrc = *pbuf->pb_flowsrc;
156 m->m_pkthdr.pkt_flowid = *pbuf->pb_flowid;
157 m->m_pkthdr.pkt_flags = *pbuf->pb_flags;
158
159 if (pbuf->pb_pftag != NULL) {
160 struct pf_mtag *pftag = m_pftag(m);
161
162 ASSERT(pftag != NULL);
163 *pftag = *pbuf->pb_pftag;
164 fragtag =
165 ((pftag->pftag_flags & PF_TAG_REASSEMBLED) != 0);
166 }
167
168 if (fragtag && pbuf->pb_pf_fragtag != NULL) {
169 if (pf_copy_fragment_tag(m, pbuf->pb_pf_fragtag,
170 M_NOWAIT) == NULL) {
171 m_freem(m);
172 m = NULL;
173 goto done;
174 }
175 }
176 }
177
178 done:
179 if (release_ptr) {
180 pbuf_destroy(pbuf);
181 }
182 return m;
183 }
184
185 struct mbuf *
pbuf_clone_to_mbuf(pbuf_t * pbuf)186 pbuf_clone_to_mbuf(pbuf_t *pbuf)
187 {
188 struct mbuf *m = NULL;
189
190 pbuf_sync(pbuf);
191
192 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
193 m = m_copy(pbuf->pb_mbuf, 0, M_COPYALL);
194 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
195 m = pbuf_to_mbuf(pbuf, FALSE);
196 } else {
197 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
198 }
199
200 return m;
201 }
202
203 void *
pbuf_ensure_writable(pbuf_t * pbuf,size_t len)204 pbuf_ensure_writable(pbuf_t *pbuf, size_t len)
205 {
206 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
207 struct mbuf *m = pbuf->pb_mbuf;
208
209 if (m_makewritable(&pbuf->pb_mbuf, 0, len, M_DONTWAIT)) {
210 return NULL;
211 }
212
213 if (pbuf->pb_mbuf == NULL) {
214 pbuf_destroy(pbuf);
215 return NULL;
216 }
217
218 if (m != pbuf->pb_mbuf) {
219 pbuf_sync(pbuf);
220 }
221 } else if (pbuf->pb_type != PBUF_TYPE_MEMORY) {
222 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
223 }
224
225 return pbuf->pb_data;
226 }
227
228 void *
pbuf_resize_segment(pbuf_t * pbuf,int off,int olen,int nlen)229 pbuf_resize_segment(pbuf_t *pbuf, int off, int olen, int nlen)
230 {
231 void *rv = NULL;
232
233 VERIFY(off >= 0);
234 VERIFY((u_int)off <= pbuf->pb_packet_len);
235
236 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
237 struct mbuf *m, *n;
238
239 VERIFY(pbuf->pb_mbuf != NULL);
240
241 m = pbuf->pb_mbuf;
242
243 if (off > 0) {
244 /* Split the mbuf chain at the specified boundary */
245 if ((n = m_split(m, off, M_DONTWAIT)) == NULL) {
246 return NULL;
247 }
248 } else {
249 n = m;
250 }
251
252 /* Trim old length */
253 m_adj(n, olen);
254
255 /* Prepend new length */
256 if (M_PREPEND(n, nlen, M_DONTWAIT, 0) == NULL) {
257 /* mbuf is freed by M_PREPEND in this case */
258 pbuf->pb_mbuf = NULL;
259 pbuf_destroy(pbuf);
260 return NULL;
261 }
262
263 rv = mtod(n, void *);
264
265 if (off > 0) {
266 /* Merge the two chains */
267 int mlen;
268
269 mlen = n->m_pkthdr.len;
270 m_cat(m, n);
271 m->m_pkthdr.len += mlen;
272 } else {
273 /* The new mbuf becomes the packet header */
274 pbuf->pb_mbuf = n;
275 }
276
277 pbuf_sync(pbuf);
278 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
279 struct pbuf_memory *nm = &pbuf->pb_memory;
280 u_int true_offset, move_len;
281 int delta_len;
282 uint8_t *psrc, *pdst;
283
284 delta_len = nlen - olen;
285 VERIFY(nm->pm_offset + (nm->pm_len + delta_len) <=
286 nm->pm_buffer_len);
287
288 true_offset = (u_int)off + nm->pm_offset;
289 rv = &nm->pm_buffer[true_offset];
290 psrc = &nm->pm_buffer[true_offset + (u_int)olen];
291 pdst = &nm->pm_buffer[true_offset + (u_int)nlen];
292 move_len = pbuf->pb_packet_len - ((u_int)off + olen);
293 memmove(pdst, psrc, move_len);
294
295 nm->pm_len += delta_len;
296
297 VERIFY((nm->pm_len + nm->pm_offset) <= nm->pm_buffer_len);
298
299 pbuf_sync(pbuf);
300 } else {
301 panic("pbuf_csum_flags_get: bad pb_type: %d", pbuf->pb_type);
302 }
303 return rv;
304 }
305
306 void *
pbuf_contig_segment(pbuf_t * pbuf,int off,int len)307 pbuf_contig_segment(pbuf_t *pbuf, int off, int len)
308 {
309 void *rv = NULL;
310
311 VERIFY(off >= 0);
312 VERIFY(len >= 0);
313 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
314
315 /*
316 * Note: If this fails, then the pbuf is destroyed. This is a
317 * side-effect of m_pulldown().
318 *
319 * PF expects this behaviour so it's not a real problem.
320 */
321 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
322 struct mbuf *n;
323 int moff;
324
325 n = m_pulldown(pbuf->pb_mbuf, off, len, &moff);
326 if (n == NULL) {
327 /* mbuf is freed by m_pulldown() in this case */
328 pbuf->pb_mbuf = NULL;
329 pbuf_destroy(pbuf);
330 return NULL;
331 }
332
333 pbuf_sync(pbuf);
334
335 rv = (void *)(mtod(n, uint8_t *) + moff);
336 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
337 /*
338 * This always succeeds since memory pbufs are fully contig.
339 */
340 rv = (void *)(uintptr_t)(((uint8_t *)pbuf->pb_data)[off]);
341 } else {
342 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
343 }
344
345 return rv;
346 }
347
348 void
pbuf_copy_back(pbuf_t * pbuf,int off,int len,void * src)349 pbuf_copy_back(pbuf_t *pbuf, int off, int len, void *src)
350 {
351 VERIFY(off >= 0);
352 VERIFY(len >= 0);
353 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
354
355 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
356 m_copyback(pbuf->pb_mbuf, off, len, src);
357 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
358 if (len) {
359 memcpy(&((uint8_t *)pbuf->pb_data)[off], src, len);
360 }
361 } else {
362 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
363 }
364 }
365
366 void
pbuf_copy_data(pbuf_t * pbuf,int off,int len,void * dst)367 pbuf_copy_data(pbuf_t *pbuf, int off, int len, void *dst)
368 {
369 VERIFY(off >= 0);
370 VERIFY(len >= 0);
371 VERIFY((u_int)(off + len) <= pbuf->pb_packet_len);
372
373 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
374 m_copydata(pbuf->pb_mbuf, off, len, dst);
375 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
376 if (len) {
377 memcpy(dst, &((uint8_t *)pbuf->pb_data)[off], len);
378 }
379 } else {
380 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
381 }
382 }
383
384 uint16_t
pbuf_inet_cksum(const pbuf_t * pbuf,uint32_t nxt,uint32_t off,uint32_t len)385 pbuf_inet_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
386 {
387 uint16_t sum = 0;
388
389 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
390 sum = inet_cksum(pbuf->pb_mbuf, nxt, off, len);
391 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
392 sum = inet_cksum_buffer(pbuf->pb_data, nxt, off, len);
393 } else {
394 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
395 }
396
397 return sum;
398 }
399
400 uint16_t
pbuf_inet6_cksum(const pbuf_t * pbuf,uint32_t nxt,uint32_t off,uint32_t len)401 pbuf_inet6_cksum(const pbuf_t *pbuf, uint32_t nxt, uint32_t off, uint32_t len)
402 {
403 uint16_t sum = 0;
404
405 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
406 sum = inet6_cksum(pbuf->pb_mbuf, nxt, off, len);
407 } else if (pbuf->pb_type == PBUF_TYPE_MEMORY) {
408 sum = inet6_cksum_buffer(pbuf->pb_data, nxt, off, len);
409 } else {
410 panic("%s: bad pb_type: %d", __func__, pbuf->pb_type);
411 }
412
413 return sum;
414 }
415
416 mbuf_svc_class_t
pbuf_get_service_class(const pbuf_t * pbuf)417 pbuf_get_service_class(const pbuf_t *pbuf)
418 {
419 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
420 return m_get_service_class(pbuf->pb_mbuf);
421 }
422
423 VERIFY(pbuf->pb_type == PBUF_TYPE_MEMORY);
424
425 return MBUF_SC_BE;
426 }
427
428 void *
pbuf_get_packet_buffer_address(const pbuf_t * pbuf)429 pbuf_get_packet_buffer_address(const pbuf_t *pbuf)
430 {
431 VERIFY(pbuf != NULL);
432
433 if (pbuf->pb_type == PBUF_TYPE_MBUF) {
434 return pbuf->pb_mbuf;
435 } else {
436 return pbuf->pb_memory.pm_buffer;
437 }
438 }
439