xref: /xnu-12377.61.12/tests/skywalk/skt_utunloop.c (revision 4d495c6e23c53686cf65f45067f79024cf5dcee8)
1 /*
2  * Copyright (c) 2017-2024 Apple Inc. All rights reserved.
3  *
4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5  *
6  * This file contains Original Code and/or Modifications of Original Code
7  * as defined in and that are subject to the Apple Public Source License
8  * Version 2.0 (the 'License'). You may not use this file except in
9  * compliance with the License. The rights granted to you under the License
10  * may not be used to create, or enable the creation or redistribution of,
11  * unlawful or unlicensed copies of an Apple operating system, or to
12  * circumvent, violate, or enable the circumvention or violation of, any
13  * terms of an Apple operating system software license agreement.
14  *
15  * Please obtain a copy of the License at
16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
17  *
18  * The Original Code and all software distributed under the License are
19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23  * Please see the License for the specific language governing rights and
24  * limitations under the License.
25  *
26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27  */
28 
29 /*
30  * <rdar://problem/31245751> User space forwarding for testing utun/ipsec
31  *
32  * A process that opens 2 channels, each one to a separate utun/ipsec interface
33  * The process would then shuttle packets from one to another.
34  *
35  */
36 
37 #include <assert.h>
38 #include <stdio.h>
39 #include <string.h>
40 #include <unistd.h>
41 #include <stdlib.h>
42 #include <errno.h>
43 #include <pthread.h>
44 #include <fcntl.h>
45 #include <mach/mach.h>
46 #include <mach/host_reboot.h>
47 
48 #include <uuid/uuid.h>
49 #include <sys/types.h>
50 #include <sys/event.h>
51 #include <net/if_utun.h>
52 #include <sys/ioctl.h>
53 #include <sys/socket.h>
54 #include <sys/kern_control.h>
55 #include <sys/reboot.h>
56 #include <sys/sys_domain.h>
57 #include <sys/sysctl.h>
58 
59 #include <arpa/inet.h> // for inet_ntop
60 
61 #include <skywalk/os_skywalk.h>
62 #include <darwintest.h>
63 
64 #include "skywalk_test_driver.h"
65 #include "skywalk_test_utils.h"
66 #include "skywalk_test_common.h"
67 
68 static volatile bool g_die;
69 
70 static volatile bool g_verbose;
71 
72 char debugcmd[1024];
73 
74 FILE *g_dumpfile;
75 
76 #define VLOG(_fmt, ...)            \
77 	do {                                               \
78 	        if (g_verbose) {                                   \
79 	                struct timeval _stamp, _delta;                  \
80 	                if (!g_dumpfile) g_dumpfile = stderr;                   \
81 	                gettimeofday(&_stamp, NULL);                            \
82 	                timersub(&_stamp, &prevtime, &_delta);                                    \
83 	                fprintf(g_dumpfile, "% 10ld.%06d % 10ld.%06d %s: " _fmt "\n", \
84 	                        _stamp.tv_sec, _stamp.tv_usec,                                                  \
85 	                        _delta.tv_sec, _delta.tv_usec, threadname, ##__VA_ARGS__); \
86 	                fflush(g_dumpfile); \
87 	                prevtime = _stamp;                                                                                      \
88 	        }                                                                                                                               \
89 	} while (0)
90 
91 static bool g_assert_stalls12;
92 static bool g_assert_stalls21;
93 
94 static void
skt_utunloop_xfer_slots(int kq,channel_t rxchannel,int rxindex,channel_t txchannel,int txindex,const char * threadname,bool xfer12)95 skt_utunloop_xfer_slots(int kq,
96     channel_t rxchannel, int rxindex,
97     channel_t txchannel, int txindex,
98     const char *threadname, bool xfer12)
99 {
100 	int error;
101 	channel_ring_t rxring, txring;
102 	struct kevent kev;
103 	int rxfd, txfd;
104 	bool rxenable, txenable;
105 	time_t start, then, now;
106 	uint64_t slotcount, bytecount;
107 	uint64_t prevslotcount, prevbytecount;
108 	struct timeval prevtime;
109 
110 	gettimeofday(&prevtime, NULL);
111 
112 	rxring = os_channel_rx_ring(rxchannel, rxindex +
113 	    os_channel_ring_id(rxchannel, CHANNEL_FIRST_RX_RING));
114 	assert(rxring);
115 	txring = os_channel_tx_ring(txchannel, txindex +
116 	    os_channel_ring_id(txchannel, CHANNEL_FIRST_TX_RING));
117 	assert(txring);
118 
119 	rxfd = os_channel_get_fd(rxchannel);
120 	EV_SET(&kev, rxfd, EVFILT_READ, EV_ADD | EV_ENABLE, 0, 0, NULL);
121 	error = kevent(kq, &kev, 1, NULL, 0, NULL);
122 	SKTC_ASSERT_ERR(!error);
123 	rxenable = true;
124 
125 	txfd = os_channel_get_fd(txchannel);
126 	EV_SET(&kev, txfd, EVFILT_WRITE, EV_ADD | EV_ENABLE, 0, 0, NULL);
127 	error = kevent(kq, &kev, 1, NULL, 0, NULL);
128 	SKTC_ASSERT_ERR(!error);
129 	txenable = true;
130 
131 	T_LOG("thread %s has kq %d rxfd %d txfd %d\n",
132 	    threadname, kq, rxfd, txfd);
133 
134 	prevslotcount = slotcount = 0;
135 	prevbytecount = bytecount = 0;
136 	int stallcount = 0;
137 	then = time(NULL);
138 	start = time(NULL);
139 
140 	while (!g_die) {
141 		uint32_t rxavail, txavail, xfer;
142 
143 		do {
144 			rxavail = os_channel_available_slot_count(rxring);
145 			txavail = os_channel_available_slot_count(txring);
146 			VLOG("rxavail %u txavail %u", rxavail, txavail);
147 
148 			/* If there's no data to receive stop asking for output notifications
149 			 * but make sure that if both rx and tx are not available, that
150 			 * the tx remains enabled to we can wake up to reenable rx when tx
151 			 * eventually becomes available
152 			 */
153 			if (txenable && !rxavail && txavail) {
154 				EV_SET(&kev, txfd, EVFILT_WRITE, EV_DISABLE, 0, 0, NULL);
155 				error = kevent(kq, &kev, 1, NULL, 0, NULL);
156 				SKTC_ASSERT_ERR(!error);
157 				txenable = false;
158 				VLOG("txenable = false");
159 			} else if (!txenable && (rxavail || (!rxavail && !txavail))) {
160 				EV_SET(&kev, txfd, EVFILT_WRITE, EV_ENABLE, 0, 0, NULL);
161 				error = kevent(kq, &kev, 1, NULL, 0, NULL);
162 				SKTC_ASSERT_ERR(!error);
163 				txenable = true;
164 				VLOG("txenable = true");
165 			}
166 
167 			/* If there's no place to put data, stop asking for input notifications */
168 			if (rxenable && !txavail) {
169 				EV_SET(&kev, rxfd, EVFILT_READ, EV_DISABLE, 0, 0, NULL);
170 				error = kevent(kq, &kev, 1, NULL, 0, NULL);
171 				SKTC_ASSERT_ERR(!error);
172 				rxenable = false;
173 				VLOG("rxenable = false");
174 			} else if (!rxenable && txavail) {
175 				EV_SET(&kev, rxfd, EVFILT_READ, EV_ENABLE, 0, 0, NULL);
176 				error = kevent(kq, &kev, 1, NULL, 0, NULL);
177 				SKTC_ASSERT_ERR(!error);
178 				rxenable = true;
179 				VLOG("rxenable = true");
180 			}
181 
182 			if (!rxavail || !txavail) {
183 				struct timespec timeout = {.tv_sec = 1, .tv_nsec = 0};  // 1 second
184 				VLOG("waiting rxen %d rx %u txen %d tx %u",
185 				    rxenable, rxavail, txenable, txavail);
186 
187 				assert(txenable || rxenable);
188 
189 				error = kevent(kq, NULL, 0, &kev, 1, &timeout);
190 				if (error == 0) {
191 					//T_LOG("%s: kevent tick\n", threadname);
192 					if (g_die) {
193 						T_LOG("%s: die set, exiting\n", threadname);
194 						goto out;
195 					}
196 				} else {
197 					SKTC_ASSERT_ERR(error != -1);
198 					SKTC_ASSERT_ERR(error == 1);
199 					if (kev.filter == EVFILT_USER) {
200 						T_LOG("%s: user event, exiting\n",
201 						    threadname);
202 						goto out;
203 					} else if (kev.filter == EVFILT_WRITE) {
204 						VLOG("write event");
205 					} else if (kev.filter == EVFILT_READ) {
206 						VLOG("read event");
207 					} else {
208 						assert(false);
209 					}
210 				}
211 			}
212 
213 			now = time(NULL);
214 			if (now > then) {
215 				T_LOG("%s: time %ld slotcount %llu "
216 				    "(total %llu) bytecount %llu (total %llu)\n",
217 				    threadname, now - start,
218 				    slotcount - prevslotcount, slotcount,
219 				    bytecount - prevbytecount, bytecount);
220 
221 				if ((now - start) > 0 && (slotcount - prevslotcount) == 0) {
222 					stallcount++;
223 					VLOG("STALLING");
224 					if ((xfer12 && g_assert_stalls12) || (!xfer12 && g_assert_stalls21)) {
225 						if (stallcount > 2) {
226 							T_LOG("%s: STALLING count %d rxavail %u txavail %u\n",
227 							    threadname, stallcount, rxavail, txavail);
228 						}
229 						assert(stallcount < 10);
230 						if (stallcount == 5) {
231 #if 0
232 							reboot_np(RB_PANIC | RB_QUICK, "skt_utunloop stalled");
233 							host_reboot(mach_host_self(), HOST_REBOOT_DEBUGGER);
234 #elif 0
235 							if (!strcmp(threadname, "sktc_channel_worker_xfer21")) {
236 								T_LOG("%s: Running %s\n", threadname, debugcmd);
237 								pclose(popen(debugcmd, "r"));
238 								//system(debugcmd);
239 							}
240 //						T_LOG("%s: Sleeping\n", threadname);
241 //						sleep(3600000);
242 							T_LOG("%s: exiting because of stall\n", threadname);
243 							exit(252);
244 #elif 0
245 							T_LOG("%s: enabling verbose\n", threadname);
246 							uint64_t verbose = (1ULL << 50);
247 							error = sysctlbyname("kern.skywalk.verbose", NULL, NULL, &verbose, sizeof(verbose));
248 							SKTC_ASSERT_ERR(!error);
249 #endif
250 						}
251 					}
252 				} else {
253 					stallcount = 0;
254 				}
255 
256 				then = now;
257 				prevslotcount = slotcount;
258 				prevbytecount = bytecount;
259 			}
260 		} while (!rxavail || !txavail);
261 
262 #if 0
263 		/*
264 		 * Yes this distribution includes syncs with 0 slots,
265 		 * but that should be handled ok, so lets include it.
266 		 */
267 		xfer = arc4random_uniform(1 + MIN(txavail, rxavail));
268 #else
269 		/* IDS only transfers one slot i think */
270 		xfer = 1;
271 		//xfer = MIN(txavail,rxavail);
272 #endif
273 
274 		VLOG("rx %u tx %u xfer %u", rxavail, txavail, xfer);
275 
276 		channel_slot_t rxprev = NULL, txprev = NULL;
277 
278 		for (uint32_t i = 0; i < xfer; i++) {
279 			slot_prop_t rxprop, txprop;
280 			channel_slot_t rxslot, txslot;
281 
282 			rxslot = os_channel_get_next_slot(rxring, rxprev, &rxprop);
283 			assert(rxslot);
284 			txslot = os_channel_get_next_slot(txring, txprev, &txprop);
285 			assert(txslot);
286 
287 			assert(txprop.sp_len >= rxprop.sp_len);
288 			memcpy((void *)txprop.sp_buf_ptr,
289 			    (void *)rxprop.sp_buf_ptr, rxprop.sp_len);
290 			txprop.sp_len = rxprop.sp_len;
291 			os_channel_set_slot_properties(txring, txslot, &txprop);
292 
293 			slotcount += 1;
294 			bytecount += txprop.sp_len;
295 
296 			rxprev = rxslot;
297 			txprev = txslot;
298 
299 #if 1 // this tries to be like IDS which syncs every outgoing packet
300 			error = os_channel_advance_slot(txring, txprev);
301 			SKTC_ASSERT_ERR(!error);
302 			error = os_channel_sync(txchannel, CHANNEL_SYNC_TX);
303 			SKTC_ASSERT_ERR(!error);
304 			txprev = NULL;
305 #endif
306 		}
307 
308 		if (txprev) {
309 			// If we don't sync every slot above we would do this
310 			error = os_channel_advance_slot(txring, txprev);
311 			SKTC_ASSERT_ERR(!error);
312 			error = os_channel_sync(txchannel, CHANNEL_SYNC_TX);
313 			SKTC_ASSERT_ERR(!error);
314 		}
315 
316 		// IDS calls rx sync, so we do it here.
317 		error = os_channel_advance_slot(rxring, rxprev);
318 		SKTC_ASSERT_ERR(!error);
319 		error = os_channel_sync(rxchannel, CHANNEL_SYNC_RX);
320 		SKTC_ASSERT_ERR(!error);
321 	}
322 
323 out:
324 	return;
325 }
326 
327 static channel_t g_channel1, g_channel2;
328 static int g_kq1, g_kq2;
329 
330 static void *
sktc_channel_worker_xfer12(void * ignored)331 sktc_channel_worker_xfer12(void *ignored)
332 {
333 	pthread_setname_np(__func__);
334 	skt_utunloop_xfer_slots(g_kq1, g_channel1, 0, g_channel2, 0, __func__, true);
335 	return NULL;
336 }
337 
338 static void *
sktc_channel_worker_xfer21(void * ignored)339 sktc_channel_worker_xfer21(void *ignored)
340 {
341 	pthread_setname_np(__func__);
342 	skt_utunloop_xfer_slots(g_kq2, g_channel2, 0, g_channel1, 0, __func__, false);
343 	return NULL;
344 }
345 
346 static bool
setblocking(int s,bool blocking)347 setblocking(int s, bool blocking)
348 {
349 	int error, flags;
350 	bool ret;
351 	error = fcntl(s, F_GETFL, 0);
352 	SKTC_ASSERT_ERR(error >= 0);
353 	flags = error;
354 
355 	ret = !(flags & O_NONBLOCK);
356 
357 	if (blocking) {
358 		flags &= ~O_NONBLOCK;
359 	} else {
360 		flags |= O_NONBLOCK;
361 	}
362 
363 	T_LOG("Setting fd %d from %s to %s\n",
364 	    s, ret ? "blocking" : "nonblocking",
365 	    blocking ? "blocking" : "nonblocking");
366 
367 	error = fcntl(s, F_SETFL, flags);
368 	SKTC_ASSERT_ERR(!error);
369 
370 	return ret;
371 }
372 
373 
374 static int
makesocket(int type,in_addr_t addr)375 makesocket(int type, in_addr_t addr)
376 {
377 	int error;
378 	int s;
379 	char sbuf[INET6_ADDRSTRLEN];
380 	struct sockaddr_in sin;
381 
382 	s = socket(PF_INET, type, 0);
383 	assert(s != -1);
384 
385 #if 0
386 	unsigned int ifidx;
387 	ifidx = if_nametoindex(ifname1); // xxx
388 	assert(ifidx != 0);
389 	error = setsockopt(s, IPPROTO_IP, IP_BOUND_IF, &ifidx, sizeof(ifidx));
390 	SKTC_ASSERT_ERR(!error);
391 #endif
392 
393 	memset(&sin, 0, sizeof(sin));
394 	sin.sin_len = sizeof(sin);
395 	sin.sin_family = AF_INET;
396 	sin.sin_addr.s_addr = htonl(addr);
397 
398 	error = bind(s, (struct sockaddr *)&sin, sizeof(sin));
399 	SKTC_ASSERT_ERR(!error);
400 
401 	inet_ntop(sin.sin_family, &sin.sin_addr.s_addr, sbuf, sizeof(sbuf));
402 	T_LOG("%s socket %d bound to %s port %d\n",
403 	    type == SOCK_DGRAM ? "udp" : "tcp",
404 	    s, sbuf, ntohs(sin.sin_port));
405 
406 	return s;
407 }
408 
409 static void
connectsocks(int s1,int s2,bool block)410 connectsocks(int s1, int s2, bool block)
411 {
412 	int error;
413 	struct sockaddr_in sin;
414 	socklen_t slen;
415 	char sbuf[INET6_ADDRSTRLEN];
416 	bool oblock;
417 
418 	slen = sizeof(sin);
419 	error = getsockname(s2, (struct sockaddr *)&sin, &slen);
420 	SKTC_ASSERT_ERR(!error);
421 	assert(slen <= sizeof(sin));
422 
423 	oblock = setblocking(s1, block);
424 
425 	inet_ntop(sin.sin_family, &sin.sin_addr.s_addr, sbuf, sizeof(sbuf));
426 	T_LOG("socket %d attempting to connect to %s port %d\n", s1, sbuf, ntohs(sin.sin_port));
427 
428 	error = connect(s1, (struct sockaddr *)&sin, slen);
429 	if (block) {
430 		SKTC_ASSERT_ERR(!error);
431 	} else {
432 		if (error == -1 && (errno == ENETDOWN || errno == EHOSTUNREACH)) {
433 			SKT_LOG("socket %d waiting 1 second for net to come up (errno %d)\n",
434 			    s1, errno);
435 			sleep(1);
436 			error = connect(s1, (struct sockaddr *)&sin, slen);
437 		}
438 		SKTC_ASSERT_ERR(error == -1);
439 		SKTC_ASSERT_ERR(errno == EINPROGRESS);
440 	}
441 
442 	setblocking(s1, oblock);
443 
444 	inet_ntop(AF_INET, &sin.sin_addr.s_addr, sbuf, sizeof(sbuf));
445 	T_LOG("socket %d connect%s to %s port %d\n", s1,
446 	    block ? "ed" : "ing", sbuf, ntohs(sin.sin_port));
447 }
448 
449 static int
acceptsock(int s)450 acceptsock(int s)
451 {
452 	int error;
453 	struct sockaddr_in sin;
454 	socklen_t slen;
455 	char sbuf[INET6_ADDRSTRLEN];
456 
457 	slen = sizeof(sin);
458 	error = accept(s, (struct sockaddr *)&sin, &slen);
459 	SKTC_ASSERT_ERR(error >= 0);
460 
461 	inet_ntop(AF_INET, &sin.sin_addr.s_addr, sbuf, sizeof(sbuf));
462 	T_LOG("tcp socket %d accepted connection from %s port %d\n", error, sbuf, ntohs(sin.sin_port));
463 
464 	return error;
465 }
466 
467 #if __LP64__
468 #define UDPXFER                 100000 /* 100k */
469 #define UDPLOSSOK               3000   /* 3% */
470 #define UDPXFER_MEMFAIL         30000  /* 30k */
471 #define UDPLOSSOK_MEMFAIL       9000   /* 30% */
472 #define UDPPACE                 100003 /* 100us (prime) */
473 #else
474 /* On 32 bit platforms, only try to xfer 10k slots */
475 #define UDPXFER                 10000  /* 10k */
476 #define UDPLOSSOK               300    /* 3% */
477 #define UDPXFER_MEMFAIL         3000   /* 30k */
478 #define UDPLOSSOK_MEMFAIL       900    /* 30% */
479 #define UDPPACE                 150001 /* 150us (prime) */
480 #endif
481 #define UDPSIZE 1000
482 
483 static uint32_t udpxfer;
484 static uint32_t udplossok;
485 
486 static void *
sinkudp(void * sockfd)487 sinkudp(void *sockfd)
488 {
489 	int s = *(int *)sockfd;
490 	ssize_t len;
491 	char buf[UDPSIZE];
492 	char threadname[20];
493 	int missed = 0;
494 	int readcount = 0;
495 	int i;
496 	struct timeval prevtime;
497 
498 	gettimeofday(&prevtime, NULL);
499 
500 	snprintf(threadname, sizeof(threadname), "%s%d", __func__, s);
501 	pthread_setname_np(threadname);
502 
503 	assert(udpxfer != 0);
504 
505 	for (i = 0; i < udpxfer; i++) {
506 		len = read(s, buf, sizeof(buf));
507 		VLOG("read %zd/%zd", len, sizeof(buf));
508 		if (len != sizeof(buf)) {
509 			SKT_LOG("%s read returned %zd errno %d count %d/%d\n",
510 			    threadname, len, errno, i, udpxfer);
511 			if (len == -1 && errno == EBADF) {
512 				goto out;
513 			}
514 		}
515 		readcount++;
516 		if (memcmp(buf, &i, sizeof(i))) {
517 			int tmp;
518 			memcpy(&tmp, buf, sizeof(tmp));
519 			if (tmp < i) {
520 				T_LOG("%s out of order expecting %d got %d\n",
521 				    threadname, i, tmp);
522 			}
523 			assert(tmp > i); // out of order will crash
524 			missed += tmp - i;
525 			i = tmp; // skip missing packets
526 		}
527 		assert(len == sizeof(buf));
528 	}
529 
530 out:
531 	T_LOG("%s received %d packets, missed %d, i = %d\n",
532 	    threadname, readcount, missed, i);
533 	assert(missed <= udplossok);
534 	assert(readcount >= udpxfer - udplossok);
535 
536 	return NULL;
537 }
538 
539 static void *
sourceudp(void * sockfd)540 sourceudp(void *sockfd)
541 {
542 	int s = *(int *)sockfd;
543 	ssize_t len;
544 	char buf[UDPSIZE];
545 	char threadname[20];
546 	int error;
547 	int kq;
548 	struct kevent kev;
549 	struct timeval prevtime;
550 
551 	gettimeofday(&prevtime, NULL);
552 
553 	snprintf(threadname, sizeof(threadname), "%s%d", __func__, s);
554 	pthread_setname_np(threadname);
555 
556 	kq = kqueue();
557 	EV_SET(&kev, s, EVFILT_WRITE, EV_ADD | EV_ENABLE, 0, 0, NULL);
558 	error = kevent(kq, &kev, 1, NULL, 0, NULL);
559 	SKTC_ASSERT_ERR(!error);
560 
561 	assert(udpxfer != 0);
562 
563 	uint64_t totalloop = 0;
564 	uint32_t loops = 0;
565 	for (int i = 0; i < udpxfer; i++) {
566 		uint32_t loopcnt = 0;
567 		struct timespec ts;
568 
569 		memcpy(buf, &i, sizeof(i));
570 
571 		do {
572 			len = write(s, buf, sizeof(buf));
573 			VLOG("wrote %zd/%zd", len, sizeof(buf));
574 
575 			/* If the very first write gets ENETDOWN, wait 1 second */
576 			if (i == 0 && loopcnt == 0 &&
577 			    len == -1 && (errno == ENETDOWN || errno == EHOSTUNREACH)) {
578 				SKT_LOG("%s waiting 1 second for net to come up (errno %d)\n",
579 				    threadname, errno);
580 				sleep(1);
581 				len = write(s, buf, sizeof(buf));
582 				VLOG("wrote %zd/%zd", len, sizeof(buf));
583 			}
584 
585 			/* Wait for buffers to be available */
586 			if (len == -1 && errno == ENOBUFS) {
587 				loopcnt++;
588 				//T_LOG("%s waiting in kevent for buffers\n", threadname);
589 				error = kevent(kq, NULL, 0, &kev, 1, NULL);
590 				SKTC_ASSERT_ERR(error == 1);
591 				assert(kev.filter == EVFILT_WRITE);
592 				assert(kev.ident == s);
593 				assert(kev.udata == NULL);
594 				assert((kev.flags & EV_ERROR) == 0);
595 			} else {
596 				if (len != sizeof(buf)) {
597 					SKT_LOG("%s write returned %zd errno %d count %d/%d\n",
598 					    threadname, len, errno, i, udpxfer);
599 				}
600 				assert(len == sizeof(buf));
601 			}
602 
603 			if (loopcnt > 1) {
604 				/* if we got ENOBUFS more than once, then sleep
605 				 * to avoid tight looping on write
606 				 */
607 				ts.tv_sec = 0;
608 				ts.tv_nsec = 1000003; // 1ms (prime)
609 				nanosleep(&ts, NULL);
610 			} else {
611 				ts.tv_sec = 0;
612 				ts.tv_nsec = UDPPACE;
613 				nanosleep(&ts, NULL);
614 			}
615 
616 			/* If we're starved for a full five seconds, crash */
617 			if (loopcnt >= 5000) {
618 				T_LOG("loopcount %d\n", loopcnt);
619 			}
620 			assert(loopcnt < 5000);
621 		} while (len != sizeof(buf));
622 
623 		/* Ideally we wouldn't get ENOBUFS immediately after getting
624 		 * a writable kevent.  However, these are coming from nx_netif_host
625 		 * when ms_classq_mbuf_to_kpkt can't allocate a packet.  In this
626 		 * case, flow control doesn't apply, so just tally the occurances.
627 		 */
628 		if (loopcnt > 1) {
629 			loops++;
630 			totalloop += loopcnt - 1;
631 			//T_LOG("%s spun in kevent %d times\n", threadname, loopcnt);
632 		}
633 	}
634 
635 	error = close(kq);
636 	SKTC_ASSERT_ERR(!error);
637 
638 	T_LOG("%s wrote %d packets, looped %u times (avg %f) exiting\n",
639 	    threadname, udpxfer, loops, (double)totalloop / loops);
640 
641 	return NULL;
642 }
643 
644 #if __LP64__
645 #define TCPXFER         100000000 /* 100mb */
646 #define TCPXFER_MEMFAIL 5000000   /* 5mb */
647 #else
648 #define TCPXFER         10000000  /* 10mb */
649 #define TCPXFER_MEMFAIL 500000    /* 0.5mb */
650 #endif
651 
652 static uint32_t tcpxfer;
653 
654 static void *
sinktcp(void * sockfd)655 sinktcp(void *sockfd)
656 {
657 	int s = *(int *)sockfd;
658 	ssize_t len;
659 	char *buf;
660 	int buflen;
661 	socklen_t optlen;
662 	char threadname[20];
663 	int error;
664 	size_t nxfer;
665 	struct timeval prevtime;
666 
667 	gettimeofday(&prevtime, NULL);
668 
669 	snprintf(threadname, sizeof(threadname), "%s%d", __func__, s);
670 	pthread_setname_np(threadname);
671 
672 	optlen = sizeof(buflen);
673 	error = getsockopt(s, SOL_SOCKET, SO_RCVBUF, &buflen, &optlen);
674 	SKTC_ASSERT_ERR(!error);
675 
676 	T_LOG("%s fd %d rcvbuf size %d\n", threadname, s, buflen);
677 
678 	buf = calloc(buflen, 1);
679 	assert(buf);
680 
681 	assert(tcpxfer != 0);
682 
683 	nxfer = 0;
684 	while (nxfer < tcpxfer) {
685 		size_t thisxfer = MIN(tcpxfer - nxfer, buflen);
686 		len = read(s, buf, thisxfer);
687 		VLOG("read %zd/%zd", len, thisxfer);
688 		//T_LOG("%s fd %d read of %zu returned %zd\n", threadname, s, thisxfer, len);
689 		error = len;
690 		SKTC_ASSERT_ERR(error != -1);
691 		SKTC_ASSERT_ERR(error > 0);
692 		SKTC_ASSERT_ERR(error <= thisxfer);
693 		nxfer += len;
694 	}
695 
696 	assert(nxfer == tcpxfer);
697 
698 	free(buf);
699 
700 	T_LOG("%s read %zu bytes exiting\n",
701 	    threadname, nxfer);
702 
703 	return NULL;
704 }
705 
706 
707 static void *
sourcetcp(void * sockfd)708 sourcetcp(void *sockfd)
709 {
710 	int s = *(int *)sockfd;
711 	ssize_t len;
712 	char *buf;
713 	int buflen;
714 	socklen_t optlen;
715 	char threadname[20];
716 	int error;
717 	size_t nxfer;
718 	struct timeval prevtime;
719 
720 	gettimeofday(&prevtime, NULL);
721 
722 	snprintf(threadname, sizeof(threadname), "%s%d", __func__, s);
723 	pthread_setname_np(threadname);
724 
725 	optlen = sizeof(buflen);
726 	error = getsockopt(s, SOL_SOCKET, SO_SNDBUF, &buflen, &optlen);
727 	SKTC_ASSERT_ERR(!error);
728 
729 	T_LOG("%s fd %d sndbuf size %d\n", threadname, s, buflen);
730 
731 	buf = calloc(buflen, 1);
732 	assert(buf);
733 
734 	assert(tcpxfer != 0);
735 
736 	nxfer = 0;
737 	while (nxfer < tcpxfer) {
738 		size_t thisxfer = MIN(tcpxfer - nxfer, buflen);
739 		len = write(s, buf, thisxfer);
740 		VLOG("wrote %zd/%zd", len, thisxfer);
741 
742 		/* If the very first write gets ENETDOWN, wait 1 second */
743 		if (nxfer == 0 && len == -1 &&
744 		    (errno == ENETDOWN || errno == EHOSTUNREACH)) {
745 			SKT_LOG("%s waiting 1 second for net to come up (errno %d)\n",
746 			    threadname, errno);
747 			sleep(1);
748 			len = write(s, buf, thisxfer);
749 			VLOG("wrote %zd/%zd", len, thisxfer);
750 		}
751 
752 		//T_LOG("%s fd %d write of %zu returned %zd\n", threadname, s, thisxfer, len);
753 		error = len;
754 		SKTC_ASSERT_ERR(error == thisxfer);
755 		nxfer += len;
756 	}
757 
758 	assert(nxfer == tcpxfer);
759 
760 	free(buf);
761 
762 	T_LOG("%s wrote %zu bytes exiting\n",
763 	    threadname, nxfer);
764 
765 	return NULL;
766 }
767 
768 static void
dotraffic(void * (* sourcefunc)(void *),void * (* sinkfunc)(void *),int sourcesock1,int sinksock2,int sourcesock2,int sinksock1)769 dotraffic(void *(*sourcefunc)(void *), void *(*sinkfunc)(void *),
770     int sourcesock1, int sinksock2, int sourcesock2, int sinksock1)
771 {
772 	int error;
773 	pthread_t sinkthread1, sinkthread2;
774 	pthread_t sourcethread1, sourcethread2;
775 
776 	assert(sourcesock1 != -1);
777 	assert(sinksock2 != -1);
778 	assert((sourcesock2 == -1) == (sinksock1 == -1));
779 
780 	if (sinksock1 != -1) {
781 		error = pthread_create(&sinkthread1, NULL, sinkfunc, &sinksock1);
782 		SKTC_ASSERT_ERR(!error);
783 	}
784 	error = pthread_create(&sinkthread2, NULL, sinkfunc, &sinksock2);
785 	SKTC_ASSERT_ERR(!error);
786 	error = pthread_create(&sourcethread1, NULL, sourcefunc, &sourcesock1);
787 	SKTC_ASSERT_ERR(!error);
788 	if (sourcesock2 != -1) {
789 		T_LOG("waiting 1 second before starting reverse traffic\n");
790 		sleep(1);
791 		error = pthread_create(&sourcethread2, NULL, sourcefunc, &sourcesock2);
792 		SKTC_ASSERT_ERR(!error);
793 	}
794 
795 	/* Wait for all data to be sent */
796 	error = pthread_join(sourcethread1, NULL);
797 	SKTC_ASSERT_ERR(!error);
798 	if (sourcesock2 != -1) {
799 		error = pthread_join(sourcethread2, NULL);
800 		SKTC_ASSERT_ERR(!error);
801 	}
802 
803 	/* Give it 1 second to drain */
804 	T_LOG("waiting 1 second for reads to drain\n");
805 	sleep(1);
806 
807 	/* Force the reads to exit by closing sockets */
808 	if (sinksock1 != -1) {
809 		T_LOG("closing sinksock1 %d\n", sinksock1);
810 		error = close(sinksock1);
811 		SKTC_ASSERT_ERR(!error);
812 	}
813 	T_LOG("closing sinksock2 %d\n", sinksock2);
814 	error = close(sinksock2);
815 	SKTC_ASSERT_ERR(!error);
816 
817 	if (sinksock1 != -1) {
818 		error = pthread_join(sinkthread1, NULL);
819 		SKTC_ASSERT_ERR(!error);
820 	}
821 	error = pthread_join(sinkthread2, NULL);
822 	SKTC_ASSERT_ERR(!error);
823 
824 	if (sourcesock1 != sinksock1) {
825 		T_LOG("closing sourcesock1 %d\n", sourcesock1);
826 		error = close(sourcesock1);
827 		SKTC_ASSERT_ERR(!error);
828 	}
829 	if (sourcesock2 != sinksock2 && sourcesock2 != -1) {
830 		T_LOG("closing sourcesock2 %d\n", sourcesock2);
831 		error = close(sourcesock2);
832 		SKTC_ASSERT_ERR(!error);
833 	}
834 }
835 
836 
837 static void
skt_tunloop_common(bool doutun,bool enable_netif,bool enable_channel,bool udp,bool udpduplex,bool tcp,bool tcpduplex,bool dualstream)838 skt_tunloop_common(bool doutun, bool enable_netif, bool enable_channel, bool udp, bool udpduplex, bool tcp, bool tcpduplex, bool dualstream)
839 {
840 	int error;
841 	int utun1, utun2;
842 	char ifname1[IFNAMSIZ];
843 	char ifname2[IFNAMSIZ];
844 	pthread_t thread1, thread2;
845 	struct kevent kev;
846 	uint32_t memfail = 0;
847 	size_t len;
848 	int keysock;
849 
850 	len = sizeof(memfail);
851 	if (sysctlbyname("kern.skywalk.mem.region_mtbf", &memfail, &len,
852 	    NULL, 0) != 0) {
853 		SKT_LOG("warning got errno %d getting "
854 		    "kern.skywalk.mem.region_mtbf: %s\n", errno,
855 		    strerror(errno));
856 	}
857 
858 	if (memfail) {
859 		udpxfer = UDPXFER_MEMFAIL;
860 		udplossok = UDPLOSSOK_MEMFAIL;
861 		tcpxfer = TCPXFER_MEMFAIL;
862 	} else {
863 		udpxfer = UDPXFER;
864 		udplossok = UDPLOSSOK;
865 		tcpxfer = TCPXFER;
866 	}
867 
868 	g_dumpfile = fopen(getenv("SKT_UTUNLOOP_DUMPFILE"), "w");
869 	if (g_dumpfile) {
870 		g_verbose = 1;
871 	}
872 
873 	sktu_if_type_t type = doutun ? SKTU_IFT_UTUN : SKTU_IFT_IPSEC;
874 	sktu_if_flag_t flags = enable_netif ? SKTU_IFF_ENABLE_NETIF : 0;
875 	flags |= enable_channel ? SKTU_IFF_ENABLE_CHANNEL : 0;
876 	utun1 = sktu_create_interface(type, flags);
877 	utun2 = sktu_create_interface(type, flags);
878 
879 	sktu_get_interface_name(type, utun1, ifname1);
880 	sktu_get_interface_name(type, utun2, ifname2);
881 	snprintf(debugcmd, sizeof(debugcmd), "netstat -qq -I %s > netstatqq.%s.txt; netstat -qq -I %s > netstatqq.%s.txt; skywalkctl netstat --flowswitch --netif > skywalkctl.txt",
882 	    ifname1, ifname1, ifname2, ifname2);
883 
884 	uint32_t utun1addr = (10 << 24) | (getpid() & 0xffff) << 8 | 150;
885 	uint32_t utun2addr = utun1addr + 1;
886 
887 	struct in_addr addr1, addr2, mask;
888 	mask  = sktc_make_in_addr(0xffffffff);
889 	addr1 = sktc_make_in_addr(utun1addr);
890 	addr2 = sktc_make_in_addr(utun2addr);
891 
892 	error = sktc_ifnet_add_addr(ifname1, &addr1, &mask, &addr2);
893 	SKTC_ASSERT_ERR(!error);
894 	error = sktc_ifnet_add_addr(ifname2, &addr2, &mask, &addr1);
895 	SKTC_ASSERT_ERR(!error);
896 
897 	if (!doutun) {
898 		keysock = sktu_create_pfkeysock();
899 		sktu_create_sa(keysock, ifname1, 12345, &addr1, &addr2);
900 		sktu_create_sa(keysock, ifname1, 12346, &addr2, &addr1);
901 		sktu_create_sa(keysock, ifname2, 12345, &addr2, &addr1);
902 		sktu_create_sa(keysock, ifname2, 12346, &addr1, &addr2);
903 	}
904 
905 	g_channel1 = sktu_create_interface_channel(type, utun1);
906 	g_channel2 = sktu_create_interface_channel(type, utun2);
907 
908 	T_LOG("Created %s and %s\n", ifname1, ifname2);
909 
910 	g_kq1 = kqueue();
911 	EV_SET(&kev, (uintptr_t)&g_die, EVFILT_USER,
912 	    EV_ADD | EV_ENABLE, 0, 0, NULL);
913 	error = kevent(g_kq1, &kev, 1, NULL, 0, NULL);
914 	SKTC_ASSERT_ERR(!error);
915 
916 	g_kq2 = kqueue();
917 	EV_SET(&kev, (uintptr_t)&g_die, EVFILT_USER,
918 	    EV_ADD | EV_ENABLE, 0, 0, NULL);
919 	error = kevent(g_kq2, &kev, 1, NULL, 0, NULL);
920 	SKTC_ASSERT_ERR(!error);
921 
922 //	T_LOG("Sleeping 10 seconds at startup\n");
923 //	sleep(10);
924 
925 	error = pthread_create(&thread1, NULL, sktc_channel_worker_xfer12, NULL);
926 	SKTC_ASSERT_ERR(!error);
927 	error = pthread_create(&thread2, NULL, sktc_channel_worker_xfer21, NULL);
928 	SKTC_ASSERT_ERR(!error);
929 
930 	if (udp) {
931 		int usock1, usock2;
932 		usock1 = makesocket(SOCK_DGRAM, utun1addr);
933 		usock2 = makesocket(SOCK_DGRAM, utun2addr);
934 		connectsocks(usock1, usock2, true);
935 		connectsocks(usock2, usock1, true);
936 		if (udpduplex) {
937 			if (dualstream) {
938 				int usock3, usock4;
939 				usock3 = makesocket(SOCK_DGRAM, utun2addr);
940 				usock4 = makesocket(SOCK_DGRAM, utun1addr);
941 				connectsocks(usock3, usock4, true);
942 				connectsocks(usock4, usock3, true);
943 				dotraffic(sourceudp, sinkudp, usock1, usock2, usock3, usock4);
944 			} else {
945 				dotraffic(sourceudp, sinkudp, usock1, usock2, usock2, usock1);
946 			}
947 		} else {
948 			dotraffic(sourceudp, sinkudp, usock1, usock2, -1, -1);
949 		}
950 	}
951 
952 	if (tcp) {
953 		int tsock1, tsock2, lsock; // listening socket
954 		tsock1 = makesocket(SOCK_STREAM, utun1addr);
955 		lsock = makesocket(SOCK_STREAM, utun2addr);
956 		error = listen(lsock, 1);
957 		SKTC_ASSERT_ERR(!error);
958 		connectsocks(tsock1, lsock, false);
959 		tsock2 = acceptsock(lsock);
960 		error = close(lsock);
961 		SKTC_ASSERT_ERR(!error);
962 		if (tcpduplex) {
963 			if (dualstream) {
964 				int tsock3, tsock4;
965 				tsock3 = makesocket(SOCK_STREAM, utun2addr);
966 				lsock = makesocket(SOCK_STREAM, utun1addr);
967 				error = listen(lsock, 1);
968 				SKTC_ASSERT_ERR(!error);
969 				connectsocks(tsock3, lsock, false);
970 				tsock4 = acceptsock(lsock);
971 				error = close(lsock);
972 				SKTC_ASSERT_ERR(!error);
973 				dotraffic(sourcetcp, sinktcp, tsock1, tsock2, tsock3, tsock4);
974 			} else {
975 				dotraffic(sourcetcp, sinktcp, tsock1, tsock2, tsock2, tsock1);
976 			}
977 		} else {
978 			dotraffic(sourcetcp, sinktcp, tsock1, tsock2, -1, -1);
979 		}
980 	}
981 
982 	/* This can be useful for just setting up two utuns */
983 	if (!udp && !tcp) {
984 		sleep(1000);
985 	}
986 
987 	/* Tell utun threads to exit */
988 	g_die = true;
989 	EV_SET(&kev, (uintptr_t)&g_die, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
990 	error = kevent(g_kq1, &kev, 1, NULL, 0, NULL);
991 	SKTC_ASSERT_ERR(!error);
992 	EV_SET(&kev, (uintptr_t)&g_die, EVFILT_USER, 0, NOTE_TRIGGER, 0, NULL);
993 	error = kevent(g_kq2, &kev, 1, NULL, 0, NULL);
994 	SKTC_ASSERT_ERR(!error);
995 
996 	error = pthread_join(thread1, NULL);
997 	SKTC_ASSERT_ERR(!error);
998 	error = pthread_join(thread2, NULL);
999 	SKTC_ASSERT_ERR(!error);
1000 
1001 	os_channel_destroy(g_channel1);
1002 	os_channel_destroy(g_channel2);
1003 
1004 	if (!doutun) {
1005 		error = close(keysock);
1006 		SKTC_ASSERT_ERR(!error);
1007 	}
1008 	error = close(utun1);
1009 	SKTC_ASSERT_ERR(!error);
1010 	error = close(utun2);
1011 	SKTC_ASSERT_ERR(!error);
1012 }
1013 
1014 /****************************************************************/
1015 
1016 static int
skt_utunloopy4u1_main(int argc,char * argv[])1017 skt_utunloopy4u1_main(int argc, char *argv[])
1018 {
1019 	g_assert_stalls12 = true;
1020 	skt_tunloop_common(true, true, true, true, false, false, false, false);
1021 	return 0;
1022 }
1023 
1024 static int
skt_utunloopy4u2_main(int argc,char * argv[])1025 skt_utunloopy4u2_main(int argc, char *argv[])
1026 {
1027 	g_assert_stalls12 = true;
1028 	g_assert_stalls21 = true;
1029 	skt_tunloop_common(true, true, true, true, true, false, false, false);
1030 	return 0;
1031 }
1032 
1033 static int
skt_utunloopy4t1_main(int argc,char * argv[])1034 skt_utunloopy4t1_main(int argc, char *argv[])
1035 {
1036 	g_assert_stalls12 = true;
1037 	skt_tunloop_common(true, true, true, false, false, true, false, false);
1038 	return 0;
1039 }
1040 
1041 static int
skt_utunloopy4t2_main(int argc,char * argv[])1042 skt_utunloopy4t2_main(int argc, char *argv[])
1043 {
1044 	g_assert_stalls12 = true;
1045 	g_assert_stalls21 = true;
1046 	skt_tunloop_common(true, true, true, false, false, true, true, false);
1047 	return 0;
1048 }
1049 
1050 static int
skt_utunloopy1000_main(int argc,char * argv[])1051 skt_utunloopy1000_main(int argc, char *argv[])
1052 {
1053 	skt_tunloop_common(true, true, true, false, false, false, false, false);
1054 	return 0;
1055 }
1056 
1057 struct skywalk_test skt_utunloopy4u1 = {
1058 	"utunloopy4u1", "open 2 utuns with netif and floods ipv4 udp packets in one direction",
1059 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1060 	skt_utunloopy4u1_main,
1061 };
1062 
1063 struct skywalk_test skt_utunloopy4u2 = {
1064 	"utunloopy4u2", "open 2 utuns with netif and floods ipv4 udp packets in two directions",
1065 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1066 	skt_utunloopy4u2_main,
1067 };
1068 
1069 struct skywalk_test skt_utunloopy4t1 = {
1070 	"utunloopy4t1", "open 2 utuns with netif and floods ipv4 tcp packets in one direction",
1071 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1072 	skt_utunloopy4t1_main,
1073 };
1074 
1075 struct skywalk_test skt_utunloopy4t2 = {
1076 	"utunloopy4t2", "open 2 utuns with netif and floods ipv4 tcp packets in two directions",
1077 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1078 	skt_utunloopy4t2_main,
1079 };
1080 
1081 struct skywalk_test skt_utunloopy1000 = {
1082 	"utunloopy1000", "open 2 utuns with netif and sleeps for 1000 seconds",
1083 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1084 	skt_utunloopy1000_main,
1085 };
1086 
1087 /****************************************************************/
1088 
1089 static int
skt_ipsecloopy4u1_main(int argc,char * argv[])1090 skt_ipsecloopy4u1_main(int argc, char *argv[])
1091 {
1092 	g_assert_stalls12 = true;
1093 	skt_tunloop_common(false, true, true, true, false, false, false, false);
1094 	return 0;
1095 }
1096 
1097 static int
skt_ipsecloopy4u2_main(int argc,char * argv[])1098 skt_ipsecloopy4u2_main(int argc, char *argv[])
1099 {
1100 	g_assert_stalls12 = true;
1101 	g_assert_stalls21 = true;
1102 	skt_tunloop_common(false, true, true, true, true, false, false, false);
1103 	return 0;
1104 }
1105 
1106 static int
skt_ipsecloopy4t1_main(int argc,char * argv[])1107 skt_ipsecloopy4t1_main(int argc, char *argv[])
1108 {
1109 	g_assert_stalls12 = true;
1110 	skt_tunloop_common(false, true, true, false, false, true, false, false);
1111 	return 0;
1112 }
1113 
1114 static int
skt_ipsecloopy4t2_main(int argc,char * argv[])1115 skt_ipsecloopy4t2_main(int argc, char *argv[])
1116 {
1117 	g_assert_stalls12 = true;
1118 	g_assert_stalls21 = true;
1119 	skt_tunloop_common(false, true, true, false, false, true, true, false);
1120 	return 0;
1121 }
1122 
1123 static int
skt_ipsecloopy1000_main(int argc,char * argv[])1124 skt_ipsecloopy1000_main(int argc, char *argv[])
1125 {
1126 	skt_tunloop_common(false, true, true, false, false, false, false, false);
1127 	return 0;
1128 }
1129 
1130 struct skywalk_test skt_ipsecloopy4u1 = {
1131 	"ipsecloopy4u1", "open 2 ipsecs with netif and floods ipv4 udp packets in one direction",
1132 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1133 	skt_ipsecloopy4u1_main,
1134 };
1135 
1136 struct skywalk_test skt_ipsecloopy4u2 = {
1137 	"ipsecloopy4u2", "open 2 ipsecs with netif and floods ipv4 udp packets in two directions",
1138 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1139 	skt_ipsecloopy4u2_main,
1140 };
1141 
1142 struct skywalk_test skt_ipsecloopy4t1 = {
1143 	"ipsecloopy4t1", "open 2 ipsecs with netif and floods ipv4 tcp packets in one direction",
1144 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1145 	skt_ipsecloopy4t1_main,
1146 };
1147 
1148 struct skywalk_test skt_ipsecloopy4t2 = {
1149 	"ipsecloopy4t2", "open 2 ipsecs with netif and floods ipv4 tcp packets in two directions",
1150 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1151 	skt_ipsecloopy4t2_main,
1152 };
1153 
1154 struct skywalk_test skt_ipsecloopy1000 = {
1155 	"ipsecloopy1000", "open 2 ipsecs with netif and sleeps for 1000 seconds",
1156 	SK_FEATURE_SKYWALK | SK_FEATURE_NEXUS_KERNEL_PIPE,
1157 	skt_ipsecloopy1000_main,
1158 };
1159 
1160 /****************************************************************/
1161