260 static int pipepair_alloc(struct pipe **rpipe, struct pipe **wpipe);
395 	struct pipe *rpipe, *wpipe;  in pipe()  local
398 	error = pipepair_alloc(&rpipe, &wpipe);  in pipe()
414 	fp_set_data(rf, rpipe);  in pipe()
425 	rpipe->pipe_peer = wpipe;  in pipe()
426 	wpipe->pipe_peer = rpipe;  in pipe()
437 	mac_pipe_label_init(rpipe);  in pipe()
438 	mac_pipe_label_associate(kauth_cred_get(), rpipe);  in pipe()
439 	mac_pipe_set_label(wpipe, mac_pipe_label(rpipe));  in pipe()
450 	pipeclose(rpipe);  in pipe()
614 	struct pipe *rpipe = &pp->pp_rpipe;  in pipepair_alloc()  local
624 	rpipe->pipe_mtxp = &pp->pp_mtx;  in pipepair_alloc()
629 	pipe_touch(rpipe, PIPE_ATIME | PIPE_MTIME | PIPE_CTIME);  in pipepair_alloc()
638 	int error = pipespace(rpipe, choose_pipespace(rpipe->pipe_buffer.size, 0));  in pipepair_alloc()
645 	*rp_out = rpipe;  in pipepair_alloc()
752 	struct pipe *rpipe = (struct pipe *)fp_get_data(fp);  in pipe_read()  local
757 	PIPE_LOCK(rpipe);  in pipe_read()
758 	++rpipe->pipe_busy;  in pipe_read()
760 	error = pipeio_lock(rpipe, 1);  in pipe_read()
766 	error = mac_pipe_check_read(kauth_cred_get(), rpipe);  in pipe_read()
777 		if (rpipe->pipe_buffer.cnt > 0) {  in pipe_read()
783 			size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;  in pipe_read()
784 			if (size > rpipe->pipe_buffer.cnt) {  in pipe_read()
785 				size = rpipe->pipe_buffer.cnt;  in pipe_read()
791 			PIPE_UNLOCK(rpipe); /* we still hold io lock.*/  in pipe_read()
792 			pipe_check_bounds_panic(rpipe);  in pipe_read()
794 				&rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],  in pipe_read()
796 			PIPE_LOCK(rpipe);  in pipe_read()
801 			rpipe->pipe_buffer.out += size;  in pipe_read()
802 			if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size) {  in pipe_read()
803 				rpipe->pipe_buffer.out = 0;  in pipe_read()
806 			rpipe->pipe_buffer.cnt -= size;  in pipe_read()
813 			if (rpipe->pipe_buffer.cnt == 0) {  in pipe_read()
814 				rpipe->pipe_buffer.in = 0;  in pipe_read()
815 				rpipe->pipe_buffer.out = 0;  in pipe_read()
823 			if ((rpipe->pipe_state & (PIPE_DRAIN | PIPE_EOF)) ||  in pipe_read()
831 			if (rpipe->pipe_state & PIPE_WANTW) {  in pipe_read()
832 				rpipe->pipe_state &= ~PIPE_WANTW;  in pipe_read()
833 				wakeup(rpipe);  in pipe_read()
848 			pipeio_unlock(rpipe);  in pipe_read()
857 				rpipe->pipe_state |= PIPE_WANTR;  in pipe_read()
858 				error = msleep(rpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH, "piperd", 0);  in pipe_read()
860 					error = pipeio_lock(rpipe, 1);  in pipe_read()
871 	pipeio_unlock(rpipe);  in pipe_read()
874 	--rpipe->pipe_busy;  in pipe_read()
879 	if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {  in pipe_read()
880 		rpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTW);  in pipe_read()
881 		wakeup(rpipe);  in pipe_read()
882 	} else if (rpipe->pipe_buffer.cnt < rpipe->pipe_buffer.size) {  in pipe_read()
886 		if (rpipe->pipe_state & PIPE_WANTW) {  in pipe_read()
887 			rpipe->pipe_state &= ~PIPE_WANTW;  in pipe_read()
888 			wakeup(rpipe);  in pipe_read()
892 	if ((rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt) > 0) {  in pipe_read()
893 		pipeselwakeup(rpipe, rpipe->pipe_peer);  in pipe_read()
898 	pipe_touch(rpipe, PIPE_ATIME);  in pipe_read()
901 	PIPE_UNLOCK(rpipe);  in pipe_read()
917 	struct pipe *wpipe, *rpipe;  in pipe_write()  local
925 	rpipe = (struct pipe *)fp_get_data(fp);  in pipe_write()
927 	PIPE_LOCK(rpipe);  in pipe_write()
928 	wpipe = rpipe->pipe_peer;  in pipe_write()
935 		PIPE_UNLOCK(rpipe);  in pipe_write()
941 		PIPE_UNLOCK(rpipe);  in pipe_write()
991 			PIPE_UNLOCK(rpipe);  in pipe_write()
1055 				PIPE_UNLOCK(rpipe);  in pipe_write()
1059 				PIPE_LOCK(rpipe);  in pipe_write()
1073 					PIPE_UNLOCK(rpipe);  in pipe_write()
1078 					PIPE_LOCK(rpipe);  in pipe_write()
1174 	pipe_touch(rpipe, PIPE_MTIME | PIPE_CTIME);  in pipe_write()
1177 	PIPE_UNLOCK(rpipe);  in pipe_write()
1245 	struct pipe *rpipe = (struct pipe *)fp_get_data(fp);  in pipe_select()  local
1249 	if (rpipe == NULL || rpipe == (struct pipe *)-1) {  in pipe_select()
1253 	PIPE_LOCK(rpipe);  in pipe_select()
1255 	wpipe = rpipe->pipe_peer;  in pipe_select()
1264 	if (mac_pipe_check_select(vfs_context_ucred(ctx), rpipe, which)) {  in pipe_select()
1265 		PIPE_UNLOCK(rpipe);  in pipe_select()
1271 		if ((rpipe->pipe_state & PIPE_DIRECTW) ||  in pipe_select()
1272 		    (rpipe->pipe_buffer.cnt > 0) ||  in pipe_select()
1273 		    (rpipe->pipe_state & (PIPE_DRAIN | PIPE_EOF)) ||  in pipe_select()
1277 			selrecord(vfs_context_proc(ctx), &rpipe->pipe_sel, wql);  in pipe_select()
1295 		selrecord(vfs_context_proc(ctx), &rpipe->pipe_sel, wql);  in pipe_select()
1298 	PIPE_UNLOCK(rpipe);  in pipe_select()
1392 filt_pipelowwat(struct knote *kn, struct pipe *rpipe, int64_t def_lowwat)  in filt_pipelowwat()  argument
1397 	if (rpipe->pipe_buffer.size && kn->kn_sdata > MAX_PIPESIZE(rpipe)) {  in filt_pipelowwat()
1398 		return MAX_PIPESIZE(rpipe);  in filt_pipelowwat()
1404 filt_pipe_draincommon(struct knote *kn, struct pipe *rpipe)  in filt_pipe_draincommon()  argument
1406 	struct pipe *wpipe = rpipe->pipe_peer;  in filt_pipe_draincommon()
1408 	if ((rpipe->pipe_state & (PIPE_DRAIN | PIPE_EOF)) ||  in filt_pipe_draincommon()
1421 	struct pipe *rpipe = kn->kn_hook;  in filt_pipenotsup()  local
1423 	return filt_pipe_draincommon(kn, rpipe);  in filt_pipenotsup()
1429 	struct pipe *rpipe = kn->kn_hook;  in filt_pipenotsuptouch()  local
1432 	PIPE_LOCK(rpipe);  in filt_pipenotsuptouch()
1439 	res = filt_pipe_draincommon(kn, rpipe);  in filt_pipenotsuptouch()
1441 	PIPE_UNLOCK(rpipe);  in filt_pipenotsuptouch()
1449 	struct pipe *rpipe = kn->kn_hook;  in filt_pipenotsupprocess()  local
1452 	PIPE_LOCK(rpipe);  in filt_pipenotsupprocess()
1453 	res = filt_pipe_draincommon(kn, rpipe);  in filt_pipenotsupprocess()
1457 	PIPE_UNLOCK(rpipe);  in filt_pipenotsupprocess()
1464 filt_piperead_common(struct knote *kn, struct kevent_qos_s *kev, struct pipe *rpipe)  in filt_piperead_common()  argument
1466 	int64_t data = rpipe->pipe_buffer.cnt;  in filt_piperead_common()
1469 	if (filt_pipe_draincommon(kn, rpipe)) {  in filt_piperead_common()
1472 		res = data >= filt_pipelowwat(kn, rpipe, 1);  in filt_piperead_common()
1484 	struct pipe *rpipe = kn->kn_hook;  in filt_piperead()  local
1486 	return filt_piperead_common(kn, NULL, rpipe);  in filt_piperead()
1492 	struct pipe *rpipe = kn->kn_hook;  in filt_pipereadtouch()  local
1495 	PIPE_LOCK(rpipe);  in filt_pipereadtouch()
1502 	retval = filt_piperead_common(kn, NULL, rpipe);  in filt_pipereadtouch()
1504 	PIPE_UNLOCK(rpipe);  in filt_pipereadtouch()
1512 	struct pipe *rpipe = kn->kn_hook;  in filt_pipereadprocess()  local
1515 	PIPE_LOCK(rpipe);  in filt_pipereadprocess()
1516 	retval = filt_piperead_common(kn, kev, rpipe);  in filt_pipereadprocess()
1517 	PIPE_UNLOCK(rpipe);  in filt_pipereadprocess()
1524 filt_pipewrite_common(struct knote *kn, struct kevent_qos_s *kev, struct pipe *rpipe)  in filt_pipewrite_common()  argument
1529 	if (filt_pipe_draincommon(kn, rpipe)) {  in filt_pipewrite_common()
1532 		data = MAX_PIPESIZE(rpipe) - rpipe->pipe_buffer.cnt;  in filt_pipewrite_common()
1533 		res = data >= filt_pipelowwat(kn, rpipe, PIPE_BUF);  in filt_pipewrite_common()
1546 	struct pipe *rpipe = kn->kn_hook;  in filt_pipewrite()  local
1548 	return filt_pipewrite_common(kn, NULL, rpipe);  in filt_pipewrite()
1555 	struct pipe *rpipe = kn->kn_hook;  in filt_pipewritetouch()  local
1558 	PIPE_LOCK(rpipe);  in filt_pipewritetouch()
1565 	res = filt_pipewrite_common(kn, NULL, rpipe);  in filt_pipewritetouch()
1567 	PIPE_UNLOCK(rpipe);  in filt_pipewritetouch()
1575 	struct pipe *rpipe = kn->kn_hook;  in filt_pipewriteprocess()  local
1578 	PIPE_LOCK(rpipe);  in filt_pipewriteprocess()
1579 	res = filt_pipewrite_common(kn, kev, rpipe);  in filt_pipewriteprocess()
1580 	PIPE_UNLOCK(rpipe);  in filt_pipewriteprocess()
1591 	struct pipe *rpipe = &PIPE_PAIR(cpipe)->pp_rpipe;  in pipe_kqfilter()  local
1626 			res = filt_piperead_common(kn, NULL, rpipe);  in pipe_kqfilter()
1629 			res = filt_pipe_draincommon(kn, rpipe);  in pipe_kqfilter()
1636 			res = filt_pipewrite_common(kn, NULL, rpipe);  in pipe_kqfilter()
1639 			res = filt_pipe_draincommon(kn, rpipe);  in pipe_kqfilter()
1649 	kn->kn_hook = rpipe;  in pipe_kqfilter()
1650 	KNOTE_ATTACH(&rpipe->pipe_sel.si_note, kn);  in pipe_kqfilter()
1660 	struct pipe *rpipe = &PIPE_PAIR(cpipe)->pp_rpipe;  in filt_pipedetach()  local
1663 	KNOTE_DETACH(&rpipe->pipe_sel.si_note, kn);  in filt_pipedetach()