685 	struct tcp_heuristic *tpheur = NULL;  in tcp_getheuristic_with_lock()  local
697 	SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) {  in tcp_getheuristic_with_lock()
698 		if (memcmp(&tpheur->th_key, &key, sizeof(key)) == 0) {  in tcp_getheuristic_with_lock()
706 	if ((tpheur == NULL) && create) {  in tcp_getheuristic_with_lock()
712 			SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) {  in tcp_getheuristic_with_lock()
713 				uint32_t age = tcp_now - tpheur->th_last_access;  in tcp_getheuristic_with_lock()
716 					oldest_heur = tpheur;  in tcp_getheuristic_with_lock()
721 			tpheur = oldest_heur;  in tcp_getheuristic_with_lock()
724 			bzero(tpheur->th_val_start,  in tcp_getheuristic_with_lock()
725 			    tpheur->th_val_end - tpheur->th_val_start);  in tcp_getheuristic_with_lock()
728 			tpheur = kalloc_type(struct tcp_heuristic, Z_NOWAIT | Z_ZERO);  in tcp_getheuristic_with_lock()
729 			if (tpheur == NULL) {  in tcp_getheuristic_with_lock()
734 			SLIST_INSERT_HEAD(&head->tcp_heuristics, tpheur, list);  in tcp_getheuristic_with_lock()
741 		tpheur->th_ecn_backoff = tcp_now;  in tcp_getheuristic_with_lock()
742 		tpheur->th_tfo_backoff_until = tcp_now;  in tcp_getheuristic_with_lock()
743 		tpheur->th_mptcp_backoff = tcp_now;  in tcp_getheuristic_with_lock()
744 		tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in tcp_getheuristic_with_lock()
746 		memcpy(&tpheur->th_key, &key, sizeof(key));  in tcp_getheuristic_with_lock()
749 	if (tpheur == NULL) {  in tcp_getheuristic_with_lock()
754 	tpheur->th_last_access = tcp_now;  in tcp_getheuristic_with_lock()
757 	return tpheur;  in tcp_getheuristic_with_lock()
768 	struct tcp_heuristic *tpheur;  in tcp_heuristic_reset_counters()  local
774 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_reset_counters()
775 	if (tpheur == NULL) {  in tcp_heuristic_reset_counters()
780 		if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
782 			    __func__, tpheur->th_tfo_data_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
784 		tpheur->th_tfo_data_loss = 0;  in tcp_heuristic_reset_counters()
788 		if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
790 			    __func__, tpheur->th_tfo_req_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
792 		tpheur->th_tfo_req_loss = 0;  in tcp_heuristic_reset_counters()
796 		if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
798 			    __func__, tpheur->th_tfo_data_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
800 		tpheur->th_tfo_data_rst = 0;  in tcp_heuristic_reset_counters()
804 		if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
806 			    __func__, tpheur->th_tfo_req_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
808 		tpheur->th_tfo_req_rst = 0;  in tcp_heuristic_reset_counters()
812 		if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS || tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) {  in tcp_heuristic_reset_counters()
814 …  __func__, tpheur->th_ecn_loss, tpheur->th_ecn_synrst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
816 		tpheur->th_ecn_loss = 0;  in tcp_heuristic_reset_counters()
817 		tpheur->th_ecn_synrst = 0;  in tcp_heuristic_reset_counters()
821 		tpheur->th_mptcp_loss = 0;  in tcp_heuristic_reset_counters()
822 		if (tpheur->th_mptcp_success < MPTCP_SUCCESS_TRIGGER) {  in tcp_heuristic_reset_counters()
823 			tpheur->th_mptcp_success++;  in tcp_heuristic_reset_counters()
825 			if (tpheur->th_mptcp_success == MPTCP_SUCCESS_TRIGGER) {  in tcp_heuristic_reset_counters()
827 				tpheur->th_mptcp_heuristic_disabled = 1;  in tcp_heuristic_reset_counters()
829 				tpheur->th_mptcp_backoff = tcp_now + tcp_min_to_hz(tcp_ecn_timeout * 12);  in tcp_heuristic_reset_counters()
875 __tcp_heuristic_tfo_middlebox_common(struct tcp_heuristic *tpheur)  in __tcp_heuristic_tfo_middlebox_common()  argument
877 	if (tpheur->th_tfo_in_backoff) {  in __tcp_heuristic_tfo_middlebox_common()
881 	tpheur->th_tfo_in_backoff = 1;  in __tcp_heuristic_tfo_middlebox_common()
883 	if (tpheur->th_tfo_enabled_time) {  in __tcp_heuristic_tfo_middlebox_common()
884 		uint32_t old_backoff = tpheur->th_tfo_backoff;  in __tcp_heuristic_tfo_middlebox_common()
886 		tpheur->th_tfo_backoff -= (tcp_now - tpheur->th_tfo_enabled_time);  in __tcp_heuristic_tfo_middlebox_common()
887 		if (tpheur->th_tfo_backoff > old_backoff) {  in __tcp_heuristic_tfo_middlebox_common()
888 			tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in __tcp_heuristic_tfo_middlebox_common()
892 	tpheur->th_tfo_backoff_until = tcp_now + tpheur->th_tfo_backoff;  in __tcp_heuristic_tfo_middlebox_common()
895 	tpheur->th_tfo_backoff *= 2;  in __tcp_heuristic_tfo_middlebox_common()
897 	if (tpheur->th_tfo_backoff > tcp_min_to_hz(tcp_backoff_maximum)) {  in __tcp_heuristic_tfo_middlebox_common()
898 		tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in __tcp_heuristic_tfo_middlebox_common()
902 	    tpheur->th_tfo_backoff_until, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in __tcp_heuristic_tfo_middlebox_common()
909 	struct tcp_heuristic *tpheur;  in tcp_heuristic_tfo_middlebox_common()  local
911 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_tfo_middlebox_common()
912 	if (tpheur == NULL) {  in tcp_heuristic_tfo_middlebox_common()
916 	__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_tfo_middlebox_common()
926 	struct tcp_heuristic *tpheur;  in tcp_heuristic_inc_counters()  local
928 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_inc_counters()
929 	if (tpheur == NULL) {  in tcp_heuristic_inc_counters()
934 	if ((flags & TCPCACHE_F_TFO_DATA) && tpheur->th_tfo_data_loss < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
935 		tpheur->th_tfo_data_loss++;  in tcp_heuristic_inc_counters()
937 		if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
938 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
942 	if ((flags & TCPCACHE_F_TFO_REQ) && tpheur->th_tfo_req_loss < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
943 		tpheur->th_tfo_req_loss++;  in tcp_heuristic_inc_counters()
945 		if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
946 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
950 	if ((flags & TCPCACHE_F_TFO_DATA_RST) && tpheur->th_tfo_data_rst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
951 		tpheur->th_tfo_data_rst++;  in tcp_heuristic_inc_counters()
953 		if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
954 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
958 	if ((flags & TCPCACHE_F_TFO_REQ_RST) && tpheur->th_tfo_req_rst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
959 		tpheur->th_tfo_req_rst++;  in tcp_heuristic_inc_counters()
961 		if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
962 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
967 	    tpheur->th_ecn_loss < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
968 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
969 		tpheur->th_ecn_loss++;  in tcp_heuristic_inc_counters()
970 		if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS) {  in tcp_heuristic_inc_counters()
973 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
975 			    (tpheur->th_ecn_loss - ECN_MAX_SYN_LOSS));  in tcp_heuristic_inc_counters()
978 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
979 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
984 	    tpheur->th_mptcp_loss < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
985 	    tpheur->th_mptcp_heuristic_disabled == 0) {  in tcp_heuristic_inc_counters()
986 		tpheur->th_mptcp_loss++;  in tcp_heuristic_inc_counters()
987 		if (tpheur->th_mptcp_loss >= MPTCP_MAX_SYN_LOSS) {  in tcp_heuristic_inc_counters()
992 			tpheur->th_mptcp_backoff = tcp_now +  in tcp_heuristic_inc_counters()
994 			    (tpheur->th_mptcp_loss - MPTCP_MAX_SYN_LOSS));  in tcp_heuristic_inc_counters()
995 			tpheur->th_mptcp_in_backoff = 1;  in tcp_heuristic_inc_counters()
998 			    __func__, tpheur->th_mptcp_backoff, tcp_now,  in tcp_heuristic_inc_counters()
999 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1004 	    tpheur->th_ecn_droprst < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
1005 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
1006 		tpheur->th_ecn_droprst++;  in tcp_heuristic_inc_counters()
1007 		if (tpheur->th_ecn_droprst >= ECN_MAX_DROPRST) {  in tcp_heuristic_inc_counters()
1011 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1013 			    (tpheur->th_ecn_droprst - ECN_MAX_DROPRST));  in tcp_heuristic_inc_counters()
1016 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1017 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1022 	    tpheur->th_ecn_droprxmt < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
1023 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
1024 		tpheur->th_ecn_droprxmt++;  in tcp_heuristic_inc_counters()
1025 		if (tpheur->th_ecn_droprxmt >= ECN_MAX_DROPRXMT) {  in tcp_heuristic_inc_counters()
1029 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1031 			    (tpheur->th_ecn_droprxmt - ECN_MAX_DROPRXMT));  in tcp_heuristic_inc_counters()
1034 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1035 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1039 	    tpheur->th_ecn_synrst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
1040 		tpheur->th_ecn_synrst++;  in tcp_heuristic_inc_counters()
1041 		if (tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) {  in tcp_heuristic_inc_counters()
1045 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1047 			    (tpheur->th_ecn_synrst - ECN_MAX_SYNRST));  in tcp_heuristic_inc_counters()
1050 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1051 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1173 	struct tcp_heuristic *tpheur;  in tcp_heuristic_ecn_aggressive_common()  local
1175 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_ecn_aggressive_common()
1176 	if (tpheur == NULL) {  in tcp_heuristic_ecn_aggressive_common()
1180 	if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_ecn_aggressive_common()
1187 	tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_ecn_aggressive_common()
1188 	    (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_aggressive));  in tcp_heuristic_ecn_aggressive_common()
1194 	if (tpheur->th_ecn_aggressive < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_ecn_aggressive_common()
1195 		tpheur->th_ecn_aggressive++;  in tcp_heuristic_ecn_aggressive_common()
1201 	    tpheur->th_ecn_backoff, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_ecn_aggressive_common()
1217 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_tfo_common()  local
1224 	tpheur = tcp_getheuristic_with_lock(tcks, 0, &head);  in tcp_heuristic_do_tfo_common()
1225 	if (tpheur == NULL) {  in tcp_heuristic_do_tfo_common()
1229 	if (tpheur->th_tfo_in_backoff == 0) {  in tcp_heuristic_do_tfo_common()
1233 	if (TSTMP_GT(tcp_now, tpheur->th_tfo_backoff_until)) {  in tcp_heuristic_do_tfo_common()
1234 		tpheur->th_tfo_in_backoff = 0;  in tcp_heuristic_do_tfo_common()
1235 		tpheur->th_tfo_enabled_time = tcp_now;  in tcp_heuristic_do_tfo_common()
1271 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_mptcp()  local
1282 	tpheur = tcp_getheuristic_with_lock(&tcks, 0, &head);  in tcp_heuristic_do_mptcp()
1283 	if (tpheur == NULL) {  in tcp_heuristic_do_mptcp()
1287 	if (tpheur->th_mptcp_in_backoff == 0 ||  in tcp_heuristic_do_mptcp()
1288 	    tpheur->th_mptcp_heuristic_disabled == 1) {  in tcp_heuristic_do_mptcp()
1292 	if (TSTMP_GT(tpheur->th_mptcp_backoff, tcp_now)) {  in tcp_heuristic_do_mptcp()
1296 	tpheur->th_mptcp_in_backoff = 0;  in tcp_heuristic_do_mptcp()
1299 	if (tpheur->th_mptcp_heuristic_disabled) {  in tcp_heuristic_do_mptcp()
1302 		if (TSTMP_GT(tcp_now, tpheur->th_mptcp_backoff)) {  in tcp_heuristic_do_mptcp()
1303 			tpheur->th_mptcp_heuristic_disabled = 0;  in tcp_heuristic_do_mptcp()
1304 			tpheur->th_mptcp_success = 0;  in tcp_heuristic_do_mptcp()
1329 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_ecn_common()  local
1337 	tpheur = tcp_getheuristic_with_lock(tcks, 0, &head);  in tcp_heuristic_do_ecn_common()
1338 	if (tpheur == NULL) {  in tcp_heuristic_do_ecn_common()
1342 	if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_do_ecn_common()
1346 		if (tpheur->th_ecn_droprst >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1347 			tpheur->th_ecn_droprst = 0;  in tcp_heuristic_do_ecn_common()
1349 		if (tpheur->th_ecn_droprxmt >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1350 			tpheur->th_ecn_droprxmt = 0;  in tcp_heuristic_do_ecn_common()
1352 		if (tpheur->th_ecn_synrst >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1353 			tpheur->th_ecn_synrst = 0;  in tcp_heuristic_do_ecn_common()
1357 		tpheur->th_ecn_backoff = tcp_now;  in tcp_heuristic_do_ecn_common()
1532 		struct tcp_heuristic *tpheur, *htmp;  in sysctl_cleartfocache()  local
1542 		SLIST_FOREACH_SAFE(tpheur, &hhead->tcp_heuristics, list, htmp) {  in sysctl_cleartfocache()
1543 			SLIST_REMOVE(&hhead->tcp_heuristics, tpheur, tcp_heuristic, list);  in sysctl_cleartfocache()
1544 			kfree_type(struct tcp_heuristic, tpheur);  in sysctl_cleartfocache()