686 	struct tcp_heuristic *tpheur = NULL;  in tcp_getheuristic_with_lock()  local
698 	SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) {  in tcp_getheuristic_with_lock()
699 		if (memcmp(&tpheur->th_key, &key, sizeof(key)) == 0) {  in tcp_getheuristic_with_lock()
707 	if ((tpheur == NULL) && create) {  in tcp_getheuristic_with_lock()
713 			SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) {  in tcp_getheuristic_with_lock()
714 				uint32_t age = tcp_now - tpheur->th_last_access;  in tcp_getheuristic_with_lock()
717 					oldest_heur = tpheur;  in tcp_getheuristic_with_lock()
722 			tpheur = oldest_heur;  in tcp_getheuristic_with_lock()
725 			bzero(tpheur->th_val_start,  in tcp_getheuristic_with_lock()
726 			    tpheur->th_val_end - tpheur->th_val_start);  in tcp_getheuristic_with_lock()
729 			tpheur = kalloc_type(struct tcp_heuristic, Z_NOPAGEWAIT | Z_ZERO);  in tcp_getheuristic_with_lock()
730 			if (tpheur == NULL) {  in tcp_getheuristic_with_lock()
735 			SLIST_INSERT_HEAD(&head->tcp_heuristics, tpheur, list);  in tcp_getheuristic_with_lock()
742 		tpheur->th_ecn_backoff = tcp_now;  in tcp_getheuristic_with_lock()
743 		tpheur->th_tfo_backoff_until = tcp_now;  in tcp_getheuristic_with_lock()
744 		tpheur->th_mptcp_backoff = tcp_now;  in tcp_getheuristic_with_lock()
745 		tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in tcp_getheuristic_with_lock()
747 		memcpy(&tpheur->th_key, &key, sizeof(key));  in tcp_getheuristic_with_lock()
750 	if (tpheur == NULL) {  in tcp_getheuristic_with_lock()
755 	tpheur->th_last_access = tcp_now;  in tcp_getheuristic_with_lock()
758 	return tpheur;  in tcp_getheuristic_with_lock()
769 	struct tcp_heuristic *tpheur;  in tcp_heuristic_reset_counters()  local
775 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_reset_counters()
776 	if (tpheur == NULL) {  in tcp_heuristic_reset_counters()
781 		if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
783 			    __func__, tpheur->th_tfo_data_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
785 		tpheur->th_tfo_data_loss = 0;  in tcp_heuristic_reset_counters()
789 		if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
791 			    __func__, tpheur->th_tfo_req_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
793 		tpheur->th_tfo_req_loss = 0;  in tcp_heuristic_reset_counters()
797 		if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
799 			    __func__, tpheur->th_tfo_data_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
801 		tpheur->th_tfo_data_rst = 0;  in tcp_heuristic_reset_counters()
805 		if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
807 			    __func__, tpheur->th_tfo_req_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
809 		tpheur->th_tfo_req_rst = 0;  in tcp_heuristic_reset_counters()
813 		if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS || tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) {  in tcp_heuristic_reset_counters()
815 …  __func__, tpheur->th_ecn_loss, tpheur->th_ecn_synrst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
817 		tpheur->th_ecn_loss = 0;  in tcp_heuristic_reset_counters()
818 		tpheur->th_ecn_synrst = 0;  in tcp_heuristic_reset_counters()
822 		tpheur->th_mptcp_loss = 0;  in tcp_heuristic_reset_counters()
823 		if (tpheur->th_mptcp_success < MPTCP_SUCCESS_TRIGGER) {  in tcp_heuristic_reset_counters()
824 			tpheur->th_mptcp_success++;  in tcp_heuristic_reset_counters()
826 			if (tpheur->th_mptcp_success == MPTCP_SUCCESS_TRIGGER) {  in tcp_heuristic_reset_counters()
828 				tpheur->th_mptcp_heuristic_disabled = 1;  in tcp_heuristic_reset_counters()
830 				tpheur->th_mptcp_backoff = tcp_now + tcp_min_to_hz(tcp_ecn_timeout * 12);  in tcp_heuristic_reset_counters()
876 __tcp_heuristic_tfo_middlebox_common(struct tcp_heuristic *tpheur)  in __tcp_heuristic_tfo_middlebox_common()  argument
878 	if (tpheur->th_tfo_in_backoff) {  in __tcp_heuristic_tfo_middlebox_common()
882 	tpheur->th_tfo_in_backoff = 1;  in __tcp_heuristic_tfo_middlebox_common()
884 	if (tpheur->th_tfo_enabled_time) {  in __tcp_heuristic_tfo_middlebox_common()
885 		uint32_t old_backoff = tpheur->th_tfo_backoff;  in __tcp_heuristic_tfo_middlebox_common()
887 		tpheur->th_tfo_backoff -= (tcp_now - tpheur->th_tfo_enabled_time);  in __tcp_heuristic_tfo_middlebox_common()
888 		if (tpheur->th_tfo_backoff > old_backoff) {  in __tcp_heuristic_tfo_middlebox_common()
889 			tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in __tcp_heuristic_tfo_middlebox_common()
893 	tpheur->th_tfo_backoff_until = tcp_now + tpheur->th_tfo_backoff;  in __tcp_heuristic_tfo_middlebox_common()
896 	tpheur->th_tfo_backoff *= 2;  in __tcp_heuristic_tfo_middlebox_common()
898 	if (tpheur->th_tfo_backoff > tcp_min_to_hz(tcp_backoff_maximum)) {  in __tcp_heuristic_tfo_middlebox_common()
899 		tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in __tcp_heuristic_tfo_middlebox_common()
903 	    tpheur->th_tfo_backoff_until, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in __tcp_heuristic_tfo_middlebox_common()
910 	struct tcp_heuristic *tpheur;  in tcp_heuristic_tfo_middlebox_common()  local
912 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_tfo_middlebox_common()
913 	if (tpheur == NULL) {  in tcp_heuristic_tfo_middlebox_common()
917 	__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_tfo_middlebox_common()
927 	struct tcp_heuristic *tpheur;  in tcp_heuristic_inc_counters()  local
929 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_inc_counters()
930 	if (tpheur == NULL) {  in tcp_heuristic_inc_counters()
935 	if ((flags & TCPCACHE_F_TFO_DATA) && tpheur->th_tfo_data_loss < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
936 		tpheur->th_tfo_data_loss++;  in tcp_heuristic_inc_counters()
938 		if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
939 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
943 	if ((flags & TCPCACHE_F_TFO_REQ) && tpheur->th_tfo_req_loss < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
944 		tpheur->th_tfo_req_loss++;  in tcp_heuristic_inc_counters()
946 		if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
947 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
951 	if ((flags & TCPCACHE_F_TFO_DATA_RST) && tpheur->th_tfo_data_rst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
952 		tpheur->th_tfo_data_rst++;  in tcp_heuristic_inc_counters()
954 		if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
955 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
959 	if ((flags & TCPCACHE_F_TFO_REQ_RST) && tpheur->th_tfo_req_rst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
960 		tpheur->th_tfo_req_rst++;  in tcp_heuristic_inc_counters()
962 		if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
963 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
968 	    tpheur->th_ecn_loss < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
969 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
970 		tpheur->th_ecn_loss++;  in tcp_heuristic_inc_counters()
971 		if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS) {  in tcp_heuristic_inc_counters()
974 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
976 			    (tpheur->th_ecn_loss - ECN_MAX_SYN_LOSS));  in tcp_heuristic_inc_counters()
979 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
980 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
985 	    tpheur->th_mptcp_loss < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
986 	    tpheur->th_mptcp_heuristic_disabled == 0) {  in tcp_heuristic_inc_counters()
987 		tpheur->th_mptcp_loss++;  in tcp_heuristic_inc_counters()
988 		if (tpheur->th_mptcp_loss >= MPTCP_MAX_SYN_LOSS) {  in tcp_heuristic_inc_counters()
993 			tpheur->th_mptcp_backoff = tcp_now +  in tcp_heuristic_inc_counters()
995 			    (tpheur->th_mptcp_loss - MPTCP_MAX_SYN_LOSS));  in tcp_heuristic_inc_counters()
996 			tpheur->th_mptcp_in_backoff = 1;  in tcp_heuristic_inc_counters()
999 			    __func__, tpheur->th_mptcp_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1000 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1005 	    tpheur->th_ecn_droprst < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
1006 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
1007 		tpheur->th_ecn_droprst++;  in tcp_heuristic_inc_counters()
1008 		if (tpheur->th_ecn_droprst >= ECN_MAX_DROPRST) {  in tcp_heuristic_inc_counters()
1012 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1014 			    (tpheur->th_ecn_droprst - ECN_MAX_DROPRST));  in tcp_heuristic_inc_counters()
1017 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1018 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1023 	    tpheur->th_ecn_droprxmt < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
1024 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
1025 		tpheur->th_ecn_droprxmt++;  in tcp_heuristic_inc_counters()
1026 		if (tpheur->th_ecn_droprxmt >= ECN_MAX_DROPRXMT) {  in tcp_heuristic_inc_counters()
1030 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1032 			    (tpheur->th_ecn_droprxmt - ECN_MAX_DROPRXMT));  in tcp_heuristic_inc_counters()
1035 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1036 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1040 	    tpheur->th_ecn_synrst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
1041 		tpheur->th_ecn_synrst++;  in tcp_heuristic_inc_counters()
1042 		if (tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) {  in tcp_heuristic_inc_counters()
1046 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1048 			    (tpheur->th_ecn_synrst - ECN_MAX_SYNRST));  in tcp_heuristic_inc_counters()
1051 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1052 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1174 	struct tcp_heuristic *tpheur;  in tcp_heuristic_ecn_aggressive_common()  local
1176 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_ecn_aggressive_common()
1177 	if (tpheur == NULL) {  in tcp_heuristic_ecn_aggressive_common()
1181 	if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_ecn_aggressive_common()
1188 	tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_ecn_aggressive_common()
1189 	    (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_aggressive));  in tcp_heuristic_ecn_aggressive_common()
1195 	if (tpheur->th_ecn_aggressive < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_ecn_aggressive_common()
1196 		tpheur->th_ecn_aggressive++;  in tcp_heuristic_ecn_aggressive_common()
1202 	    tpheur->th_ecn_backoff, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_ecn_aggressive_common()
1218 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_tfo_common()  local
1225 	tpheur = tcp_getheuristic_with_lock(tcks, 0, &head);  in tcp_heuristic_do_tfo_common()
1226 	if (tpheur == NULL) {  in tcp_heuristic_do_tfo_common()
1230 	if (tpheur->th_tfo_in_backoff == 0) {  in tcp_heuristic_do_tfo_common()
1234 	if (TSTMP_GT(tcp_now, tpheur->th_tfo_backoff_until)) {  in tcp_heuristic_do_tfo_common()
1235 		tpheur->th_tfo_in_backoff = 0;  in tcp_heuristic_do_tfo_common()
1236 		tpheur->th_tfo_enabled_time = tcp_now;  in tcp_heuristic_do_tfo_common()
1272 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_mptcp()  local
1283 	tpheur = tcp_getheuristic_with_lock(&tcks, 0, &head);  in tcp_heuristic_do_mptcp()
1284 	if (tpheur == NULL) {  in tcp_heuristic_do_mptcp()
1288 	if (tpheur->th_mptcp_in_backoff == 0 ||  in tcp_heuristic_do_mptcp()
1289 	    tpheur->th_mptcp_heuristic_disabled == 1) {  in tcp_heuristic_do_mptcp()
1293 	if (TSTMP_GT(tpheur->th_mptcp_backoff, tcp_now)) {  in tcp_heuristic_do_mptcp()
1297 	tpheur->th_mptcp_in_backoff = 0;  in tcp_heuristic_do_mptcp()
1300 	if (tpheur->th_mptcp_heuristic_disabled) {  in tcp_heuristic_do_mptcp()
1303 		if (TSTMP_GT(tcp_now, tpheur->th_mptcp_backoff)) {  in tcp_heuristic_do_mptcp()
1304 			tpheur->th_mptcp_heuristic_disabled = 0;  in tcp_heuristic_do_mptcp()
1305 			tpheur->th_mptcp_success = 0;  in tcp_heuristic_do_mptcp()
1330 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_ecn_common()  local
1338 	tpheur = tcp_getheuristic_with_lock(tcks, 0, &head);  in tcp_heuristic_do_ecn_common()
1339 	if (tpheur == NULL) {  in tcp_heuristic_do_ecn_common()
1343 	if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_do_ecn_common()
1347 		if (tpheur->th_ecn_droprst >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1348 			tpheur->th_ecn_droprst = 0;  in tcp_heuristic_do_ecn_common()
1350 		if (tpheur->th_ecn_droprxmt >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1351 			tpheur->th_ecn_droprxmt = 0;  in tcp_heuristic_do_ecn_common()
1353 		if (tpheur->th_ecn_synrst >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1354 			tpheur->th_ecn_synrst = 0;  in tcp_heuristic_do_ecn_common()
1358 		tpheur->th_ecn_backoff = tcp_now;  in tcp_heuristic_do_ecn_common()
1533 		struct tcp_heuristic *tpheur, *htmp;  in sysctl_cleartfocache()  local
1543 		SLIST_FOREACH_SAFE(tpheur, &hhead->tcp_heuristics, list, htmp) {  in sysctl_cleartfocache()
1544 			SLIST_REMOVE(&hhead->tcp_heuristics, tpheur, tcp_heuristic, list);  in sysctl_cleartfocache()
1545 			kfree_type(struct tcp_heuristic, tpheur);  in sysctl_cleartfocache()