690 	struct tcp_heuristic *tpheur = NULL;  in tcp_getheuristic_with_lock()  local
702 	SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) {  in tcp_getheuristic_with_lock()
703 		if (memcmp(&tpheur->th_key, &key, sizeof(key)) == 0) {  in tcp_getheuristic_with_lock()
711 	if ((tpheur == NULL) && create) {  in tcp_getheuristic_with_lock()
717 			SLIST_FOREACH(tpheur, &head->tcp_heuristics, list) {  in tcp_getheuristic_with_lock()
718 				uint32_t age = tcp_now - tpheur->th_last_access;  in tcp_getheuristic_with_lock()
721 					oldest_heur = tpheur;  in tcp_getheuristic_with_lock()
726 			tpheur = oldest_heur;  in tcp_getheuristic_with_lock()
729 			bzero(tpheur->th_val_start,  in tcp_getheuristic_with_lock()
730 			    tpheur->th_val_end - tpheur->th_val_start);  in tcp_getheuristic_with_lock()
733 			tpheur = kalloc_type(struct tcp_heuristic, Z_NOWAIT | Z_ZERO);  in tcp_getheuristic_with_lock()
734 			if (tpheur == NULL) {  in tcp_getheuristic_with_lock()
739 			SLIST_INSERT_HEAD(&head->tcp_heuristics, tpheur, list);  in tcp_getheuristic_with_lock()
746 		tpheur->th_ecn_backoff = tcp_now;  in tcp_getheuristic_with_lock()
747 		tpheur->th_tfo_backoff_until = tcp_now;  in tcp_getheuristic_with_lock()
748 		tpheur->th_mptcp_backoff = tcp_now;  in tcp_getheuristic_with_lock()
749 		tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in tcp_getheuristic_with_lock()
751 		memcpy(&tpheur->th_key, &key, sizeof(key));  in tcp_getheuristic_with_lock()
754 	if (tpheur == NULL) {  in tcp_getheuristic_with_lock()
759 	tpheur->th_last_access = tcp_now;  in tcp_getheuristic_with_lock()
762 	return tpheur;  in tcp_getheuristic_with_lock()
773 	struct tcp_heuristic *tpheur;  in tcp_heuristic_reset_counters()  local
779 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_reset_counters()
780 	if (tpheur == NULL) {  in tcp_heuristic_reset_counters()
785 		if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
787 			    __func__, tpheur->th_tfo_data_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
789 		tpheur->th_tfo_data_loss = 0;  in tcp_heuristic_reset_counters()
793 		if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
795 			    __func__, tpheur->th_tfo_req_loss, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
797 		tpheur->th_tfo_req_loss = 0;  in tcp_heuristic_reset_counters()
801 		if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
803 			    __func__, tpheur->th_tfo_data_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
805 		tpheur->th_tfo_data_rst = 0;  in tcp_heuristic_reset_counters()
809 		if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_reset_counters()
811 			    __func__, tpheur->th_tfo_req_rst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
813 		tpheur->th_tfo_req_rst = 0;  in tcp_heuristic_reset_counters()
817 		if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS || tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) {  in tcp_heuristic_reset_counters()
819 …  __func__, tpheur->th_ecn_loss, tpheur->th_ecn_synrst, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_reset_counters()
821 		tpheur->th_ecn_loss = 0;  in tcp_heuristic_reset_counters()
822 		tpheur->th_ecn_synrst = 0;  in tcp_heuristic_reset_counters()
826 		tpheur->th_mptcp_loss = 0;  in tcp_heuristic_reset_counters()
827 		if (tpheur->th_mptcp_success < MPTCP_SUCCESS_TRIGGER) {  in tcp_heuristic_reset_counters()
828 			tpheur->th_mptcp_success++;  in tcp_heuristic_reset_counters()
830 			if (tpheur->th_mptcp_success == MPTCP_SUCCESS_TRIGGER) {  in tcp_heuristic_reset_counters()
832 				tpheur->th_mptcp_heuristic_disabled = 1;  in tcp_heuristic_reset_counters()
834 				tpheur->th_mptcp_backoff = tcp_now + tcp_min_to_hz(tcp_ecn_timeout * 12);  in tcp_heuristic_reset_counters()
880 __tcp_heuristic_tfo_middlebox_common(struct tcp_heuristic *tpheur)  in __tcp_heuristic_tfo_middlebox_common()  argument
882 	if (tpheur->th_tfo_in_backoff) {  in __tcp_heuristic_tfo_middlebox_common()
886 	tpheur->th_tfo_in_backoff = 1;  in __tcp_heuristic_tfo_middlebox_common()
888 	if (tpheur->th_tfo_enabled_time) {  in __tcp_heuristic_tfo_middlebox_common()
889 		uint32_t old_backoff = tpheur->th_tfo_backoff;  in __tcp_heuristic_tfo_middlebox_common()
891 		tpheur->th_tfo_backoff -= (tcp_now - tpheur->th_tfo_enabled_time);  in __tcp_heuristic_tfo_middlebox_common()
892 		if (tpheur->th_tfo_backoff > old_backoff) {  in __tcp_heuristic_tfo_middlebox_common()
893 			tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in __tcp_heuristic_tfo_middlebox_common()
897 	tpheur->th_tfo_backoff_until = tcp_now + tpheur->th_tfo_backoff;  in __tcp_heuristic_tfo_middlebox_common()
900 	tpheur->th_tfo_backoff *= 2;  in __tcp_heuristic_tfo_middlebox_common()
902 	if (tpheur->th_tfo_backoff > tcp_min_to_hz(tcp_backoff_maximum)) {  in __tcp_heuristic_tfo_middlebox_common()
903 		tpheur->th_tfo_backoff = tcp_min_to_hz(tcp_ecn_timeout);  in __tcp_heuristic_tfo_middlebox_common()
907 	    tpheur->th_tfo_backoff_until, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in __tcp_heuristic_tfo_middlebox_common()
914 	struct tcp_heuristic *tpheur;  in tcp_heuristic_tfo_middlebox_common()  local
916 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_tfo_middlebox_common()
917 	if (tpheur == NULL) {  in tcp_heuristic_tfo_middlebox_common()
921 	__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_tfo_middlebox_common()
931 	struct tcp_heuristic *tpheur;  in tcp_heuristic_inc_counters()  local
933 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_inc_counters()
934 	if (tpheur == NULL) {  in tcp_heuristic_inc_counters()
939 	if ((flags & TCPCACHE_F_TFO_DATA) && tpheur->th_tfo_data_loss < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
940 		tpheur->th_tfo_data_loss++;  in tcp_heuristic_inc_counters()
942 		if (tpheur->th_tfo_data_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
943 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
947 	if ((flags & TCPCACHE_F_TFO_REQ) && tpheur->th_tfo_req_loss < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
948 		tpheur->th_tfo_req_loss++;  in tcp_heuristic_inc_counters()
950 		if (tpheur->th_tfo_req_loss >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
951 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
955 	if ((flags & TCPCACHE_F_TFO_DATA_RST) && tpheur->th_tfo_data_rst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
956 		tpheur->th_tfo_data_rst++;  in tcp_heuristic_inc_counters()
958 		if (tpheur->th_tfo_data_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
959 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
963 	if ((flags & TCPCACHE_F_TFO_REQ_RST) && tpheur->th_tfo_req_rst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
964 		tpheur->th_tfo_req_rst++;  in tcp_heuristic_inc_counters()
966 		if (tpheur->th_tfo_req_rst >= TFO_MAX_COOKIE_LOSS) {  in tcp_heuristic_inc_counters()
967 			__tcp_heuristic_tfo_middlebox_common(tpheur);  in tcp_heuristic_inc_counters()
972 	    tpheur->th_ecn_loss < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
973 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
974 		tpheur->th_ecn_loss++;  in tcp_heuristic_inc_counters()
975 		if (tpheur->th_ecn_loss >= ECN_MAX_SYN_LOSS) {  in tcp_heuristic_inc_counters()
978 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
980 			    (tpheur->th_ecn_loss - ECN_MAX_SYN_LOSS));  in tcp_heuristic_inc_counters()
983 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
984 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
989 	    tpheur->th_mptcp_loss < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
990 	    tpheur->th_mptcp_heuristic_disabled == 0) {  in tcp_heuristic_inc_counters()
991 		tpheur->th_mptcp_loss++;  in tcp_heuristic_inc_counters()
992 		if (tpheur->th_mptcp_loss >= MPTCP_MAX_SYN_LOSS) {  in tcp_heuristic_inc_counters()
997 			tpheur->th_mptcp_backoff = tcp_now +  in tcp_heuristic_inc_counters()
999 			    (tpheur->th_mptcp_loss - MPTCP_MAX_SYN_LOSS));  in tcp_heuristic_inc_counters()
1000 			tpheur->th_mptcp_in_backoff = 1;  in tcp_heuristic_inc_counters()
1003 			    __func__, tpheur->th_mptcp_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1004 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1009 	    tpheur->th_ecn_droprst < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
1010 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
1011 		tpheur->th_ecn_droprst++;  in tcp_heuristic_inc_counters()
1012 		if (tpheur->th_ecn_droprst >= ECN_MAX_DROPRST) {  in tcp_heuristic_inc_counters()
1016 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1018 			    (tpheur->th_ecn_droprst - ECN_MAX_DROPRST));  in tcp_heuristic_inc_counters()
1021 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1022 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1027 	    tpheur->th_ecn_droprxmt < TCP_CACHE_OVERFLOW_PROTECT &&  in tcp_heuristic_inc_counters()
1028 	    TSTMP_LEQ(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_inc_counters()
1029 		tpheur->th_ecn_droprxmt++;  in tcp_heuristic_inc_counters()
1030 		if (tpheur->th_ecn_droprxmt >= ECN_MAX_DROPRXMT) {  in tcp_heuristic_inc_counters()
1034 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1036 			    (tpheur->th_ecn_droprxmt - ECN_MAX_DROPRXMT));  in tcp_heuristic_inc_counters()
1039 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1040 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1044 	    tpheur->th_ecn_synrst < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_inc_counters()
1045 		tpheur->th_ecn_synrst++;  in tcp_heuristic_inc_counters()
1046 		if (tpheur->th_ecn_synrst >= ECN_MAX_SYNRST) {  in tcp_heuristic_inc_counters()
1050 			tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_inc_counters()
1052 			    (tpheur->th_ecn_synrst - ECN_MAX_SYNRST));  in tcp_heuristic_inc_counters()
1055 			    __func__, tpheur->th_ecn_backoff, tcp_now,  in tcp_heuristic_inc_counters()
1056 			    (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_inc_counters()
1178 	struct tcp_heuristic *tpheur;  in tcp_heuristic_ecn_aggressive_common()  local
1180 	tpheur = tcp_getheuristic_with_lock(tcks, 1, &head);  in tcp_heuristic_ecn_aggressive_common()
1181 	if (tpheur == NULL) {  in tcp_heuristic_ecn_aggressive_common()
1185 	if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_ecn_aggressive_common()
1192 	tpheur->th_ecn_backoff = tcp_now +  in tcp_heuristic_ecn_aggressive_common()
1193 	    (tcp_min_to_hz(tcp_ecn_timeout) << (tpheur->th_ecn_aggressive));  in tcp_heuristic_ecn_aggressive_common()
1199 	if (tpheur->th_ecn_aggressive < TCP_CACHE_OVERFLOW_PROTECT) {  in tcp_heuristic_ecn_aggressive_common()
1200 		tpheur->th_ecn_aggressive++;  in tcp_heuristic_ecn_aggressive_common()
1206 	    tpheur->th_ecn_backoff, tcp_now, (unsigned long)VM_KERNEL_ADDRPERM(tpheur));  in tcp_heuristic_ecn_aggressive_common()
1222 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_tfo_common()  local
1229 	tpheur = tcp_getheuristic_with_lock(tcks, 0, &head);  in tcp_heuristic_do_tfo_common()
1230 	if (tpheur == NULL) {  in tcp_heuristic_do_tfo_common()
1234 	if (tpheur->th_tfo_in_backoff == 0) {  in tcp_heuristic_do_tfo_common()
1238 	if (TSTMP_GT(tcp_now, tpheur->th_tfo_backoff_until)) {  in tcp_heuristic_do_tfo_common()
1239 		tpheur->th_tfo_in_backoff = 0;  in tcp_heuristic_do_tfo_common()
1240 		tpheur->th_tfo_enabled_time = tcp_now;  in tcp_heuristic_do_tfo_common()
1276 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_mptcp()  local
1287 	tpheur = tcp_getheuristic_with_lock(&tcks, 0, &head);  in tcp_heuristic_do_mptcp()
1288 	if (tpheur == NULL) {  in tcp_heuristic_do_mptcp()
1292 	if (tpheur->th_mptcp_in_backoff == 0 ||  in tcp_heuristic_do_mptcp()
1293 	    tpheur->th_mptcp_heuristic_disabled == 1) {  in tcp_heuristic_do_mptcp()
1297 	if (TSTMP_GT(tpheur->th_mptcp_backoff, tcp_now)) {  in tcp_heuristic_do_mptcp()
1301 	tpheur->th_mptcp_in_backoff = 0;  in tcp_heuristic_do_mptcp()
1304 	if (tpheur->th_mptcp_heuristic_disabled) {  in tcp_heuristic_do_mptcp()
1307 		if (TSTMP_GT(tcp_now, tpheur->th_mptcp_backoff)) {  in tcp_heuristic_do_mptcp()
1308 			tpheur->th_mptcp_heuristic_disabled = 0;  in tcp_heuristic_do_mptcp()
1309 			tpheur->th_mptcp_success = 0;  in tcp_heuristic_do_mptcp()
1334 	struct tcp_heuristic *tpheur;  in tcp_heuristic_do_ecn_common()  local
1342 	tpheur = tcp_getheuristic_with_lock(tcks, 0, &head);  in tcp_heuristic_do_ecn_common()
1343 	if (tpheur == NULL) {  in tcp_heuristic_do_ecn_common()
1347 	if (TSTMP_GT(tpheur->th_ecn_backoff, tcp_now)) {  in tcp_heuristic_do_ecn_common()
1351 		if (tpheur->th_ecn_droprst >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1352 			tpheur->th_ecn_droprst = 0;  in tcp_heuristic_do_ecn_common()
1354 		if (tpheur->th_ecn_droprxmt >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1355 			tpheur->th_ecn_droprxmt = 0;  in tcp_heuristic_do_ecn_common()
1357 		if (tpheur->th_ecn_synrst >= ECN_RETRY_LIMIT) {  in tcp_heuristic_do_ecn_common()
1358 			tpheur->th_ecn_synrst = 0;  in tcp_heuristic_do_ecn_common()
1362 		tpheur->th_ecn_backoff = tcp_now;  in tcp_heuristic_do_ecn_common()
1537 		struct tcp_heuristic *tpheur, *htmp;  in sysctl_cleartfocache()  local
1547 		SLIST_FOREACH_SAFE(tpheur, &hhead->tcp_heuristics, list, htmp) {  in sysctl_cleartfocache()
1548 			SLIST_REMOVE(&hhead->tcp_heuristics, tpheur, tcp_heuristic, list);  in sysctl_cleartfocache()
1549 			kfree_type(struct tcp_heuristic, tpheur);  in sysctl_cleartfocache()