Merge tag 'nf-26-05-05' of git://git.kernel.org/pub/scm/linux/kernel/git/netfilter/nf

+5 -1

include/linux/sched/isolation.h

··· 21 21 HK_TYPE_MAX, 22 22 23 23 /* 24 + * HK_TYPE_KTHREAD is now an alias of HK_TYPE_DOMAIN 25 + */ 26 + HK_TYPE_KTHREAD = HK_TYPE_DOMAIN, 27 + 28 + /* 24 29 * The following housekeeping types are only set by the nohz_full 25 30 * boot commandline option. So they can share the same value. 26 31 */ ··· 34 29 HK_TYPE_RCU = HK_TYPE_KERNEL_NOISE, 35 30 HK_TYPE_MISC = HK_TYPE_KERNEL_NOISE, 36 31 HK_TYPE_WQ = HK_TYPE_KERNEL_NOISE, 37 - HK_TYPE_KTHREAD = HK_TYPE_KERNEL_NOISE 38 32 }; 39 33 40 34 #ifdef CONFIG_CPU_ISOLATION

+26 -5

include/net/ip_vs.h

··· 491 491 DECLARE_BITMAP(avail, IPVS_EST_NTICKS); /* tick has space for ests */ 492 492 unsigned long est_timer; /* estimation timer (jiffies) */ 493 493 struct ip_vs_stats *calc_stats; /* Used for calculation */ 494 + int needed; /* task is needed */ 494 495 int tick_len[IPVS_EST_NTICKS]; /* est count */ 495 496 int id; /* ktid per netns */ 496 497 int chain_max; /* max ests per tick chain */ ··· 1412 1411 return ipvs->sysctl_run_estimation; 1413 1412 } 1414 1413 1415 - static inline const struct cpumask *sysctl_est_cpulist(struct netns_ipvs *ipvs) 1414 + static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1416 1415 { 1417 1416 if (ipvs->est_cpulist_valid) 1418 1417 return ipvs->sysctl_est_cpulist; ··· 1530 1529 return 1; 1531 1530 } 1532 1531 1533 - static inline const struct cpumask *sysctl_est_cpulist(struct netns_ipvs *ipvs) 1532 + static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1534 1533 { 1535 1534 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1536 1535 } ··· 1563 1562 static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs) 1564 1563 { 1565 1564 return READ_ONCE(ipvs->sysctl_svc_lfactor); 1565 + } 1566 + 1567 + static inline bool sysctl_est_cpulist_empty(struct netns_ipvs *ipvs) 1568 + { 1569 + guard(rcu)(); 1570 + return cpumask_empty(__sysctl_est_cpulist(ipvs)); 1571 + } 1572 + 1573 + static inline unsigned int sysctl_est_cpulist_weight(struct netns_ipvs *ipvs) 1574 + { 1575 + guard(rcu)(); 1576 + return cpumask_weight(__sysctl_est_cpulist(ipvs)); 1566 1577 } 1567 1578 1568 1579 /* IPVS core functions ··· 1897 1884 void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1898 1885 void ip_vs_zero_estimator(struct ip_vs_stats *stats); 1899 1886 void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats); 1900 - void ip_vs_est_reload_start(struct netns_ipvs *ipvs); 1887 + void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart); 1901 1888 int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, 1902 1889 struct ip_vs_est_kt_data *kd); 1903 1890 void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd); 1891 + 1892 + static inline void ip_vs_stop_estimator_tot_stats(struct netns_ipvs *ipvs) 1893 + { 1894 + #ifdef CONFIG_SYSCTL 1895 + ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); 1896 + ipvs->tot_stats->s.est.ktid = -2; 1897 + #endif 1898 + } 1904 1899 1905 1900 static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs) 1906 1901 { ··· 1916 1895 /* Stop tasks while cpulist is empty or if disabled with flag */ 1917 1896 ipvs->est_stopped = !sysctl_run_estimation(ipvs) || 1918 1897 (ipvs->est_cpulist_valid && 1919 - cpumask_empty(sysctl_est_cpulist(ipvs))); 1898 + sysctl_est_cpulist_empty(ipvs)); 1920 1899 #endif 1921 1900 } 1922 1901 ··· 1932 1911 static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs) 1933 1912 { 1934 1913 unsigned int limit = IPVS_EST_CPU_KTHREADS * 1935 - cpumask_weight(sysctl_est_cpulist(ipvs)); 1914 + sysctl_est_cpulist_weight(ipvs); 1936 1915 1937 1916 return max(1U, limit); 1938 1917 }

+41 -33

net/netfilter/ipvs/ip_vs_conn.c

··· 267 267 hash_key2 = hash_key; 268 268 use2 = false; 269 269 } 270 + 270 271 conn_tab_lock(t, cp, hash_key, hash_key2, use2, true /* new_hash */, 271 272 &head, &head2); 272 - spin_lock(&cp->lock); 273 273 274 - if (!(cp->flags & IP_VS_CONN_F_HASHED)) { 275 - cp->flags |= IP_VS_CONN_F_HASHED; 276 - WRITE_ONCE(cp->hn0.hash_key, hash_key); 277 - WRITE_ONCE(cp->hn1.hash_key, hash_key2); 278 - refcount_inc(&cp->refcnt); 279 - hlist_bl_add_head_rcu(&cp->hn0.node, head); 280 - if (use2) 281 - hlist_bl_add_head_rcu(&cp->hn1.node, head2); 282 - ret = 1; 283 - } else { 284 - pr_err("%s(): request for already hashed, called from %pS\n", 285 - __func__, __builtin_return_address(0)); 286 - ret = 0; 287 - } 274 + cp->flags |= IP_VS_CONN_F_HASHED; 275 + WRITE_ONCE(cp->hn0.hash_key, hash_key); 276 + WRITE_ONCE(cp->hn1.hash_key, hash_key2); 277 + refcount_inc(&cp->refcnt); 278 + hlist_bl_add_head_rcu(&cp->hn0.node, head); 279 + if (use2) 280 + hlist_bl_add_head_rcu(&cp->hn1.node, head2); 288 281 289 - spin_unlock(&cp->lock); 290 282 conn_tab_unlock(head, head2); 283 + ret = 1; 291 284 292 285 /* Schedule resizing if load increases */ 293 286 if (atomic_read(&ipvs->conn_count) > t->u_thresh && ··· 314 321 315 322 conn_tab_lock(t, cp, hash_key, hash_key2, use2, false /* new_hash */, 316 323 &head, &head2); 317 - spin_lock(&cp->lock); 318 324 319 325 if (cp->flags & IP_VS_CONN_F_HASHED) { 320 326 /* Decrease refcnt and unlink conn only if we are last user */ ··· 326 334 } 327 335 } 328 336 329 - spin_unlock(&cp->lock); 330 337 conn_tab_unlock(head, head2); 331 338 332 339 rcu_read_unlock(); ··· 628 637 struct ip_vs_conn_hnode *hn; 629 638 u32 hash_key, hash_key_new; 630 639 struct ip_vs_conn_param p; 640 + bool by_me = false; 631 641 int ntbl; 632 642 int dir; 633 643 ··· 656 664 t = rcu_dereference(t->new_tbl); 657 665 ntbl++; 658 666 /* We are lost? */ 659 - if (ntbl >= 2) 667 + if (ntbl >= 2) { 668 + spin_lock_bh(&cp->lock); 669 + if (cp->flags & IP_VS_CONN_F_NO_CPORT && by_me) 670 + cp->cport = 0; 671 + /* hn1 will be rehashed on next packet */ 672 + spin_unlock_bh(&cp->lock); 673 + IP_VS_ERR_RL("%s(): Too many ht changes for dir %d\n", 674 + __func__, dir); 660 675 return; 676 + } 661 677 } 662 678 663 679 /* Rehashing during resize? Use the recent table for adds */ ··· 683 683 if (head > head2 && t == t2) 684 684 swap(head, head2); 685 685 686 + /* Protect the cp->flags modification */ 687 + spin_lock_bh(&cp->lock); 688 + 686 689 /* Lock seqcount only for the old bucket, even if we are on new table 687 690 * because it affects the del operation, not the adding. 688 691 */ 689 - spin_lock_bh(&t->lock[hash_key & t->lock_mask].l); 692 + spin_lock(&t->lock[hash_key & t->lock_mask].l); 690 693 preempt_disable_nested(); 691 694 write_seqcount_begin(&t->seqc[hash_key & t->seqc_mask]); 692 695 ··· 707 704 hlist_bl_unlock(head); 708 705 write_seqcount_end(&t->seqc[hash_key & t->seqc_mask]); 709 706 preempt_enable_nested(); 710 - spin_unlock_bh(&t->lock[hash_key & t->lock_mask].l); 707 + spin_unlock(&t->lock[hash_key & t->lock_mask].l); 708 + spin_unlock_bh(&cp->lock); 711 709 hash_key = hash_key_new; 712 710 goto retry; 713 711 } 714 712 715 - spin_lock(&cp->lock); 716 - if ((cp->flags & IP_VS_CONN_F_NO_CPORT) && 717 - (cp->flags & IP_VS_CONN_F_HASHED)) { 713 + /* Fill cport once, even if multiple packets try to do it */ 714 + if (cp->flags & IP_VS_CONN_F_NO_CPORT && (!cp->cport || by_me)) { 715 + /* If we race with resizing make sure cport is set for dir 1 */ 716 + if (!cp->cport) { 717 + cp->cport = cport; 718 + by_me = true; 719 + } 720 + if (!dir) { 721 + atomic_dec(&ipvs->no_cport_conns[af_id]); 722 + cp->flags &= ~IP_VS_CONN_F_NO_CPORT; 723 + } 718 724 /* We do not recalc hash_key_r under lock, we assume the 719 725 * parameters in cp do not change, i.e. cport is 720 726 * the only possible change. ··· 738 726 hlist_bl_del_rcu(&hn->node); 739 727 hlist_bl_add_head_rcu(&hn->node, head_new); 740 728 } 741 - if (!dir) { 742 - atomic_dec(&ipvs->no_cport_conns[af_id]); 743 - cp->flags &= ~IP_VS_CONN_F_NO_CPORT; 744 - cp->cport = cport; 745 - } 746 729 } 747 - spin_unlock(&cp->lock); 748 730 749 731 if (head != head2) 750 732 hlist_bl_unlock(head2); 751 733 hlist_bl_unlock(head); 752 734 write_seqcount_end(&t->seqc[hash_key & t->seqc_mask]); 753 735 preempt_enable_nested(); 754 - spin_unlock_bh(&t->lock[hash_key & t->lock_mask].l); 755 - if (dir--) 736 + spin_unlock(&t->lock[hash_key & t->lock_mask].l); 737 + 738 + spin_unlock_bh(&cp->lock); 739 + if (dir-- && by_me) 756 740 goto next_dir; 757 741 } 758 742 ··· 1843 1835 1844 1836 if (!rcu_dereference_protected(ipvs->conn_tab, 1)) 1845 1837 return; 1846 - cancel_delayed_work_sync(&ipvs->conn_resize_work); 1838 + disable_delayed_work_sync(&ipvs->conn_resize_work); 1847 1839 if (!atomic_read(&ipvs->conn_count)) 1848 1840 goto unreg; 1849 1841

+1 -1

net/netfilter/ipvs/ip_vs_core.c

··· 237 237 { 238 238 if (!t) 239 239 return 1 << min_bits; 240 - n = roundup_pow_of_two(n); 240 + n = n > 0 ? roundup_pow_of_two(n) : 1; 241 241 if (lfactor < 0) { 242 242 int factor = min(-lfactor, max_bits); 243 243

+119 -45

net/netfilter/ipvs/ip_vs_ctl.c

··· 261 261 if (!kd) 262 262 continue; 263 263 /* New config ? Stop kthread tasks */ 264 - if (genid != genid_done) 265 - ip_vs_est_kthread_stop(kd); 264 + if (genid != genid_done) { 265 + if (!id) { 266 + /* Only we can stop kt 0 but not under mutex */ 267 + mutex_unlock(&ipvs->est_mutex); 268 + ip_vs_est_kthread_stop(kd); 269 + mutex_lock(&ipvs->est_mutex); 270 + if (!READ_ONCE(ipvs->enable)) 271 + goto unlock; 272 + /* kd for kt 0 is never destroyed */ 273 + } else { 274 + ip_vs_est_kthread_stop(kd); 275 + } 276 + } 266 277 if (!kd->task && !ip_vs_est_stopped(ipvs)) { 278 + bool start; 279 + 267 280 /* Do not start kthreads above 0 in calc phase */ 268 - if ((!id || !ipvs->est_calc_phase) && 269 - ip_vs_est_kthread_start(ipvs, kd) < 0) 281 + if (id) 282 + start = !ipvs->est_calc_phase; 283 + else 284 + start = kd->needed; 285 + if (start && ip_vs_est_kthread_start(ipvs, kd) < 0) 270 286 repeat = true; 271 287 } 272 288 } ··· 1118 1102 return dest; 1119 1103 } 1120 1104 1105 + /* Put destination in trash */ 1106 + static void ip_vs_trash_put_dest(struct netns_ipvs *ipvs, 1107 + struct ip_vs_dest *dest, unsigned long istart, 1108 + bool cleanup) 1109 + { 1110 + spin_lock_bh(&ipvs->dest_trash_lock); 1111 + IP_VS_DBG_BUF(3, "Moving dest %s:%u into trash, dest->refcnt=%d\n", 1112 + IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port), 1113 + refcount_read(&dest->refcnt)); 1114 + if (list_empty(&ipvs->dest_trash) && !cleanup) 1115 + mod_timer(&ipvs->dest_trash_timer, 1116 + jiffies + (IP_VS_DEST_TRASH_PERIOD >> 1)); 1117 + /* dest lives in trash with reference */ 1118 + list_add(&dest->t_list, &ipvs->dest_trash); 1119 + dest->idle_start = istart; 1120 + spin_unlock_bh(&ipvs->dest_trash_lock); 1121 + } 1122 + 1121 1123 static void ip_vs_dest_rcu_free(struct rcu_head *head) 1122 1124 { 1123 1125 struct ip_vs_dest *dest; ··· 1495 1461 ntohs(dest->vport)); 1496 1462 1497 1463 ret = ip_vs_start_estimator(svc->ipvs, &dest->stats); 1464 + /* On error put back dest into the trash */ 1498 1465 if (ret < 0) 1499 - return ret; 1500 - __ip_vs_update_dest(svc, dest, udest, 1); 1466 + ip_vs_trash_put_dest(svc->ipvs, dest, dest->idle_start, 1467 + false); 1468 + else 1469 + __ip_vs_update_dest(svc, dest, udest, 1); 1501 1470 } else { 1502 1471 /* 1503 1472 * Allocate and initialize the dest structure ··· 1570 1533 */ 1571 1534 ip_vs_rs_unhash(dest); 1572 1535 1573 - spin_lock_bh(&ipvs->dest_trash_lock); 1574 - IP_VS_DBG_BUF(3, "Moving dest %s:%u into trash, dest->refcnt=%d\n", 1575 - IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port), 1576 - refcount_read(&dest->refcnt)); 1577 - if (list_empty(&ipvs->dest_trash) && !cleanup) 1578 - mod_timer(&ipvs->dest_trash_timer, 1579 - jiffies + (IP_VS_DEST_TRASH_PERIOD >> 1)); 1580 - /* dest lives in trash with reference */ 1581 - list_add(&dest->t_list, &ipvs->dest_trash); 1582 - dest->idle_start = 0; 1583 - spin_unlock_bh(&ipvs->dest_trash_lock); 1536 + ip_vs_trash_put_dest(ipvs, dest, 0, cleanup); 1584 1537 1585 1538 /* Queue up delayed work to expire all no destination connections. 1586 1539 * No-op when CONFIG_SYSCTL is disabled. ··· 1839 1812 *svc_p = svc; 1840 1813 1841 1814 if (!READ_ONCE(ipvs->enable)) { 1815 + mutex_lock(&ipvs->est_mutex); 1816 + 1842 1817 /* Now there is a service - full throttle */ 1843 1818 WRITE_ONCE(ipvs->enable, 1); 1844 1819 1820 + ipvs->est_max_threads = ip_vs_est_max_threads(ipvs); 1821 + 1845 1822 /* Start estimation for first time */ 1846 - ip_vs_est_reload_start(ipvs); 1823 + ip_vs_est_reload_start(ipvs, true); 1824 + mutex_unlock(&ipvs->est_mutex); 1847 1825 } 1848 1826 1849 1827 return 0; ··· 2064 2032 cancel_delayed_work_sync(&ipvs->svc_resize_work); 2065 2033 if (t) { 2066 2034 rcu_assign_pointer(ipvs->svc_table, NULL); 2035 + /* Inform readers that table is removed */ 2036 + smp_mb__before_atomic(); 2037 + atomic_inc(&ipvs->svc_table_changes); 2067 2038 while (1) { 2068 2039 p = rcu_dereference_protected(t->new_tbl, 1); 2069 2040 call_rcu(&t->rcu_head, ip_vs_rht_rcu_free); ··· 2113 2078 t = rcu_dereference_protected(ipvs->svc_table, 1); 2114 2079 if (t) { 2115 2080 rcu_assign_pointer(ipvs->svc_table, NULL); 2081 + /* Inform readers that table is removed */ 2082 + smp_mb__before_atomic(); 2083 + atomic_inc(&ipvs->svc_table_changes); 2116 2084 while (1) { 2117 2085 p = rcu_dereference_protected(t->new_tbl, 1); 2118 2086 call_rcu(&t->rcu_head, ip_vs_rht_rcu_free); ··· 2124 2086 t = p; 2125 2087 } 2126 2088 } 2089 + /* Stop the tot_stats estimator early under service_mutex 2090 + * to avoid locking it again later. 2091 + */ 2092 + if (cleanup) 2093 + ip_vs_stop_estimator_tot_stats(ipvs); 2127 2094 return 0; 2128 2095 } 2129 2096 ··· 2374 2331 /* est_max_threads may depend on cpulist size */ 2375 2332 ipvs->est_max_threads = ip_vs_est_max_threads(ipvs); 2376 2333 ipvs->est_calc_phase = 1; 2377 - ip_vs_est_reload_start(ipvs); 2334 + ip_vs_est_reload_start(ipvs, true); 2378 2335 2379 2336 unlock: 2380 2337 mutex_unlock(&ipvs->est_mutex); ··· 2394 2351 2395 2352 mutex_lock(&ipvs->est_mutex); 2396 2353 2397 - if (ipvs->est_cpulist_valid) 2398 - mask = *valp; 2399 - else 2400 - mask = (struct cpumask *)housekeeping_cpumask(HK_TYPE_KTHREAD); 2401 - ret = scnprintf(buffer, size, "%*pbl\n", cpumask_pr_args(mask)); 2354 + /* HK_TYPE_KTHREAD cpumask needs RCU protection */ 2355 + scoped_guard(rcu) { 2356 + if (ipvs->est_cpulist_valid) 2357 + mask = *valp; 2358 + else 2359 + mask = (struct cpumask *)housekeeping_cpumask(HK_TYPE_KTHREAD); 2360 + ret = scnprintf(buffer, size, "%*pbl\n", cpumask_pr_args(mask)); 2361 + } 2402 2362 2403 2363 mutex_unlock(&ipvs->est_mutex); 2404 2364 ··· 2457 2411 mutex_lock(&ipvs->est_mutex); 2458 2412 if (*valp != val) { 2459 2413 *valp = val; 2460 - ip_vs_est_reload_start(ipvs); 2414 + ip_vs_est_reload_start(ipvs, true); 2461 2415 } 2462 2416 mutex_unlock(&ipvs->est_mutex); 2463 2417 } ··· 2484 2438 mutex_lock(&ipvs->est_mutex); 2485 2439 if (*valp != val) { 2486 2440 *valp = val; 2487 - ip_vs_est_reload_start(ipvs); 2441 + ip_vs_est_reload_start(ipvs, true); 2488 2442 } 2489 2443 mutex_unlock(&ipvs->est_mutex); 2490 2444 } ··· 2509 2463 if (val < -8 || val > 8) { 2510 2464 ret = -EINVAL; 2511 2465 } else { 2512 - *valp = val; 2466 + WRITE_ONCE(*valp, val); 2513 2467 if (rcu_access_pointer(ipvs->conn_tab)) 2514 2468 mod_delayed_work(system_unbound_wq, 2515 2469 &ipvs->conn_resize_work, 0); ··· 2536 2490 if (val < -8 || val > 8) { 2537 2491 ret = -EINVAL; 2538 2492 } else { 2539 - *valp = val; 2540 - if (rcu_access_pointer(ipvs->svc_table)) 2493 + mutex_lock(&ipvs->service_mutex); 2494 + WRITE_ONCE(*valp, val); 2495 + /* Make sure the services are present */ 2496 + if (rcu_access_pointer(ipvs->svc_table) && 2497 + READ_ONCE(ipvs->enable) && 2498 + !test_bit(IP_VS_WORK_SVC_NORESIZE, 2499 + &ipvs->work_flags)) 2541 2500 mod_delayed_work(system_unbound_wq, 2542 2501 &ipvs->svc_resize_work, 0); 2502 + mutex_unlock(&ipvs->service_mutex); 2543 2503 } 2544 2504 } 2545 2505 return ret; ··· 3056 3004 int old_gen, new_gen; 3057 3005 u32 counts[8]; 3058 3006 u32 bucket; 3059 - int count; 3007 + u32 count; 3008 + int loops; 3060 3009 u32 sum1; 3061 3010 u32 sum; 3062 3011 int i; ··· 3073 3020 if (!atomic_read(&ipvs->conn_count)) 3074 3021 goto after_conns; 3075 3022 old_gen = atomic_read(&ipvs->conn_tab_changes); 3023 + loops = 0; 3076 3024 3077 3025 repeat_conn: 3078 3026 smp_rmb(); /* ipvs->conn_tab and conn_tab_changes */ ··· 3086 3032 resched_score++; 3087 3033 ip_vs_rht_walk_bucket_rcu(t, bucket, head) { 3088 3034 count = 0; 3089 - hlist_bl_for_each_entry_rcu(hn, e, head, node) 3035 + hlist_bl_for_each_entry_rcu(hn, e, head, node) { 3090 3036 count++; 3037 + if (count >= ARRAY_SIZE(counts) - 1) 3038 + break; 3039 + } 3091 3040 } 3092 3041 resched_score += count; 3093 3042 if (resched_score >= 100) { ··· 3099 3042 new_gen = atomic_read(&ipvs->conn_tab_changes); 3100 3043 /* New table installed ? */ 3101 3044 if (old_gen != new_gen) { 3045 + /* Too many changes? */ 3046 + if (++loops >= 5) 3047 + goto after_conns; 3102 3048 old_gen = new_gen; 3103 3049 goto repeat_conn; 3104 3050 } 3105 3051 } 3106 - counts[min(count, (int)ARRAY_SIZE(counts) - 1)]++; 3052 + counts[count]++; 3107 3053 } 3108 3054 } 3109 3055 for (sum = 0, i = 0; i < ARRAY_SIZE(counts); i++) 3110 3056 sum += counts[i]; 3111 3057 sum1 = sum - counts[0]; 3112 - seq_printf(seq, "Conn buckets empty:\t%u (%lu%%)\n", 3113 - counts[0], (unsigned long)counts[0] * 100 / max(sum, 1U)); 3058 + seq_printf(seq, "Conn buckets empty:\t%u (%llu%%)\n", 3059 + counts[0], div_u64((u64)counts[0] * 100U, max(sum, 1U))); 3114 3060 for (i = 1; i < ARRAY_SIZE(counts); i++) { 3115 3061 if (!counts[i]) 3116 3062 continue; 3117 - seq_printf(seq, "Conn buckets len-%d:\t%u (%lu%%)\n", 3063 + seq_printf(seq, "Conn buckets len-%d:\t%u (%llu%%)\n", 3118 3064 i, counts[i], 3119 - (unsigned long)counts[i] * 100 / max(sum1, 1U)); 3065 + div_u64((u64)counts[i] * 100U, max(sum1, 1U))); 3120 3066 } 3121 3067 3122 3068 after_conns: 3123 3069 t = rcu_dereference(ipvs->svc_table); 3124 3070 3125 3071 count = ip_vs_get_num_services(ipvs); 3126 - seq_printf(seq, "Services:\t%d\n", count); 3072 + seq_printf(seq, "Services:\t%u\n", count); 3127 3073 seq_printf(seq, "Service buckets:\t%d (%d bits, lfactor %d)\n", 3128 3074 t ? t->size : 0, t ? t->bits : 0, t ? t->lfactor : 0); 3129 3075 3130 3076 if (!count) 3131 3077 goto after_svc; 3132 3078 old_gen = atomic_read(&ipvs->svc_table_changes); 3079 + loops = 0; 3133 3080 3134 3081 repeat_svc: 3135 3082 smp_rmb(); /* ipvs->svc_table and svc_table_changes */ ··· 3147 3086 ip_vs_rht_walk_bucket_rcu(t, bucket, head) { 3148 3087 count = 0; 3149 3088 hlist_bl_for_each_entry_rcu(svc, e, head, 3150 - s_list) 3089 + s_list) { 3151 3090 count++; 3091 + if (count >= ARRAY_SIZE(counts) - 1) 3092 + break; 3093 + } 3152 3094 } 3153 3095 resched_score += count; 3154 3096 if (resched_score >= 100) { ··· 3160 3096 new_gen = atomic_read(&ipvs->svc_table_changes); 3161 3097 /* New table installed ? */ 3162 3098 if (old_gen != new_gen) { 3099 + /* Too many changes? */ 3100 + if (++loops >= 5) 3101 + goto after_svc; 3163 3102 old_gen = new_gen; 3164 3103 goto repeat_svc; 3165 3104 } 3166 3105 } 3167 - counts[min(count, (int)ARRAY_SIZE(counts) - 1)]++; 3106 + counts[count]++; 3168 3107 } 3169 3108 } 3170 3109 for (sum = 0, i = 0; i < ARRAY_SIZE(counts); i++) 3171 3110 sum += counts[i]; 3172 3111 sum1 = sum - counts[0]; 3173 - seq_printf(seq, "Service buckets empty:\t%u (%lu%%)\n", 3174 - counts[0], (unsigned long)counts[0] * 100 / max(sum, 1U)); 3112 + seq_printf(seq, "Service buckets empty:\t%u (%llu%%)\n", 3113 + counts[0], div_u64((u64)counts[0] * 100U, max(sum, 1U))); 3175 3114 for (i = 1; i < ARRAY_SIZE(counts); i++) { 3176 3115 if (!counts[i]) 3177 3116 continue; 3178 - seq_printf(seq, "Service buckets len-%d:\t%u (%lu%%)\n", 3117 + seq_printf(seq, "Service buckets len-%d:\t%u (%llu%%)\n", 3179 3118 i, counts[i], 3180 - (unsigned long)counts[i] * 100 / max(sum1, 1U)); 3119 + div_u64((u64)counts[i] * 100U, max(sum1, 1U))); 3181 3120 } 3182 3121 3183 3122 after_svc: ··· 5034 4967 cancel_delayed_work_sync(&ipvs->defense_work); 5035 4968 cancel_work_sync(&ipvs->defense_work.work); 5036 4969 unregister_net_sysctl_table(ipvs->sysctl_hdr); 5037 - ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); 4970 + if (ipvs->tot_stats->s.est.ktid != -2) { 4971 + /* Not stopped yet? This happens only on netns init error and 4972 + * we even do not need to lock the service_mutex for this case. 4973 + */ 4974 + mutex_lock(&ipvs->service_mutex); 4975 + ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); 4976 + mutex_unlock(&ipvs->service_mutex); 4977 + } 5038 4978 5039 4979 if (ipvs->est_cpulist_valid) 5040 4980 free_cpumask_var(ipvs->sysctl_est_cpulist); ··· 5113 5039 ipvs->net->proc_net, 5114 5040 ip_vs_stats_percpu_show, NULL)) 5115 5041 goto err_percpu; 5116 - if (!proc_create_net_single("ip_vs_status", 0, ipvs->net->proc_net, 5042 + if (!proc_create_net_single("ip_vs_status", 0440, ipvs->net->proc_net, 5117 5043 ip_vs_status_show, NULL)) 5118 5044 goto err_status; 5119 5045 #endif

+48 -35

net/netfilter/ipvs/ip_vs_est.c

··· 68 68 and the limit of estimators per kthread 69 69 - est_add_ktid: ktid where to add new ests, can point to empty slot where 70 70 we should add kt data 71 + - data protected by service_mutex: est_temp_list, est_add_ktid, 72 + est_kt_count(R/W), est_kt_arr(R/W), est_genid_done, kd->needed(R/W) 73 + - data protected by est_mutex: est_genid, est_max_threads, sysctl_est_cpulist, 74 + est_cpulist_valid, sysctl_est_nice, est_stopped, sysctl_run_estimation, 75 + est_kt_count(R), est_kt_arr(R), kd->needed(R), kd->task (id > 0) 71 76 */ 72 77 73 78 static struct lock_class_key __ipvs_est_key; ··· 232 227 } 233 228 234 229 /* Schedule stop/start for kthread tasks */ 235 - void ip_vs_est_reload_start(struct netns_ipvs *ipvs) 230 + void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart) 236 231 { 232 + lockdep_assert_held(&ipvs->est_mutex); 233 + 237 234 /* Ignore reloads before first service is added */ 238 235 if (!READ_ONCE(ipvs->enable)) 239 236 return; 240 237 ip_vs_est_stopped_recalc(ipvs); 241 - /* Bump the kthread configuration genid */ 242 - atomic_inc(&ipvs->est_genid); 238 + /* Bump the kthread configuration genid if stopping is requested */ 239 + if (restart) 240 + atomic_inc(&ipvs->est_genid); 243 241 queue_delayed_work(system_long_wq, &ipvs->est_reload_work, 0); 244 242 } 245 243 ··· 312 304 void *arr = NULL; 313 305 int i; 314 306 315 - if ((unsigned long)ipvs->est_kt_count >= ipvs->est_max_threads && 316 - READ_ONCE(ipvs->enable) && ipvs->est_max_threads) 317 - return -EINVAL; 318 - 319 307 mutex_lock(&ipvs->est_mutex); 308 + 309 + /* Allow kt 0 data to be created before the services are added 310 + * and limit the kthreads when services are present. 311 + */ 312 + if ((unsigned long)ipvs->est_kt_count >= ipvs->est_max_threads && 313 + READ_ONCE(ipvs->enable) && ipvs->est_max_threads) { 314 + ret = -EINVAL; 315 + goto out; 316 + } 320 317 321 318 for (i = 0; i < id; i++) { 322 319 if (!ipvs->est_kt_arr[i]) ··· 346 333 kd->est_timer = jiffies; 347 334 kd->id = id; 348 335 ip_vs_est_set_params(ipvs, kd); 336 + kd->needed = 1; 349 337 350 338 /* Pre-allocate stats used in calc phase */ 351 339 if (!id && !kd->calc_stats) { ··· 355 341 goto out; 356 342 } 357 343 358 - /* Start kthread tasks only when services are present */ 359 - if (READ_ONCE(ipvs->enable) && !ip_vs_est_stopped(ipvs)) { 360 - ret = ip_vs_est_kthread_start(ipvs, kd); 361 - if (ret < 0) 362 - goto out; 363 - } 344 + /* Request kthread to be started */ 345 + ip_vs_est_reload_start(ipvs, false); 364 346 365 347 if (arr) 366 348 ipvs->est_kt_count++; ··· 492 482 /* Start estimation for stats */ 493 483 int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats) 494 484 { 485 + struct ip_vs_est_kt_data *kd = ipvs->est_kt_count > 0 ? 486 + ipvs->est_kt_arr[0] : NULL; 495 487 struct ip_vs_estimator *est = &stats->est; 496 488 int ret; 497 - 498 - if (!ipvs->est_max_threads && READ_ONCE(ipvs->enable)) 499 - ipvs->est_max_threads = ip_vs_est_max_threads(ipvs); 500 489 501 490 est->ktid = -1; 502 491 est->ktrow = IPVS_EST_NTICKS - 1; /* Initial delay */ ··· 505 496 * will not allocate much memory, just for kt 0. 506 497 */ 507 498 ret = 0; 508 - if (!ipvs->est_kt_count || !ipvs->est_kt_arr[0]) 499 + if (!kd) { 509 500 ret = ip_vs_est_add_kthread(ipvs); 501 + } else if (!kd->needed) { 502 + mutex_lock(&ipvs->est_mutex); 503 + /* We have job for the kt 0 task */ 504 + kd->needed = 1; 505 + ip_vs_est_reload_start(ipvs, true); 506 + mutex_unlock(&ipvs->est_mutex); 507 + } 510 508 if (ret >= 0) 511 509 hlist_add_head(&est->list, &ipvs->est_temp_list); 512 510 else ··· 594 578 } 595 579 596 580 end_kt0: 597 - /* kt 0 is freed after all other kthreads and chains are empty */ 581 + /* kt 0 task is stopped after all other kt slots and chains are empty */ 598 582 if (ipvs->est_kt_count == 1 && hlist_empty(&ipvs->est_temp_list)) { 599 583 kd = ipvs->est_kt_arr[0]; 600 - if (!kd || !kd->est_count) { 584 + if (kd && !kd->est_count) { 601 585 mutex_lock(&ipvs->est_mutex); 602 - if (kd) { 603 - ip_vs_est_kthread_destroy(kd); 604 - ipvs->est_kt_arr[0] = NULL; 605 - } 606 - ipvs->est_kt_count--; 586 + /* Keep the kt0 data but request kthread_stop */ 587 + kd->needed = 0; 588 + ip_vs_est_reload_start(ipvs, true); 607 589 mutex_unlock(&ipvs->est_mutex); 608 590 ipvs->est_add_ktid = 0; 609 591 } ··· 661 647 u64 val; 662 648 663 649 INIT_HLIST_HEAD(&chain); 664 - mutex_lock(&ipvs->service_mutex); 650 + mutex_lock(&ipvs->est_mutex); 665 651 kd = ipvs->est_kt_arr[0]; 666 - mutex_unlock(&ipvs->service_mutex); 652 + mutex_unlock(&ipvs->est_mutex); 667 653 s = kd ? kd->calc_stats : NULL; 668 654 if (!s) 669 655 goto out; ··· 762 748 if (!ip_vs_est_calc_limits(ipvs, &chain_max)) 763 749 return; 764 750 765 - mutex_lock(&ipvs->service_mutex); 766 - 767 751 /* Stop all other tasks, so that we can immediately move the 768 752 * estimators to est_temp_list without RCU grace period 769 753 */ 770 754 mutex_lock(&ipvs->est_mutex); 771 755 for (id = 1; id < ipvs->est_kt_count; id++) { 772 756 /* netns clean up started, abort */ 773 - if (!READ_ONCE(ipvs->enable)) 774 - goto unlock2; 757 + if (kthread_should_stop() || !READ_ONCE(ipvs->enable)) { 758 + mutex_unlock(&ipvs->est_mutex); 759 + return; 760 + } 775 761 kd = ipvs->est_kt_arr[id]; 776 762 if (!kd) 777 763 continue; ··· 779 765 } 780 766 mutex_unlock(&ipvs->est_mutex); 781 767 768 + mutex_lock(&ipvs->service_mutex); 769 + 782 770 /* Move all estimators to est_temp_list but carefully, 783 771 * all estimators and kthread data can be released while 784 - * we reschedule. Even for kthread 0. 772 + * we reschedule. 785 773 */ 786 774 step = 0; 787 775 ··· 865 849 ip_vs_stop_estimator(ipvs, stats); 866 850 /* Tasks are stopped, move without RCU grace period */ 867 851 est->ktid = -1; 868 - est->ktrow = row - kd->est_row; 869 - if (est->ktrow < 0) 870 - est->ktrow += IPVS_EST_NTICKS; 852 + est->ktrow = delay; 871 853 hlist_add_head(&est->list, &ipvs->est_temp_list); 872 854 /* kd freed ? */ 873 855 if (last) ··· 903 889 if (genid == atomic_read(&ipvs->est_genid)) 904 890 ipvs->est_calc_phase = 0; 905 891 906 - unlock2: 907 892 mutex_unlock(&ipvs->est_mutex); 908 893 909 894 unlock:

Configure Feed

Configure Feed