[lvs-users] Kernel 2.6.35 and 100% S.I. CPU Time

[Julian Anastasov]

 	Hello,

On Mon, 27 Sep 2010, John Sullivan wrote:

> I backed out my patch and tried this instead.
>
> It does appear to fix the 100% SI problem, but creates a new
> one. Under my test load (fire 100 simultaneous HTTPS requests using
> wget at the load-balanced address), all the wgets hang, and netstat
> on the backup server shows all 100 connections in this state:
>
> Proto Recv-Q Send-Q Local Address Foreign Address State       PID/Program name
> tcp      113      0 lb-addr:443   ext-host:49914  ESTABLISHED -
>
>
> It looks to me like packets on those connections are now just being
> discarded (even when they shouldn't be).

 	Sorry for the delay. I'm attaching new patch for
testing. You should apply it on backup server. For your
setup only the last hunk in patch is needed. One of the
changes will keep the forwarding methods as specified,
may be you will not see "Local" real servers. If this is
a problem for you let me know. You can test then only
the last change in ip_vs_dr_xmit. Attached patch is for
2.6.35. It allows backup server after being used as
DR/TUN server in master to forward locally the packets
for connections created after sync.

Regards

--
Julian Anastasov <ja at ssi.bg>
-------------- next part --------------
diff -urp v2.6.35/linux/net/netfilter/ipvs/ip_vs_conn.c linux/net/netfilter/ipvs/ip_vs_conn.c
--- v2.6.35/linux/net/netfilter/ipvs/ip_vs_conn.c	2010-08-02 09:37:49.000000000 +0300
+++ linux/net/netfilter/ipvs/ip_vs_conn.c	2010-10-06 10:42:40.244948333 +0300
@@ -465,13 +465,15 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, s
 
 	/* Bind with the destination and its corresponding transmitter */
 	if ((cp->flags & IP_VS_CONN_F_SYNC) &&
-	    (!(cp->flags & IP_VS_CONN_F_TEMPLATE)))
+	    (!(cp->flags & IP_VS_CONN_F_TEMPLATE))) {
+		/* connections inherit forwarding method from dest */
+		cp->flags &= ~IP_VS_CONN_F_FWD_MASK;
 		/* if the connection is not template and is created
 		 * by sync, preserve the activity flag.
 		 */
 		cp->flags |= atomic_read(&dest->conn_flags) &
 			     (~IP_VS_CONN_F_INACTIVE);
-	else
+	} else
 		cp->flags |= atomic_read(&dest->conn_flags);
 	cp->dest = dest;
 
diff -urp v2.6.35/linux/net/netfilter/ipvs/ip_vs_ctl.c linux/net/netfilter/ipvs/ip_vs_ctl.c
--- v2.6.35/linux/net/netfilter/ipvs/ip_vs_ctl.c	2010-05-17 10:49:01.000000000 +0300
+++ linux/net/netfilter/ipvs/ip_vs_ctl.c	2010-10-06 10:45:48.337316299 +0300
@@ -767,20 +767,6 @@ __ip_vs_update_dest(struct ip_vs_service
 	atomic_set(&dest->weight, udest->weight);
 	conn_flags = udest->conn_flags | IP_VS_CONN_F_INACTIVE;
 
-	/* check if local node and update the flags */
-#ifdef CONFIG_IP_VS_IPV6
-	if (svc->af == AF_INET6) {
-		if (__ip_vs_addr_is_local_v6(&udest->addr.in6)) {
-			conn_flags = (conn_flags & ~IP_VS_CONN_F_FWD_MASK)
-				| IP_VS_CONN_F_LOCALNODE;
-		}
-	} else
-#endif
-		if (inet_addr_type(&init_net, udest->addr.ip) == RTN_LOCAL) {
-			conn_flags = (conn_flags & ~IP_VS_CONN_F_FWD_MASK)
-				| IP_VS_CONN_F_LOCALNODE;
-		}
-
 	/* set the IP_VS_CONN_F_NOOUTPUT flag if not masquerading/NAT */
 	if ((conn_flags & IP_VS_CONN_F_FWD_MASK) != 0) {
 		conn_flags |= IP_VS_CONN_F_NOOUTPUT;
@@ -942,6 +928,10 @@ ip_vs_add_dest(struct ip_vs_service *svc
 		 */
 		IP_VS_WAIT_WHILE(atomic_read(&svc->usecnt) > 1);
 
+		spin_lock(&dest->dst_lock);
+		ip_vs_dst_reset(dest);
+		spin_unlock(&dest->dst_lock);
+
 		list_add(&dest->n_list, &svc->destinations);
 		svc->num_dests++;
 
@@ -1030,6 +1020,10 @@ ip_vs_edit_dest(struct ip_vs_service *sv
 	/* Wait until all other svc users go away */
 	IP_VS_WAIT_WHILE(atomic_read(&svc->usecnt) > 1);
 
+	spin_lock(&dest->dst_lock);
+	ip_vs_dst_reset(dest);
+	spin_unlock(&dest->dst_lock);
+
 	/* call the update_service, because server weight may be changed */
 	if (svc->scheduler->update_service)
 		svc->scheduler->update_service(svc);
diff -urp v2.6.35/linux/net/netfilter/ipvs/ip_vs_xmit.c linux/net/netfilter/ipvs/ip_vs_xmit.c
--- v2.6.35/linux/net/netfilter/ipvs/ip_vs_xmit.c	2010-08-02 09:37:49.000000000 +0300
+++ linux/net/netfilter/ipvs/ip_vs_xmit.c	2010-10-06 10:59:31.577947469 +0300
@@ -548,6 +548,10 @@ ip_vs_tunnel_xmit(struct sk_buff *skb, s
 
 	if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(tos))))
 		goto tx_error_icmp;
+	if (rt->rt_flags & RTCF_LOCAL) {
+		ip_rt_put(rt);
+		return NF_ACCEPT;
+	}
 
 	tdev = rt->u.dst.dev;
 
@@ -758,6 +762,10 @@ ip_vs_dr_xmit(struct sk_buff *skb, struc
 
 	if (!(rt = __ip_vs_get_out_rt(cp, RT_TOS(iph->tos))))
 		goto tx_error_icmp;
+	if (rt->rt_flags & RTCF_LOCAL) {
+		ip_rt_put(rt);
+		return NF_ACCEPT;
+	}
 
 	/* MTU checking */
 	mtu = dst_mtu(&rt->u.dst);