[lvs-users] LVS-SNAT only works locally (ubuntu 11.04 amd64)

Tom van Leeuwen tom.van.leeuwen at saasplaza.com
Mon May 30 09:00:47 BST 2011


Hi Julian, thank you for your swift reply.
I've rebuild the default ubuntu 11.04 amd64 server kernel with the 
changes from your diff applied.
My diff is:

    tom at tole:/unprotected/kernel$ diff -up
    linux-2.6.38-patched/net/netfilter/ipvs/ip_vs_core.c
    linux-2.6.38/net/netfilter/ipvs/ip_vs_core.c
    --- linux-2.6.38-patched/net/netfilter/ipvs/ip_vs_core.c   
    2011-05-30 08:05:31.778443588 +0200
    +++ linux-2.6.38/net/netfilter/ipvs/ip_vs_core.c    2011-03-15
    02:20:32.000000000 +0100
    @@ -1683,7 +1683,7 @@ static struct nf_hook_ops ip_vs_ops[] __
              .owner        = THIS_MODULE,
              .pf        = PF_INET,
              .hooknum    = NF_INET_LOCAL_IN,
    -        .priority    = 98,
    +        .priority    = 99,
          },
          /* After packet filtering, forward packet through VS/DR, VS/TUN,
           * or VS/NAT(change destination), so that filtering rules can be
    @@ -1693,7 +1693,7 @@ static struct nf_hook_ops ip_vs_ops[] __
              .owner        = THIS_MODULE,
              .pf        = PF_INET,
              .hooknum    = NF_INET_LOCAL_IN,
    -        .priority    = 99,
    +        .priority    = 101,
          },
          /* Before ip_vs_in, change source only for VS/NAT */
          {
    @@ -1735,7 +1735,7 @@ static struct nf_hook_ops ip_vs_ops[] __
              .owner        = THIS_MODULE,
              .pf        = PF_INET6,
              .hooknum    = NF_INET_LOCAL_IN,
    -        .priority    = 98,
    +        .priority    = 99,
          },
          /* After packet filtering, forward packet through VS/DR, VS/TUN,
           * or VS/NAT(change destination), so that filtering rules can be
    @@ -1745,7 +1745,7 @@ static struct nf_hook_ops ip_vs_ops[] __
              .owner        = THIS_MODULE,
              .pf        = PF_INET6,
              .hooknum    = NF_INET_LOCAL_IN,
    -        .priority    = 99,
    +        .priority    = 101,
          },
          /* Before ip_vs_in, change source only for VS/NAT */
          {

I've installed the new kernel package and am now running:
loadbalancer-ng ~ # uname -a
Linux loadbalancer-ng 2.6.38-8-server #42 SMP Mon May 30 08:18:35 CEST 
2011 x86_64 x86_64 x86_64 GNU/Linux

I can confirm that your patch works!

I've done several tests:

    TESTS
    1) routed vip without alias:
    ipvsadm -A -t 3.3.3.3:80 -s rr
    ipvsadm -a -t 3.3.3.3:80 -r 172.16.29.5:80 -m
    loadbalancer-ng seems to forward the traffic to the default gw
    without triggering ipvs related stuff
    DOES NOT WORK, BUT HAS ALWAYS BEEN A REQUIREMENT IF I'M NOT MISTAKEN

    2) alias vip
    ip addr add 3.3.3.3 dev lo
    ipvsadm -A -t 3.3.3.3:80 -s rr
    ipvsadm -a -t 3.3.3.3:80 -r 172.16.29.5:80 -m
    WORKS

    3) alias vip with snat
    ip addr add 3.3.3.3 dev lo
    iptables -t nat -A POSTROUTING -m ipvs --vaddr 3.3.3.3 --vport 80 -j
    SNAT --to-source 172.16.29.10
    ipvsadm -A -t 3.3.3.3:80 -s rr
    ipvsadm -a -t 3.3.3.3:80 -r 172.16.29.5:80 -m
    WORKS

    4) alias vip with snat using some random ip
    ip addr add 3.3.3.3 dev lo
    ipvsadm -A -t 3.3.3.3:80 -s rr
    ipvsadm -a -t 3.3.3.3:80 -r 172.16.29.5:80 -m
    iptables -t nat -A POSTROUTING -m ipvs --vaddr 3.3.3.3 --vport 80 -j
    SNAT --to-source 1.1.1.1
    WORKS

    5) realserver with loadbalancer-ng as default gw. and outgoing
    MASQUERADE on the loadbalancer-ng
    So: test 4) setup +
    iptables -t nat -A POSTROUTING -o eth1 -s 172.16.29.5 -j MASQUERADE
    WORKS

    6) test 5) but with SNAT --to-source 3.3.3.3 (VIP)
    iptables -t nat -A POSTROUTING -o eth1 -s 172.16.29.5 -j SNAT
    --to-source 3.3.3.3
    WORKS

Thanks again!

With kind regards,
Tom van Leeuwen


On 05/28/2011 12:47 AM, Julian Anastasov wrote:
> 	Hello,
>
> On Fri, 27 May 2011, Tom van Leeuwen wrote:
>
>> Hi list, this is my first post, so please be gentle.
>> I'm trying to get a lvs (lab)setup to work with snat.
>> I've been reading several posts and the mailinglist, and they told that
>> the snat should work with kernel 2.6.36 and higher.
> 	May be the problem starts with 2.6.37, I now
> see that the things worked by luck before 2.6.37.
> I'm appending a patch against 2.6.39 for testing.
> It should apply for 2.6.38 too. I hope xt_ipvs has
> no problem with matching the traffic when conntrack=1.
>
>>       6   360 SNAT       all  --  *      *       0.0.0.0/0
>> 0.0.0.0/0           vaddr 172.16.31.10 vport 80 to:172.16.29.10
>>
>> loadbalancer-ng ~ # sysctl net.ipv4.vs.conntrack
>> net.ipv4.vs.conntrack = 1
> =========================================================
>
> 	Fix the IPVS priority in LOCAL_IN hook,
> so that SNAT target in POSTROUTING is supported for IPVS
> traffic.
>
> 	Before 2.6.37 we used priority 100 in LOCAL_IN to
> process remote requests. We used the same priority as
> iptables SNAT and if IPVS handlers are installed before
> SNAT handlers we supported SNAT in POSTROUTING for the IPVS
> traffic. If SNAT is installed before IPVS, the netfilter
> handlers are before IPVS and netfilter checks the NAT
> table twice for the IPVS requests: once in LOCAL_IN where
> IPS_SRC_NAT_DONE is set and second time in POSTROUTING
> where the SNAT rules are ignored because IPS_SRC_NAT_DONE
> was already set in LOCAL_IN.
>
> 	But in 2.6.37 we changed the IPVS priority for
> LOCAL_IN to be unique (101) forgetting the fact that for
> IPVS traffic we should not walk both LOCAL_IN and
> POSTROUTING nat tables.
>
> 	So, change the priority for processing remote
> IPVS requests from 101 to 99, i.e. before NAT_SRC (100)
> because we prefer to support SNAT in POSTROUTING
> instead of LOCAL_IN. It also moves the priority for
> IPVS replies from 99 to 98.
>
> Signed-off-by: Julian Anastasov<ja at ssi.bg>
> ---
>
> diff -urp v2.6.39/linux/net/netfilter/ipvs/ip_vs_core.c linux/net/netfilter/ipvs/ip_vs_core.c
> --- v2.6.39/linux/net/netfilter/ipvs/ip_vs_core.c	2011-05-20 10:38:08.000000000 +0300
> +++ linux/net/netfilter/ipvs/ip_vs_core.c	2011-05-28 00:42:53.845348561 +0300
> @@ -1792,7 +1792,7 @@ static struct nf_hook_ops ip_vs_ops[] __
>   		.owner		= THIS_MODULE,
>   		.pf		= PF_INET,
>   		.hooknum	= NF_INET_LOCAL_IN,
> -		.priority	= 99,
> +		.priority	= 98,
>   	},
>   	/* After packet filtering, forward packet through VS/DR, VS/TUN,
>   	 * or VS/NAT(change destination), so that filtering rules can be
> @@ -1802,7 +1802,7 @@ static struct nf_hook_ops ip_vs_ops[] __
>   		.owner		= THIS_MODULE,
>   		.pf		= PF_INET,
>   		.hooknum	= NF_INET_LOCAL_IN,
> -		.priority	= 101,
> +		.priority	= 99,
>   	},
>   	/* Before ip_vs_in, change source only for VS/NAT */
>   	{
> @@ -1844,7 +1844,7 @@ static struct nf_hook_ops ip_vs_ops[] __
>   		.owner		= THIS_MODULE,
>   		.pf		= PF_INET6,
>   		.hooknum	= NF_INET_LOCAL_IN,
> -		.priority	= 99,
> +		.priority	= 98,
>   	},
>   	/* After packet filtering, forward packet through VS/DR, VS/TUN,
>   	 * or VS/NAT(change destination), so that filtering rules can be
> @@ -1854,7 +1854,7 @@ static struct nf_hook_ops ip_vs_ops[] __
>   		.owner		= THIS_MODULE,
>   		.pf		= PF_INET6,
>   		.hooknum	= NF_INET_LOCAL_IN,
> -		.priority	= 101,
> +		.priority	= 99,
>   	},
>   	/* Before ip_vs_in, change source only for VS/NAT */
>   	{



More information about the lvs-users mailing list