RE: RE: Disable Nagle (enable TCP_NODELAY) ?

[Michael Zervakis]

I finally did a comparison test for Nagle using SMPP. For the test I
used a real SMSC with SMPP interface that was accessible through a VPN
connection over WAN which represents a common real world case. The SMSC
had a throttling limit at 80sms/sec triggered every 10 seconds. I built
two versions of Kannel cvs_20090921, one umodified and another one with
Nagle disabled and I connected both of them to the same SMSC. The test
included the submission of 2000 MTs each containing a 100 character
payload with sequential submission (2000 MT to first bearerbox, wait to
finish, 2000MT to second bearerbox, wait to finish). The test was
repeated 10 times in a time span of eight hours. During the whole
process I used tcpdump to calculate the seconds it took for each
bearerbox to send the 2000MT pack. The results are presented in the
table below (first column Nagle enabled, second column Nagle disabled).
The bearerbox configuration for both versions and the patch to disable
Nagle are also included. The dumps are not included but I can send them
if anyone is interested.

TEST #    |DEFAULT    |NAGLE DISABLED
__________________________________
1    |37        |26
2    |34        |28
3    |43        |24
4    |30        |25
5    |28        |23
6    |35        |24
7    |34        |18
8    |34        |29
9    |39        |33
10    |32        |30
___________________________________
AVERAGE    |34,6        |26
___________________________________
STDEV    |4,32        |4,22



# Core Group Configuration
group = core
# HTTP Admin Interface Configuration
admin-port = 22000
admin-port-ssl = no
admin-password = sms-admin
admin-deny-ip = "*.*.*.*"
admin-allow-ip = "127.0.0.1"
# SMSBOX Interface
smsbox-port = 19000
smsbox-port-ssl = no
# SMSBOX & WAPBOX access-list
box-deny-ip = "*.*.*.*"
box-allow-ip = "127.0.0.1"
udp-allow-ip = "127.0.0.1"
#wdp-interface-name = "*"
# BEARERBOX log files
log-file = /var/log/kannel/bearerbox.log
log-level = 1
access-log = /var/log/kannel/bearerboxaccess.log
#unified-prefix = ""
sms-incoming-queue-limit = -1
store-type = spool
store-location = "/var/spool/kannel/"
store-dump-freq = 5
sms-combine-concatenated-mo = no
sms-resend-retry = 15

group = smsbox
bearerbox-host = 127.0.0.1
bearerbox-port = 19000
smsbox-id = "BOX"
sendsms-port = 11000
sendsms-port-ssl = no
sendsms-chars = "0123456789 +"
log-file = /var/log/kannel/smsbox.log
log-level = 2
access-log = /var/log/kannel/smsboxaccess.log
reply-couldnotfetch = "Sorry no content"
reply-couldnotrepresent = "Sorry invalid content"
reply-requestfailed = "Sorry service failed"
reply-emptymessage = ""
http-request-retry = 50
http-queue-delay = 30
mo-recode = yes

group = smsbox-route
smsbox-id = "BOX"
smsc-id = "cell"

group = sendsms-user
username = "user"
password = "password"
concatenation = yes
max-messages = 3
omit-empty = yes
user-deny-ip = "*.*.*.*"
user-allow-ip = "127.0.0.1"

group = smsc
smsc = smpp
smsc-id = "cell"
host = XXX.XXX.XXX.XXX
port = XXXXX
transceiver-mode = yes
smsc-username = "user"
smsc-password = "password"
system-type = ""
interface-version = 34
address-range = ""
connection-timeout = 60
enquire-link-interval = 30
reconnect-delay = 30
source-addr-ton = 0
source-addr-npi = 1
dest-addr-ton = 0
dest-addr-npi = 1
allowed-smsc-id = "cell"
max-pending-submits = 100


-----Original Message-----
From: devel-bounces@... [mailto:devel-bounces@...] On
Behalf Of Stipe Tolj
Sent: Wednesday, February 25, 2009 5:40 PM
Cc: devel@...
Subject: Re: Disable Nagle (enable TCP_NODELAY) ?

 

Michael Zervakis schrieb:


 > Hello,

 >

 > Has anyone ever tried to disable Nagle at Kannel? I'm experiencing TCP

 > latency with some connections and I'm suspecting Nagle has to do with it.

 

didn't try so far. You could patch the relevant source part for the
setsockopt()

call and try.

 

We'd appreciate seeing test results. ;)

 

Stipe

 

--

-------------------------------------------------------------------

Kölner Landstrasse 419

40589 Düsseldorf, NRW, Germany

 

tolj.org system architecture      Kannel Software Foundation (KSF)

http://www.tolj.org/              http://www.kannel.org/

 

mailto:st_{at}_tolj.org           mailto:stolj_{at}_kannel.org

-------------------------------------------------------------------

 



--- gwlib/socket.c.orig 2009-09-21 11:37:20.000000000 +0300
+++ gwlib/socket.c 2009-09-24 19:26:43.000000000 +0300
@@ -68,6 +68,7 @@
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <netdb.h>
 #include <arpa/inet.h>
 #include <sys/utsname.h>
@@ -94,7 +95,7 @@
 {
     struct sockaddr_in addr;
     int s;
-    int reuse;
+    int reuse, iTcpNoDelay;
     struct hostent hostinfo;
     char *buff = NULL;
 
@@ -120,7 +121,14 @@
     reuse = 1;
     if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse,
                    sizeof(reuse)) == -1) {
-        error(errno, "setsockopt failed for server address");
+        error(errno, "setsockopt failed for server address socket reuse");
+        goto error;
+    }
+
+    iTcpNoDelay = 1;
+if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *) &iTcpNoDelay,
+                   sizeof(iTcpNoDelay)) == -1) {
+        error(errno, "setsockopt failed for server address tcp nodelay");
         goto error;
     }
 
@@ -176,7 +184,7 @@
     }
 
     if (our_port > 0 || (interface_name != NULL && strcmp(interface_name, "*") != 0))  {
-        int reuse;
+        int reuse, iTcpNoDelay;
 
         o_addr = empty_sockaddr_in;
         o_addr.sin_family = AF_INET;
@@ -193,7 +201,12 @@
 
         reuse = 1;
         if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse, sizeof(reuse)) == -1) {
-            error(errno, "setsockopt failed before bind");
+            error(errno, "setsockopt failed before bind for socket reuse");
+            goto error;
+        }
+        iTcpNoDelay = 1;
+        if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *) &iTcpNoDelay, sizeof(iTcpNoDelay)) == -1) {
+            error(errno, "setsockopt failed before bind for tcp nodelay");
             goto error;
         }
         if (bind(s, (struct sockaddr *) &o_addr, sizeof(o_addr)) == -1) {
@@ -267,7 +280,7 @@
     }
 
     if (our_port > 0 || (interface_name != NULL && strcmp(interface_name, "*") != 0)) {
-        int reuse;
+        int reuse, iTcpNoDelay;
 
         o_addr = empty_sockaddr_in;
         o_addr.sin_family = AF_INET;
@@ -284,7 +297,12 @@
 
         reuse = 1;
         if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse, sizeof(reuse)) == -1) {
-            error(errno, "setsockopt failed before bind");
+            error(errno, "setsockopt failed before bind for socket reuse");
+            goto error;
+        }
+        iTcpNoDelay = 1;
+        if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *) &iTcpNoDelay, sizeof(iTcpNoDelay)) == -1) {
+            error(errno, "setsockopt failed before bind for tcp nodelay");
             goto error;
         }
         if (bind(s, (struct sockaddr *) &o_addr, sizeof(o_addr)) == -1) {

[Nikos Balkanas-2]

Hi,

Thanks for a very thorough job. It seems that you are experiencing better
throughput with Nagle enabled. It is a good thing to have Nagle enabled in a
highly congested network. Like most production kannel installation. On the
other hand, if you have plenty of bandwith to spare and you want immediate
delivery you might want to disable it. But who is hoing to guarantee that
downstream connections (SMSCs etc.) will not use Nagle as well? In fact most
likely they will.

The very nature of SMS is not immediate delivery. This is not reflected in
tcpdumps from your own server. I think that Nagle is a good overall choice.
I believe that you have reached the same conclusion in your case. If you
consider actual SMS delivery, then the whole issue becomes academic.

BR,
Nikos
----- Original Message -----
From: "Michael Zervakis" <michael@...>
To: <devel@...>
Sent: Wednesday, October 14, 2009 5:37 PM
Subject: RE: RE: Disable Nagle (enable TCP_NODELAY) ?



I finally did a comparison test for Nagle using SMPP. For the test I
used a real SMSC with SMPP interface that was accessible through a VPN
connection over WAN which represents a common real world case. The SMSC
had a throttling limit at 80sms/sec triggered every 10 seconds. I built
two versions of Kannel cvs_20090921, one umodified and another one with
Nagle disabled and I connected both of them to the same SMSC. The test
included the submission of 2000 MTs each containing a 100 character
payload with sequential submission (2000 MT to first bearerbox, wait to
finish, 2000MT to second bearerbox, wait to finish). The test was
repeated 10 times in a time span of eight hours. During the whole
process I used tcpdump to calculate the seconds it took for each
bearerbox to send the 2000MT pack. The results are presented in the
table below (first column Nagle enabled, second column Nagle disabled).
The bearerbox configuration for both versions and the patch to disable
Nagle are also included. The dumps are not included but I can send them
if anyone is interested.

TEST #    |DEFAULT    |NAGLE DISABLED
__________________________________
1    |37        |26
2    |34        |28
3    |43        |24
4    |30        |25
5    |28        |23
6    |35        |24
7    |34        |18
8    |34        |29
9    |39        |33
10    |32        |30
___________________________________
AVERAGE    |34,6        |26
___________________________________
STDEV    |4,32        |4,22



# Core Group Configuration
group = core
# HTTP Admin Interface Configuration
admin-port = 22000
admin-port-ssl = no
admin-password = sms-admin
admin-deny-ip = "*.*.*.*"
admin-allow-ip = "127.0.0.1"
# SMSBOX Interface
smsbox-port = 19000
smsbox-port-ssl = no
# SMSBOX & WAPBOX access-list
box-deny-ip = "*.*.*.*"
box-allow-ip = "127.0.0.1"
udp-allow-ip = "127.0.0.1"
#wdp-interface-name = "*"
# BEARERBOX log files
log-file = /var/log/kannel/bearerbox.log
log-level = 1
access-log = /var/log/kannel/bearerboxaccess.log
#unified-prefix = ""
sms-incoming-queue-limit = -1
store-type = spool
store-location = "/var/spool/kannel/"
store-dump-freq = 5
sms-combine-concatenated-mo = no
sms-resend-retry = 15

group = smsbox
bearerbox-host = 127.0.0.1
bearerbox-port = 19000
smsbox-id = "BOX"
sendsms-port = 11000
sendsms-port-ssl = no
sendsms-chars = "0123456789 +"
log-file = /var/log/kannel/smsbox.log
log-level = 2
access-log = /var/log/kannel/smsboxaccess.log
reply-couldnotfetch = "Sorry no content"
reply-couldnotrepresent = "Sorry invalid content"
reply-requestfailed = "Sorry service failed"
reply-emptymessage = ""
http-request-retry = 50
http-queue-delay = 30
mo-recode = yes

group = smsbox-route
smsbox-id = "BOX"
smsc-id = "cell"

group = sendsms-user
username = "user"
password = "password"
concatenation = yes
max-messages = 3
omit-empty = yes
user-deny-ip = "*.*.*.*"
user-allow-ip = "127.0.0.1"

group = smsc
smsc = smpp
smsc-id = "cell"
host = XXX.XXX.XXX.XXX
port = XXXXX
transceiver-mode = yes
smsc-username = "user"
smsc-password = "password"
system-type = ""
interface-version = 34
address-range = ""
connection-timeout = 60
enquire-link-interval = 30
reconnect-delay = 30
source-addr-ton = 0
source-addr-npi = 1
dest-addr-ton = 0
dest-addr-npi = 1
allowed-smsc-id = "cell"
max-pending-submits = 100


-----Original Message-----
From: devel-bounces@... [mailto:devel-bounces@...] On
Behalf Of Stipe Tolj
Sent: Wednesday, February 25, 2009 5:40 PM
Cc: devel@...
Subject: Re: Disable Nagle (enable TCP_NODELAY) ?



Michael Zervakis schrieb:


 > Hello,

 >

 > Has anyone ever tried to disable Nagle at Kannel? I'm experiencing TCP

 > latency with some connections and I'm suspecting Nagle has to do with it.



didn't try so far. You could patch the relevant source part for the
setsockopt()

call and try.



We'd appreciate seeing test results. ;)



Stipe



--

-------------------------------------------------------------------

Kφlner Landstrasse 419

40589 Dόsseldorf, NRW, Germany



tolj.org system architecture      Kannel Software Foundation (KSF)

http://www.tolj.org/              http://www.kannel.org/



mailto:st_{at}_tolj.org           mailto:stolj_{at}_kannel.org

-------------------------------------------------------------------







--------------------------------------------------------------------------------

> --- gwlib/socket.c.orig 2009-09-21 11:37:20.000000000 +0300
> +++ gwlib/socket.c 2009-09-24 19:26:43.000000000 +0300
> @@ -68,6 +68,7 @@
> #include <sys/types.h>
> #include <sys/socket.h>
> #include <netinet/in.h>
> +#include <netinet/tcp.h>
> #include <netdb.h>
> #include <arpa/inet.h>
> #include <sys/utsname.h>
> @@ -94,7 +95,7 @@
> {
>     struct sockaddr_in addr;
>     int s;
> -    int reuse;
> +    int reuse, iTcpNoDelay;
>     struct hostent hostinfo;
>     char *buff = NULL;
>
> @@ -120,7 +121,14 @@
>     reuse = 1;
>     if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse,
>                    sizeof(reuse)) == -1) {
> -        error(errno, "setsockopt failed for server address");
> +        error(errno, "setsockopt failed for server address socket
> reuse");
> +        goto error;
> +    }
> +
> +    iTcpNoDelay = 1;
> +if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *) &iTcpNoDelay,
> +                   sizeof(iTcpNoDelay)) == -1) {
> +        error(errno, "setsockopt failed for server address tcp nodelay");
>         goto error;
>     }
>
> @@ -176,7 +184,7 @@
>     }
>
>     if (our_port > 0 || (interface_name != NULL && strcmp(interface_name,
> "*") != 0))  {
> -        int reuse;
> +        int reuse, iTcpNoDelay;
>
>         o_addr = empty_sockaddr_in;
>         o_addr.sin_family = AF_INET;
> @@ -193,7 +201,12 @@
>
>         reuse = 1;
>         if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse,
> sizeof(reuse)) == -1) {
> -            error(errno, "setsockopt failed before bind");
> +            error(errno, "setsockopt failed before bind for socket
> reuse");
> +            goto error;
> +        }
> +        iTcpNoDelay = 1;
> +        if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *)
> &iTcpNoDelay, sizeof(iTcpNoDelay)) == -1) {
> +            error(errno, "setsockopt failed before bind for tcp
> nodelay");
>             goto error;
>         }
>         if (bind(s, (struct sockaddr *) &o_addr, sizeof(o_addr)) == -1) {
> @@ -267,7 +280,7 @@
>     }
>
>     if (our_port > 0 || (interface_name != NULL && strcmp(interface_name,
> "*") != 0)) {
> -        int reuse;
> +        int reuse, iTcpNoDelay;
>
>         o_addr = empty_sockaddr_in;
>         o_addr.sin_family = AF_INET;
> @@ -284,7 +297,12 @@
>
>         reuse = 1;
>         if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, (char *) &reuse,
> sizeof(reuse)) == -1) {
> -            error(errno, "setsockopt failed before bind");
> +            error(errno, "setsockopt failed before bind for socket
> reuse");
> +            goto error;
> +        }
> +        iTcpNoDelay = 1;
> +        if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char *)
> &iTcpNoDelay, sizeof(iTcpNoDelay)) == -1) {
> +            error(errno, "setsockopt failed before bind for tcp
> nodelay");
>             goto error;
>         }
>         if (bind(s, (struct sockaddr *) &o_addr, sizeof(o_addr)) == -1) {
>

[Damian Viano]

Hi,

 For what it's worth, I can see some scenarios benefiting from more immediate
delivery, specially if this increases total throughput for some scenarios.

 So IMHO it would be nice to have the knob to tweak this setting, at least I
would appreciate it since I have been tuning some services that might very well
benefit from it.

 If added, as you said Nikos, the default should by all means be to have Nagle
enabled.

  Damián Viano.

On Thu, Oct 15, 2009 at 11:38:34AM +0300, Nikos Balkanas wrote:

> Hi,
>
> Thanks for a very thorough job. It seems that you are experiencing better
> throughput with Nagle enabled. It is a good thing to have Nagle enabled
> in a highly congested network. Like most production kannel installation.
> On the other hand, if you have plenty of bandwith to spare and you want
> immediate delivery you might want to disable it. But who is hoing to
> guarantee that downstream connections (SMSCs etc.) will not use Nagle as
> well? In fact most likely they will.
>
> The very nature of SMS is not immediate delivery. This is not reflected
> in tcpdumps from your own server. I think that Nagle is a good overall
> choice. I believe that you have reached the same conclusion in your case.
> If you consider actual SMS delivery, then the whole issue becomes
> academic.
>
> BR,
> Nikos
> ----- Original Message ----- From: "Michael Zervakis"
> <michael@...>
> To: <devel@...>
> Sent: Wednesday, October 14, 2009 5:37 PM
> Subject: RE: RE: Disable Nagle (enable TCP_NODELAY) ?
>
>
>
> I finally did a comparison test for Nagle using SMPP. For the test I
> used a real SMSC with SMPP interface that was accessible through a VPN
> connection over WAN which represents a common real world case. The SMSC
> had a throttling limit at 80sms/sec triggered every 10 seconds. I built
> two versions of Kannel cvs_20090921, one umodified and another one with
> Nagle disabled and I connected both of them to the same SMSC. The test
> included the submission of 2000 MTs each containing a 100 character
> payload with sequential submission (2000 MT to first bearerbox, wait to
> finish, 2000MT to second bearerbox, wait to finish). The test was
> repeated 10 times in a time span of eight hours. During the whole
> process I used tcpdump to calculate the seconds it took for each
> bearerbox to send the 2000MT pack. The results are presented in the
> table below (first column Nagle enabled, second column Nagle disabled).
> The bearerbox configuration for both versions and the patch to disable
> Nagle are also included. The dumps are not included but I can send them
> if anyone is interested.
>
> TEST #    |DEFAULT    |NAGLE DISABLED
> __________________________________
> 1    |37        |26
> 2    |34        |28
> 3    |43        |24
> 4    |30        |25
> 5    |28        |23
> 6    |35        |24
> 7    |34        |18
> 8    |34        |29
> 9    |39        |33
> 10    |32        |30
> ___________________________________
> AVERAGE    |34,6        |26
> ___________________________________
> STDEV    |4,32        |4,22

[snip config and patch]