Improve send/close performance

git-svn-id: svn://svn.code.sf.net/p/nuttx/code/trunk@410 42af7a65-404d-4744-a932-0658087f49c3

9 files changed, 213 insertions, 101 deletions

diff --git a/nuttx/ChangeLog b/nuttx/ChangeLog
index bdd2ad17e..2e69d72f5 100644
--- a/nuttx/ChangeLog
+++ b/nuttx/ChangeLog
@@ -254,3 +254,7 @@
 	* Added vsnprintf
 	* Integrated uIP telnetd
 	* Add missing logic to readahead buffer logic
+	* examples/nettest uses larger buffers
+	* Improved ACK handling in send() to better hander deferred acknowledgements
+	  and polling intervals.  Greatly improves send performance.
+
diff --git a/nuttx/Documentation/NuttX.html b/nuttx/Documentation/NuttX.html
index fe6c5d560..8247fa14c 100644
--- a/nuttx/Documentation/NuttX.html
+++ b/nuttx/Documentation/NuttX.html
@@ -734,6 +734,9 @@ Other memory:
 	* Added vsnprintf
 	* Integrated uIP telnetd
 	* Add missing logic to readahead buffer logic
+	* examples/nettest uses larger buffers
+	* Improved ACK handling in send() to better hander deferred acknowledgements
+	  and polling intervals.  Greatly improves send performance.
 </pre></ul>
 
 <table width ="100%">
diff --git a/nuttx/TODO b/nuttx/TODO
index 4484be424..c0aff5318 100644
--- a/nuttx/TODO
+++ b/nuttx/TODO
@@ -33,10 +33,6 @@ o Network
   but is not implemented.
 - uIP's netutils/smtp, dpcpc, resolv, webclient -- untested
 - Should implement SOCK_RAW
-- Performance Improvements (uIP is not very fast):
-  Need to extend logic so that uIP can have more than on packet in flight and to
-  handle deferred acknowledgements.  This is supposed to improve send performance by
-  an order of magnitude.
 - uIP polling issues:
   (1) Current logic will not support multiple ethernet drivers.  Each driver should
       poll on TCP connections connect on the network supported by the driver; UDP
@@ -78,8 +74,8 @@ o C5471
 
 o DM320
 - It seems that when a lot of debug statements are added, the system no
-  longer boots.  There could be some issue with the bootloader or with
-  the programming of the SDRAM MMU regions.
+  longer boots.  This has been diagnosed as a stack problem.. making the stack
+  bigger or removing arrays on the stack fixes the problem.
 
 o LPC214x
 - Finish bringup
diff --git a/nuttx/examples/nettest/nettest-client.c b/nuttx/examples/nettest/nettest-client.c
index 2e5991a86..371f4d4f8 100644
--- a/nuttx/examples/nettest/nettest-client.c
+++ b/nuttx/examples/nettest/nettest-client.c
@@ -56,9 +56,9 @@
 void send_client(void)
 {
   struct sockaddr_in myaddr;
-  char outbuf[SENDSIZE];
+  char *outbuf;
 #ifndef CONFIG_EXAMPLE_NETTEST_PERFORMANCE
-  char inbuf[SENDSIZE];
+  char *inbuf;
 #endif
   int sockfd;
   int nbytessent;
@@ -69,13 +69,28 @@ void send_client(void)
   int ch;
   int i;
 
+  /* Allocate buffers */
+
+  outbuf = (char*)malloc(SENDSIZE);
+#ifndef CONFIG_EXAMPLE_NETTEST_PERFORMANCE
+  inbuf  = (char*)malloc(SENDSIZE);
+  if (!outbuf || !inbuf)
+#else
+  if (!outbuf)
+#endif
+    {
+      message("client: failed to allocate buffers\n");
+      exit(1);
+    }
+
+
   /* Create a new TCP socket */
 
   sockfd = socket(PF_INET, SOCK_STREAM, 0);
   if (sockfd < 0)
     {
       message("client socket failure %d\n", errno);
-      exit(1);
+      goto errout_with_buffers;
     }
 
   /* Connect the socket to the server */
@@ -92,7 +107,7 @@ void send_client(void)
   if (connect( sockfd, (struct sockaddr*)&myaddr, sizeof(struct sockaddr_in)) < 0)
     {
       message("client: connect failure: %d\n", errno);
-      exit(1);
+      goto errout_with_socket;
     }
   message("client: Connected\n");
 
@@ -117,14 +132,12 @@ void send_client(void)
       if (nbytessent < 0)
         {
           message("client: send failed: %d\n", errno);
-          close(sockfd);
-          exit(-1);
+          goto errout_with_socket;
         }
       else if (nbytessent != 512)
         {
           message("client: Bad send length=%d: %d\n", nbytessent);
-          close(sockfd);
-          exit(-1);
+          goto errout_with_socket;
         }
     }
 #else
@@ -137,14 +150,12 @@ void send_client(void)
   if (nbytessent < 0)
     {
       message("client: send failed: %d\n", errno);
-      close(sockfd);
-      exit(-1);
+      goto errout_with_socket;
     }
   else if (nbytessent != SENDSIZE)
     {
       message("client: Bad send length: %d Expected: %d\n", nbytessent, SENDSIZE);
-      close(sockfd);
-      exit(-1);
+      goto errout_with_socket;
     }
 
   totalbytesrecvd = 0;
@@ -156,8 +167,7 @@ void send_client(void)
       if (nbytesrecvd < 0)
         {
           message("client: recv failed: %d\n", errno);
-          close(sockfd);
-          exit(-1);
+          goto errout_with_socket;
         }
       totalbytesrecvd += nbytesrecvd;
       message("client: Received %d of %d bytes\n", totalbytesrecvd, SENDSIZE);
@@ -167,16 +177,29 @@ void send_client(void)
   if (totalbytesrecvd != SENDSIZE)
     {
       message("client: Bad recv length: %d Expected: %d\n", totalbytesrecvd, SENDSIZE);
-      close(sockfd);
-      exit(-1);
+      goto errout_with_socket;
     }
   else if (memcmp(inbuf, outbuf, SENDSIZE) != 0)
     {
       message("client: Received buffer does not match sent buffer\n");
-      close(sockfd);
-      exit(-1);
+      goto errout_with_socket;
     }
 
   close(sockfd);
+  free(outbuf);
+#ifndef CONFIG_EXAMPLE_NETTEST_PERFORMANCE
+  free(inbuf);
+#endif
+  return;
+#endif
+
+errout_with_socket:
+  close(sockfd);
+
+errout_with_buffers:
+  free(outbuf);
+#ifndef CONFIG_EXAMPLE_NETTEST_PERFORMANCE
+  free(inbuf);
 #endif
+  exit(1);
 }
diff --git a/nuttx/examples/nettest/nettest-server.c b/nuttx/examples/nettest/nettest-server.c
index 6dbede997..6d9ccaeba 100644
--- a/nuttx/examples/nettest/nettest-server.c
+++ b/nuttx/examples/nettest/nettest-server.c
@@ -58,7 +58,7 @@ void recv_server(void)
 #ifdef NETTEST_HAVE_SOLINGER
   struct linger ling;
 #endif
-  char buffer[1024];
+  char *buffer;
   int listensd;
   int acceptsd;
   socklen_t addrlen;
@@ -71,13 +71,23 @@ void recv_server(void)
 #endif
   int optval;
 
+  /* Allocate a BIG buffer */
+
+  buffer = (char*)malloc(2*SENDSIZE);
+  if (!buffer)
+    {
+      message("server: failed to allocate buffer\n");
+      exit(1);
+    }
+
+
   /* Create a new TCP socket */
 
   listensd = socket(PF_INET, SOCK_STREAM, 0);
   if (listensd < 0)
     {
       message("server: socket failure: %d\n", errno);
-      exit(1);
+      goto errout_with_buffer;
     }
 
   /* Set socket to reuse address */
@@ -86,7 +96,7 @@ void recv_server(void)
   if (setsockopt(listensd, SOL_SOCKET, SO_REUSEADDR, (void*)&optval, sizeof(int)) < 0)
     {
       message("server: setsockopt SO_REUSEADDR failure: %d\n", errno);
-      exit(1);
+      goto errout_with_listensd;
     }
 
   /* Bind the socket to a local address */
@@ -98,7 +108,7 @@ void recv_server(void)
   if (bind(listensd, (struct sockaddr*)&myaddr, sizeof(struct sockaddr_in)) < 0)
     {
       message("server: bind failure: %d\n", errno);
-      exit(1);
+      goto errout_with_listensd;
     }
 
   /* Listen for connections on the bound TCP socket */
@@ -106,7 +116,7 @@ void recv_server(void)
   if (listen(listensd, 5) < 0)
     {
       message("server: listen failure %d\n", errno);
-      exit(1);
+      goto errout_with_listensd;
     }
 
   /* Accept only one connection */
@@ -117,7 +127,7 @@ void recv_server(void)
   if (acceptsd < 0)
     {
       message("server: accept failure: %d\n", errno);
-      exit(1);
+      goto errout_with_listensd;
     }
   message("server: Connection accepted -- receiving\n");
 
@@ -129,7 +139,7 @@ void recv_server(void)
     if (setsockopt(acceptsd, SOL_SOCKET, SO_LINGER, &ling, sizeof(struct linger)) < 0)
       {
       message("server: setsockopt SO_LINGER failure: %d\n", errno);
-      exit(1);
+      goto errout_with_acceptsd;
     }
 #endif
 
@@ -138,13 +148,11 @@ void recv_server(void)
 
   for (;;)
     {
-      nbytesread = recv(acceptsd, buffer, 1024, 0);
+      nbytesread = recv(acceptsd, buffer, 2*SENDSIZE, 0);
       if (nbytesread <= 0)
         {
           message("server: recv failed: %d\n", errno);
-          close(listensd);
-          close(acceptsd);
-          exit(-1);
+          goto errout_with_acceptsd;
         }
     }
 #else
@@ -154,13 +162,11 @@ void recv_server(void)
   while (totalbytesread < SENDSIZE)
     {
       message("server: Reading...\n");
-      nbytesread = recv(acceptsd, &buffer[totalbytesread], 1024 - totalbytesread, 0);
+      nbytesread = recv(acceptsd, &buffer[totalbytesread], 2*SENDSIZE - totalbytesread, 0);
       if (nbytesread <= 0)
         {
           message("server: recv failed: %d\n", errno);
-          close(listensd);
-          close(acceptsd);
-          exit(-1);
+          goto errout_with_acceptsd;
         }
 
       totalbytesread += nbytesread;
@@ -172,9 +178,7 @@ void recv_server(void)
   if (totalbytesread != SENDSIZE)
     {
       message("server: Received %d / Expected %d bytes\n", totalbytesread, SENDSIZE);
-      close(listensd);
-      close(acceptsd);
-      exit(-1);
+      goto errout_with_acceptsd;
     }
 
   ch = 0x20;
@@ -183,9 +187,7 @@ void recv_server(void)
       if (buffer[i] != ch)
         {
           message("server: Byte %d is %02x / Expected %02x\n", i, buffer[i], ch);
-          close(listensd);
-          close(acceptsd);
-          exit(-1);
+          goto errout_with_acceptsd;
         }
 
       if (++ch > 0x7e)
@@ -201,9 +203,7 @@ void recv_server(void)
   if (nbytessent <= 0)
     {
       message("server: send failed: %d\n", errno);
-      close(listensd);
-      close(acceptsd);
-      exit(-1);
+      goto errout_with_acceptsd;
     }
   message("server: Sent %d bytes\n", nbytessent);
 
@@ -218,5 +218,17 @@ void recv_server(void)
 
   close(listensd);
   close(acceptsd);
+  free(buffer);
+  return;
 #endif
+
+errout_with_acceptsd:
+  close(acceptsd);
+
+errout_with_listensd:
+  close(listensd);
+
+errout_with_buffer:
+  free(buffer);
+  exit(1);
 }
diff --git a/nuttx/examples/nettest/nettest.h b/nuttx/examples/nettest/nettest.h
index eb7c83c84..318211e79 100644
--- a/nuttx/examples/nettest/nettest.h
+++ b/nuttx/examples/nettest/nettest.h
@@ -85,7 +85,7 @@
 #endif
 
 #define PORTNO     5471
-#define SENDSIZE   512
+#define SENDSIZE   4096
 
 /****************************************************************************
  * Public Function Prototypes
diff --git a/nuttx/net/net-close.c b/nuttx/net/net-close.c
index f64d70de7..8d1be9099 100644
--- a/nuttx/net/net-close.c
+++ b/nuttx/net/net-close.c
@@ -165,6 +165,10 @@ static inline void netclose_disconnect(FAR struct socket *psock)
        conn->data_private = (void*)&state;
        conn->data_event   = netclose_interrupt;
 
+       /* Notify the device driver of the availaibilty of TX data */
+
+       netdev_txnotify(&conn->ripaddr);
+
        /* Wait for the disconnect event */
 
        (void)sem_wait(&state.cl_sem);
diff --git a/nuttx/net/send.c b/nuttx/net/send.c
index 69cf883ac..67acc6285 100644
--- a/nuttx/net/send.c
+++ b/nuttx/net/send.c
@@ -48,6 +48,7 @@
 #include <debug.h>
 
 #include <arch/irq.h>
+#include <net/uip/uip-arch.h>
 
 #include "net-internal.h"
 
@@ -55,8 +56,7 @@
  * Definitions
  ****************************************************************************/
 
-#define STATE_POLLWAIT   1
-#define STATE_DATA_SENT  2
+#define TCPBUF ((struct uip_tcpip_hdr *)&dev->d_buf[UIP_LLH_LEN])
 
 /****************************************************************************
  * Private Types
@@ -69,11 +69,12 @@
 struct send_s
 {
   FAR struct socket *snd_sock;    /* Points to the parent socket structure */
-  sem_t                snd_sem;     /* Used to wake up the waiting thread */
-  FAR const uint8     *snd_buffer;  /* Points to the buffer of data to send */
-  size_t               snd_buflen;  /* Number of bytes in the buffer to send */
-  ssize_t              snd_sent;    /* The number of bytes sent */
-  uint8                snd_state;   /* The state of the send operation. */
+  sem_t              snd_sem;     /* Used to wake up the waiting thread */
+  FAR const uint8   *snd_buffer;  /* Points to the buffer of data to send */
+  size_t             snd_buflen;  /* Number of bytes in the buffer to send */
+  ssize_t            snd_sent;    /* The number of bytes sent */
+  uint32             snd_isn;     /* Initial sequence number */
+  uint32             snd_acked;   /* The number of bytes acked */
 };
 
 /****************************************************************************
@@ -81,6 +82,54 @@ struct send_s
  ****************************************************************************/
 
 /****************************************************************************
+ * Function: send_getisn
+ *
+ * Description:
+ *   Get the next initial sequence number from the connection stucture
+ *
+ * Parameters:
+ *   conn     The connection structure associated with the socket
+ *
+ * Returned Value:
+ *   None
+ *
+ * Assumptions:
+ *   Running at the interrupt level
+ *
+ ****************************************************************************/
+
+static uint32 send_getisn(struct uip_conn *conn)
+{
+   uint32 tmp;
+   memcpy(&tmp, conn->snd_nxt, 4);
+   return ntohl(tmp);
+}
+
+/****************************************************************************
+ * Function: send_getackno
+ *
+ * Description:
+ *   Extract the current acknowledgement sequence number from the incoming packet
+ *
+ * Parameters:
+ *   dev - The sructure of the network driver that caused the interrupt
+ *
+ * Returned Value:
+ *   None
+ *
+ * Assumptions:
+ *   Running at the interrupt level
+ *
+ ****************************************************************************/
+
+static uint32 send_getackno(struct uip_driver_s *dev)
+{
+   uint32 tmp;
+   memcpy(&tmp, TCPBUF->ackno, 4);
+   return ntohl(tmp);
+}
+
+/****************************************************************************
  * Function: send_interrupt
  *
  * Description:
@@ -106,51 +155,31 @@ static uint8 send_interrupt(struct uip_driver_s *dev, struct uip_conn *conn, uin
 
   nvdbg("flags: %02x state: %d\n", flags, pstate->snd_state);
 
-  /* If the data has not been sent OR if it needs to be retransmitted,
-   * then send it now.
+  /* If this packet contains an acknowledgement, then update the count of
+   * acknowldged bytes.
    */
 
-  if (pstate->snd_state != STATE_DATA_SENT || (flags & UIP_REXMIT) != 0)
+  if ((flags & UIP_ACKDATA) != 0)
     {
-      if (pstate->snd_buflen > uip_mss(conn))
-        {
-          uip_send(dev, pstate->snd_buffer, uip_mss(conn));
-        }
-      else
-        {
-          uip_send(dev, pstate->snd_buffer, pstate->snd_buflen);
-        }
-
-      pstate->snd_state = STATE_DATA_SENT;
-    }
-
-  /* Check if all data has been sent and acknowledged */
-
-  else if (pstate->snd_state == STATE_DATA_SENT && (flags & UIP_ACKDATA) != 0)
-    {
-      /* Yes.. the data has been sent AND acknowledged */
-
-      if (pstate->snd_buflen > uip_mss(conn))
-        {
-          /* Not all data has been sent */
+      /* The current acknowledgement number number is the (relative) offset of
+       * the of the next byte needed by the receiver.  The snd_isn is the offset
+       * of the first byte to send to the receiver.  The difference is the number
+       * of bytes to be acknowledged.
+       */
 
-          pstate->snd_sent   += uip_mss(conn);
-          pstate->snd_buflen -= uip_mss(conn);
-          pstate->snd_buffer += uip_mss(conn);
+      pstate->snd_acked = send_getackno(dev) - pstate->snd_isn;
+      nvdbg("ACK: acked=%d sent=%d buflen=%d\n",
+            pstate->snd_acked, pstate->snd_sent, pstate->snd_buflen);
 
-          /* Send again on the next poll */
+      /* Have all of the bytes in the buffer been sent and ACKed? */
 
-          pstate->snd_state = STATE_POLLWAIT;
-        }
-      else
+      if ( pstate->snd_acked >= pstate->snd_buflen)
         {
-          /* All data has been sent */
-
-          pstate->snd_sent   += pstate->snd_buflen;
-          pstate->snd_buffer += pstate->snd_buflen;
-          pstate->snd_buflen  = 0;
-
-          /* Don't allow any further call backs. */
+          /* Yes.  Then pstate->snd_len should hold the number of bytes actually
+           * sent.
+           *
+           * Don't allow any further call backs.
+           */
 
           conn->data_flags   = 0;
           conn->data_private = NULL;
@@ -161,9 +190,21 @@ static uint8 send_interrupt(struct uip_driver_s *dev, struct uip_conn *conn, uin
            */
 
           sem_post(&pstate->snd_sem);
+          return flags;
         }
     }
 
+  /* Check if we are being asked to retransmit data */
+
+  else if ((flags & UIP_REXMIT) != 0)
+    {
+      /* Yes.. in this case, reset the number of bytes that have been sent
+       * to the number of bytes that have been ACKed.
+       */
+
+      pstate->snd_sent = pstate->snd_acked;
+    }
+
  /* Check for a loss of connection */
 
   else if ((flags & (UIP_CLOSE|UIP_ABORT|UIP_TIMEDOUT)) != 0)
@@ -183,6 +224,32 @@ static uint8 send_interrupt(struct uip_driver_s *dev, struct uip_conn *conn, uin
       sem_post(&pstate->snd_sem);
     }
 
+  /* We get here if (1) not all of the data has been ACKed, (2) we have been
+   * asked to retransmit data, and (3) the connection is still healthy.
+   * We are now free to send more data to receiver.
+   */
+
+  if (pstate->snd_sent < pstate->snd_buflen)
+    {
+      /* Get the amount of data that we can send in the next packet */
+
+      uint32 sndlen = pstate->snd_buflen - pstate->snd_sent;
+      if (sndlen > uip_mss(conn))
+        {
+          sndlen = uip_mss(conn);
+        }
+
+      /* Then send that amount of data */
+
+      uip_send(dev, &pstate->snd_buffer[pstate->snd_sent], sndlen);
+
+      /* And update the amount of data sent (but not necessarily ACKed) */
+
+      pstate->snd_sent += sndlen;
+      nvdbg("SEND: acked=%d sent=%d buflen=%d\n",
+            pstate->snd_acked, pstate->snd_sent, pstate->snd_buflen);
+    }
+
   return flags;
 }
 
@@ -293,16 +360,19 @@ ssize_t send(int sockfd, const void *buf, size_t len, int flags)
   save                = irqsave();
   memset(&state, 0, sizeof(struct send_s));
   (void)sem_init(&state. snd_sem, 0, 0); /* Doesn't really fail */
-  state.snd_sock      = psock;
-  state.snd_buflen    = len;
-  state.snd_buffer    = buf;
-  state.snd_state     = STATE_POLLWAIT;
+  state.snd_sock      = psock;             /* Socket descriptor to use */
+  state.snd_buflen    = len;               /* Number of bytes to send */
+  state.snd_buffer    = buf;               /* Buffer to send from */
 
   if (len > 0)
     {
-      /* Set up the callback in the connection */
+     /* Get the initial sequence number that will be used */
 
       conn               = (struct uip_conn *)psock->s_conn;
+      state.snd_isn      = send_getisn(conn); /* Initial sequence number */
+
+      /* Set up the callback in the connection */
+
       conn->data_flags   = UIP_REXMIT|UIP_ACKDATA|UIP_CLOSE|UIP_ABORT|UIP_TIMEDOUT;
       conn->data_private = (void*)&state;
       conn->data_event   = send_interrupt;
diff --git a/nuttx/netutils/webserver/httpd.c b/nuttx/netutils/webserver/httpd.c
index 755d11892..9e37e46aa 100644
--- a/nuttx/netutils/webserver/httpd.c
+++ b/nuttx/netutils/webserver/httpd.c
@@ -78,8 +78,8 @@
 
 #define errno *get_errno_ptr()
 
-#define CONFIG_NETUTILS_HTTPD_DUMPBUFFER 1
-#undef  CONFIG_NETUTILS_HTTPD_DUMPPSTATE
+#undef CONFIG_NETUTILS_HTTPD_DUMPBUFFER
+#undef CONFIG_NETUTILS_HTTPD_DUMPPSTATE
 
 /****************************************************************************
  * Private Data