In order to boost performance, when doing the scan we use

	the approach of downloading part of the header of the total
	set of messages.  The it was done before is that the headers
	were ask one by one per message.  This was causing performance
	delay because we were doing a lot of small transaction on
	the TCP/IP stack.  By bundling the entire thing in one request
	there was much less transcation and the speed was increase
	by a factor of 10++.
	The Imap code folder_imap.c and mbx_imap.c is now messy.
	The rewrite in mailbox2 should clean this up though.

	* mailbox/folder_imap.c (imap_quoted_string): Allocated the
	memory instead.
	(imap_body): If we were doing the scan free the memory and create
	the cache header.
	(imap_fetch): Create the message if was not done, important for
	the scan.
	* mailbox/mbx_imap.c (imap_scan): Bundle one big request to
	fetch as much information as possible.
	(message_operation): Free the buffer when done.
	* mailbox/include/imap0.h: Add IMAP_SCAN states.

	* mailbox/file_stream.c (_file_readline): It seems that fgets() can
	not handle binary data.  Try to do our best.

 2001-10-11 Alain Magloire
 2001-10-11 Alain Magloire
 
 
+	In order to boost performance, when doing the scan we use
+	the approach of downloading part of the header of the total
+	set of messages.  The it was done before is that the headers
+	were ask one by one per message.  This was causing performance
+	delay because we were doing a lot of small transaction on
+	the TCP/IP stack.  By bundling the entire thing in one request
+	there was much less transcation and the speed was increase
+	by a factor of 10++.
+	The Imap code folder_imap.c and mbx_imap.c is now messy.
+	The rewrite in mailbox2 should clean this up though.
+
+	* mailbox/folder_imap.c (imap_quoted_string): Allocated the
+	memory instead.
+	(imap_body): If we were doing the scan free the memory and create
+	the cache header.
+	(imap_fetch): Create the message if was not done, important for
+	the scan.
+	* mailbox/mbx_imap.c (imap_scan): Bundle one big request to
+	fetch as much information as possible.
+	(message_operation): Free the buffer when done.
+	* mailbox/include/imap0.h: Add IMAP_SCAN states.
+
+	* mailbox/file_stream.c (_file_readline): It seems that fgets() can
+	not handle binary data.  Try to do our best.
+
+2001-10-11 Alain Magloire
+
 	* mailbox/header.c (header_get_fvalue): When it fails set the number
 	* mailbox/header.c (header_get_fvalue): When it fails set the number
 	of byte written to 0.
 	of byte written to 0.
 	(header_get_value): Likewised.
 	(header_get_value): Likewised.

-SUBDIR = mailutils
 pkginclude_HEADERS = \
 pkginclude_HEADERS = \
  address.h \
  address.h \
  attribute.h \
  attribute.h \

@@ -110,7 +110,11 @@ _file_readline (stream_t stream, char *optr, size_t osize,
       char *tmp = optr;
       char *tmp = optr;
       while (*tmp) tmp++; /* strlen(optr) */
       while (*tmp) tmp++; /* strlen(optr) */
       n = tmp - optr;
       n = tmp - optr;
-      fs->offset += n;
+      /* !!!!! WTF ??? */
+      if (n == 0)
+	n++;
+      else
+	fs->offset += n;
     }
     }
   else
   else
     {
     {

@@ -912,6 +912,8 @@ imap_quoted_string (f_imap_t f_imap, char **ptr)
       (*ptr)++;
       (*ptr)++;
     }
     }
   f_imap->callback.total = *ptr - bquote;
   f_imap->callback.total = *ptr - bquote;
+  f_imap->callback.buffer = calloc (f_imap->callback.total + 1, 1);
+  f_imap->callback.buflen = f_imap->callback.total;
   /* Fill the call back buffer. The if is redundant there should always
   /* Fill the call back buffer. The if is redundant there should always
      be enough room since the request is base on the buffer size.  */
      be enough room since the request is base on the buffer size.  */
   if (f_imap->callback.total <= f_imap->callback.buflen
   if (f_imap->callback.total <= f_imap->callback.buflen
@@ -940,6 +942,8 @@ imap_string (f_imap_t f_imap, char **ptr)
 	{
 	{
 	  (*ptr)++;
 	  (*ptr)++;
 	  /* Reset the buffer to the beginning.  */
 	  /* Reset the buffer to the beginning.  */
+	  f_imap->callback.buffer = calloc (f_imap->callback.nleft + 1, 1);
+	  f_imap->callback.buflen = f_imap->callback.nleft;
 	  f_imap->ptr = f_imap->buffer;
 	  f_imap->ptr = f_imap->buffer;
 	  status = imap_literal_string (f_imap, ptr);
 	  status = imap_literal_string (f_imap, ptr);
 	}
 	}
@@ -1333,26 +1337,22 @@ imap_body (f_imap_t f_imap, char **ptr)
 	  strncpy (section, *ptr, len);
 	  strncpy (section, *ptr, len);
 	  section[len] = '\0';
 	  section[len] = '\0';
 	  /* strupper.  */
 	  /* strupper.  */
-	  for (; *p; p++)
+	  for (; *p; p++) if (isupper((unsigned)*p)) *p = toupper ((unsigned)*p);
+	  /* Set the callback type to update the correct line count.  */
+	  //if (!strstr (section, "FIELD"))
 	    {
 	    {
-	      if (isalpha((unsigned)*p))
-		*p = toupper ((unsigned)*p);
-	    }
-	  /* Check to see the callback type to update the line count.  */
-	  if (!strstr (section, "FIELD"))
-	    {
-	      if (strstr (section, "MIME") || (strstr (section, "HEADER")))
-		{
-		  f_imap->callback.type = IMAP_HEADER;
-		}
-	      else if (strstr (section, "TEXT") || len > 0)
-		{
-		  f_imap->callback.type = IMAP_BODY;
-		}
-	      else if (len == 0) /* body[]  */
-		{
-		  f_imap->callback.type = IMAP_MESSAGE;
-		}
+              if (strstr (section, "MIME") || (strstr (section, "HEADER")))
+                {
+                  f_imap->callback.type = IMAP_HEADER;
+                }
+              else if (strstr (section, "TEXT") || len > 0)
+                {
+                  f_imap->callback.type = IMAP_BODY;
+                }
+              else if (len == 0) /* body[]  */
+                {
+                  f_imap->callback.type = IMAP_MESSAGE;
+                }
 	    }
 	    }
 	  sep++; /* Move pass the ']'  */
 	  sep++; /* Move pass the ']'  */
 	  *ptr = sep;
 	  *ptr = sep;
@@ -1370,7 +1370,17 @@ imap_body (f_imap_t f_imap, char **ptr)
 	}
 	}
     }
     }
   status = imap_string (f_imap, ptr);
   status = imap_string (f_imap, ptr);
-  f_imap->callback.type = IMAP_MESSAGE;
+  if (f_imap->state == IMAP_SCAN_ACK && f_imap->callback.total)
+    {
+      status = header_create (&f_imap->callback.msg_imap->fheader,
+			      f_imap->callback.buffer, f_imap->callback.total,
+			      NULL);
+      free (f_imap->callback.buffer);
+      f_imap->callback.buffer = NULL;
+      f_imap->callback.buflen = 0;
+      f_imap->callback.total = 0;
+      f_imap->callback.nleft = 0;
+    }
   return status;
   return status;
 }
 }
 
 
@@ -1479,6 +1489,8 @@ imap_fetch (f_imap_t f_imap)
     {
     {
       /* Find the imap mesg struct.  */
       /* Find the imap mesg struct.  */
       size_t i;
       size_t i;
+      message_t msg = NULL;
+      mailbox_get_message (m_imap->mailbox, msgno, &msg);
       for (i = 0; i < m_imap->imessages_count; i++)
       for (i = 0; i < m_imap->imessages_count; i++)
 	{
 	{
 	  if (m_imap->imessages[i] && m_imap->imessages[i]->num == msgno)
 	  if (m_imap->imessages[i] && m_imap->imessages[i]->num == msgno)
@@ -1487,6 +1499,7 @@ imap_fetch (f_imap_t f_imap)
 	      break;
 	      break;
 	    }
 	    }
 	}
 	}
+      /* message_destroy (&msg);  */
     }
     }
 
 
   while (*sp && *sp != ')')
   while (*sp && *sp != ')')
@@ -1781,7 +1794,6 @@ imap_parse (f_imap_t f_imap)
       /* Is the response untagged ?  */
       /* Is the response untagged ?  */
       if (tag && tag[0] == '*')
       if (tag && tag[0] == '*')
 	{
 	{
-
 	  /* Is it a Status Response.  */
 	  /* Is it a Status Response.  */
 	  if (strcasecmp (response, "OK") == 0)
 	  if (strcasecmp (response, "OK") == 0)
 	    {
 	    {
@@ -1923,6 +1935,7 @@ imap_parse (f_imap_t f_imap)
 	    }
 	    }
 	  else if (strcasecmp (response, "PREAUTH") == 0)
 	  else if (strcasecmp (response, "PREAUTH") == 0)
 	    {
 	    {
+	      /* Should we be dealing with this?  */
 	    }
 	    }
 	  else if (strcasecmp (response, "BYE") == 0)
 	  else if (strcasecmp (response, "BYE") == 0)
 	    {
 	    {

@@ -111,6 +111,7 @@ enum imap_state
   IMAP_NOOP, IMAP_NOOP_ACK,
   IMAP_NOOP, IMAP_NOOP_ACK,
   IMAP_OPEN_CONNECTION,
   IMAP_OPEN_CONNECTION,
   IMAP_RENAME, IMAP_RENAME_ACK,
   IMAP_RENAME, IMAP_RENAME_ACK,
+  IMAP_SCAN, IMAP_SCAN_ACK,
   IMAP_SELECT, IMAP_SELECT_ACK,
   IMAP_SELECT, IMAP_SELECT_ACK,
   IMAP_STORE, IMAP_STORE_ACK,
   IMAP_STORE, IMAP_STORE_ACK,
   IMAP_SUBSCRIBE, IMAP_SUBSCRIBE_ACK,
   IMAP_SUBSCRIBE, IMAP_SUBSCRIBE_ACK,

@@ -34,6 +34,10 @@
 #include <mailbox0.h>
 #include <mailbox0.h>
 #include <registrar0.h>
 #include <registrar0.h>
 #include <imap0.h>
 #include <imap0.h>
+#undef min
+#define min(a,b) ((a) < (b) ? (a) : (b))
+
+#define MU_IMAP_CACHE_HEADERS "Bcc Cc Content-Language Content-Transfer-Encoding Content-Type Date From In-Reply-To Message-ID Reference Reply-To Sender Subject To X-UIDL"
 
 
 /* Functions to overload the mailbox_t API.  */
 /* Functions to overload the mailbox_t API.  */
 static void mailbox_imap_destroy __P ((mailbox_t));
 static void mailbox_imap_destroy __P ((mailbox_t));
@@ -217,7 +221,7 @@ mailbox_imap_close (mailbox_t mailbox)
   f_imap_t f_imap = m_imap->f_imap;
   f_imap_t f_imap = m_imap->f_imap;
   int status = 0;
   int status = 0;
 
 
- /* Select first.  */
+  /* Select first.  */
   status = imap_messages_count (mailbox, NULL);
   status = imap_messages_count (mailbox, NULL);
   if (status != 0)
   if (status != 0)
     return status;
     return status;
@@ -279,7 +283,7 @@ imap_get_message (mailbox_t mailbox, size_t msgno, message_t *pmsg)
   int status = 0;
   int status = 0;
   size_t i;
   size_t i;
 
 
-  if (pmsg == NULL)
+  if (pmsg == NULL || msgno == 0 || msgno > m_imap->messages_count)
     return EINVAL;
     return EINVAL;
 
 
   monitor_rdlock (mailbox->monitor);
   monitor_rdlock (mailbox->monitor);
@@ -535,6 +539,8 @@ imap_scan (mailbox_t mailbox, size_t msgno, size_t *pcount)
   int status;
   int status;
   size_t i;
   size_t i;
   size_t count = 0;
   size_t count = 0;
+  m_imap_t m_imap = mailbox->data;
+  f_imap_t f_imap = m_imap->f_imap;
 
 
   /* Selected.  */
   /* Selected.  */
   status = imap_messages_count (mailbox, &count);
   status = imap_messages_count (mailbox, &count);
@@ -542,8 +548,38 @@ imap_scan (mailbox_t mailbox, size_t msgno, size_t *pcount)
     *pcount = count;
     *pcount = count;
   if (status != 0)
   if (status != 0)
     return status;
     return status;
+
+  switch (f_imap->state)
+    {
+    case IMAP_NO_STATE:
+      status = imap_writeline (f_imap,
+			       "g%d FETCH 1:* (FLAGS RFC822.SIZE BODY.PEEK[HEADER.FIELDS (%s)])\r\n",
+			       f_imap->seq++, MU_IMAP_CACHE_HEADERS);
+      CHECK_ERROR (f_imap, status);
+      MAILBOX_DEBUG0 (mailbox, MU_DEBUG_PROT, f_imap->buffer);
+      f_imap->state = IMAP_SCAN;
+
+    case IMAP_SCAN:
+      status = imap_send (f_imap);
+      CHECK_EAGAIN (f_imap, status);
+      f_imap->state = IMAP_SCAN_ACK;
+
+    case IMAP_SCAN_ACK:
+      status = imap_parse (f_imap);
+      CHECK_EAGAIN (f_imap, status);
+      MAILBOX_DEBUG0 (mailbox, MU_DEBUG_PROT, f_imap->buffer);
+
+    default:
+      CHECK_EAGAIN (f_imap, status);
+      return status;
+    }
+
+  f_imap->state = IMAP_NO_STATE;
+
+  /* If no callbacks bail out early.  */
   if (mailbox->observable == NULL)
   if (mailbox->observable == NULL)
     return 0;
     return 0;
+
   for (i = msgno; i <= *pcount; i++)
   for (i = msgno; i <= *pcount; i++)
     {
     {
       if (observable_notify (mailbox->observable, MU_EVT_MESSAGE_ADD) != 0)
       if (observable_notify (mailbox->observable, MU_EVT_MESSAGE_ADD) != 0)
@@ -1591,7 +1627,6 @@ imap_header_get_fvalue (header_t header, const char *field, char * buffer,
       status = imap_messages_count (m_imap->mailbox, NULL);
       status = imap_messages_count (m_imap->mailbox, NULL);
       if (status != 0)
       if (status != 0)
         return status;
         return status;
-#define MU_IMAP_CACHE_HEADERS "Bcc Cc Content-Language Content-Transfer-Encoding Content-Type Date From In-Reply-To Message-ID Reference Reply-To Sender Subject To X-UIDL"
       status = imap_writeline (f_imap,
       status = imap_writeline (f_imap,
                                "g%d FETCH %d (FLAGS RFC822.SIZE BODY.PEEK[HEADER.FIELDS (%s)])\r\n",
                                "g%d FETCH %d (FLAGS RFC822.SIZE BODY.PEEK[HEADER.FIELDS (%s)])\r\n",
                                f_imap->seq++, msg_imap->num,
                                f_imap->seq++, msg_imap->num,
@@ -1800,9 +1835,12 @@ message_operation (f_imap_t f_imap, msg_imap_t msg_imap, char *buffer,
       status = imap_send (f_imap);
       status = imap_send (f_imap);
       CHECK_EAGAIN (f_imap, status);
       CHECK_EAGAIN (f_imap, status);
       /* Set the callback,  we want the results.  */
       /* Set the callback,  we want the results.  */
-      f_imap->callback.buffer = buffer;
-      f_imap->callback.buflen = buflen;
+      if (f_imap->callback.buffer)
+	free (f_imap->callback.buffer);
+      f_imap->callback.buffer = NULL;
+      f_imap->callback.buflen = 0;
       f_imap->callback.total = 0;
       f_imap->callback.total = 0;
+      f_imap->callback.nleft = 0;
       f_imap->callback.msg_imap = msg_imap;
       f_imap->callback.msg_imap = msg_imap;
       f_imap->state = IMAP_FETCH_ACK;
       f_imap->state = IMAP_FETCH_ACK;
 
 
@@ -1816,15 +1854,22 @@ message_operation (f_imap_t f_imap, msg_imap_t msg_imap, char *buffer,
     default:
     default:
       break;
       break;
     }
     }
-  if (plen)
-    *plen = f_imap->callback.total;
 
 
   if (status == 0 && f_imap->isopen == 0 && f_imap->callback.total == 0)
   if (status == 0 && f_imap->isopen == 0 && f_imap->callback.total == 0)
     status = EBADF;
     status = EBADF;
+
   /* Clear the callback.  */
   /* Clear the callback.  */
+  buflen = min (buflen, f_imap->callback.total);
+  if (buffer && f_imap->callback.buffer)
+    memcpy (buffer, f_imap->callback.buffer, buflen);
+  if (plen)
+    *plen = buflen;
+  if (f_imap->callback.buffer)
+    free (f_imap->callback.buffer);
   f_imap->callback.buffer = NULL;
   f_imap->callback.buffer = NULL;
   f_imap->callback.buflen = 0;
   f_imap->callback.buflen = 0;
   f_imap->callback.total = 0;
   f_imap->callback.total = 0;
+  f_imap->callback.nleft = 0;
   f_imap->callback.type = 0;
   f_imap->callback.type = 0;
   f_imap->callback.msg_imap = NULL;
   f_imap->callback.msg_imap = NULL;
   f_imap->state = IMAP_NO_STATE;
   f_imap->state = IMAP_NO_STATE;