Finish pop3 mailbox implementation.

* mailbox/msgscan.c: New file.
* mailbox/Makefile.am (libmailutils_la_SOURCES): Add it.

* include/mailutils/body.h (mu_body_set_get_stream): New prototype.
* include/mailutils/message.h (MU_SCAN_SEEK, MU_SCAN_SIZE): New
defines.
(mu_message_scan): New structure.
(mu_stream_scan_message): New prototype.
(mu_message_set_get_stream): New prototype.
* include/mailutils/stream.h (mu_stream_copy): Change signature: takes
4 arguments now.
* include/mailutils/sys/body.h (_mu_body) <_get_stream>: New method.
* include/mailutils/sys/message.h (_mu_message) <_get_stream>: New method.
* mailbox/body.c (_body_get_stream): Call _get_stream, if provided.
* mailbox/message.c (_message_get_stream): Call _get_stream, if provided.

* mailbox/stream.c (_stream_flush_buffer): Avoid infinite recursion:
call stream->seek directly.
* mailbox/streamcpy.c (mu_stream_copy): Return the number of bytes
actually copied in the fourth argument. All uses updated.
* mailbox/streamref.c (streamref_return): Do not propagate internal
flags.
(_streamref_readdelim): Ensure there is enough buffer space for the
mu_stream_readdelim call.

* libproto/pop/mbox.c: Finish client implementation.

* mail/print.c (mail_print_msg): Close pager before returning on
error.

@@ -55,7 +55,7 @@ c_copy (mu_stream_t out, mu_stream_t in)
 	}
     }
   else
-    MU_ASSERT (mu_stream_copy (out, in, 0));
+    MU_ASSERT (mu_stream_copy (out, in, 0, NULL));
   mu_stream_write (out, "\n", 1, NULL);
   mu_stream_close (out);
   mu_stream_close (in);

@@ -69,14 +69,14 @@ main (int argc, char * argv [])
       MU_ASSERT (mu_stream_seek (in, skip_off, MU_SEEK_SET, NULL));
     }
   
-  MU_ASSERT (mu_stream_copy (out, in, 0));
+  MU_ASSERT (mu_stream_copy (out, in, 0, NULL));
 
   if (reread_option)
     {
       mu_stream_printf (out, "rereading from %lu:\n",
 			(unsigned long) reread_off);
       MU_ASSERT (mu_stream_seek (in, reread_off, MU_SEEK_SET, NULL));
-      MU_ASSERT (mu_stream_copy (out, in, 0));
+      MU_ASSERT (mu_stream_copy (out, in, 0, NULL));
     }
   
   mu_stream_close (in);

@@ -152,7 +152,7 @@ sc2string (int rc)
 int
 io_copy_out (mu_stream_t str, size_t size)
 {
-  return mu_stream_copy (iostream, str, size);
+  return mu_stream_copy (iostream, str, size, NULL);
 }
 
 int

@@ -36,7 +36,10 @@ extern int mu_body_get_stream     (mu_body_t, mu_stream_t *)
                                       __attribute__ ((deprecated));
 extern int mu_body_get_streamref  (mu_body_t body, mu_stream_t *pstream);
 extern int mu_body_set_stream     (mu_body_t, mu_stream_t, void *owner);
-
+extern int mu_body_set_get_stream (mu_body_t,
+				   int (*) (mu_body_t, mu_stream_t *),
+				   void *owner);
+  
 extern int mu_body_get_filename   (mu_body_t, char *, size_t, size_t *);
 
 extern int mu_body_size           (mu_body_t, size_t *);

@@ -26,6 +26,25 @@
 extern "C" {
 #endif
 
+#define MU_SCAN_SEEK  0x01
+#define MU_SCAN_SIZE  0x02
+
+struct mu_message_scan
+{
+  int flags;
+  mu_off_t message_start;
+  mu_off_t message_size;
+
+  mu_off_t body_start;
+  mu_off_t body_end;
+  size_t header_lines;
+  size_t body_lines;
+  int attr_flags;
+  unsigned long uidvalidity;
+};
+
+int mu_stream_scan_message (mu_stream_t stream, struct mu_message_scan *sp);
+  
 /* A message is considered to be a container for:
   mu_header_t, mu_body_t, and its mu_attribute_t.  */
 
@@ -34,7 +53,7 @@ extern void mu_message_destroy (mu_message_t *, void *owner);
 
 extern int mu_message_create_copy (mu_message_t *to, mu_message_t from);
 
-extern void * mu_message_get_owner (mu_message_t);
+extern void *mu_message_get_owner (mu_message_t);
 extern int mu_message_is_modified (mu_message_t);
 extern int mu_message_clear_modified (mu_message_t);
 extern int mu_message_get_mailbox (mu_message_t, mu_mailbox_t *);
@@ -64,6 +83,9 @@ extern int mu_message_set_attribute (mu_message_t, mu_attribute_t, void *);
 
 extern int mu_message_get_observable (mu_message_t, mu_observable_t *);
 
+extern int mu_message_set_get_stream (mu_message_t,
+				      int (*) (mu_message_t, mu_stream_t *),
+				      void *);
 extern int mu_message_is_multipart (mu_message_t, int *);
 extern int mu_message_set_is_multipart (mu_message_t, 
 					int (*_is_multipart) (mu_message_t,

@@ -115,7 +115,8 @@ int mu_stream_clr_flags (mu_stream_t stream, int fl);
 int mu_stream_vprintf (mu_stream_t str, const char *fmt, va_list ap);
 int mu_stream_printf (mu_stream_t stream, const char *fmt, ...);
 
-int mu_stream_copy (mu_stream_t dst, mu_stream_t src, size_t size);
+int mu_stream_copy (mu_stream_t dst, mu_stream_t src, mu_off_t size,
+		    mu_off_t *pcsz);
 
 
 int mu_file_stream_create (mu_stream_t *pstream, const char *filename, int flags);

@@ -40,6 +40,7 @@ struct _mu_body
 
   int (*_size)  (mu_body_t, size_t*);
   int (*_lines) (mu_body_t, size_t*);
+  int (*_get_stream) (mu_body_t, mu_stream_t *);
 };
 
 #ifdef __cplusplus

@@ -52,6 +52,7 @@ struct _mu_message
   /* Reference count.  */
   int ref;
 
+  int (*_get_stream)     (mu_message_t, mu_stream_t *);
   int (*_get_uidl)       (mu_message_t, char *, size_t, size_t *);
   int (*_get_uid)        (mu_message_t, size_t *);
   int (*_get_qid)        (mu_message_t,	mu_message_qid_t *);

@@ -219,7 +219,7 @@ SCM_DEFINE_PUBLIC (scm_mu_message_copy, "mu-message-copy", 1, 0, 0,
 		    "Cannot get output stream", SCM_BOOL_F);
     }
 
-  status = mu_stream_copy (out, in, 0);
+  status = mu_stream_copy (out, in, 0, NULL);
   mu_stream_destroy (&in);
   mu_stream_destroy (&out);
   if (status)

@@ -252,7 +252,7 @@ mime_create_quote (mu_mime_t mime, mu_message_t msg)
   mu_body_get_streamref (body, &ostream);
   mu_message_get_streamref (msg, &istream);
 
-  rc = mu_stream_copy (ostream, istream, 0);
+  rc = mu_stream_copy (ostream, istream, 0, NULL);
 
   mu_stream_destroy (&istream);
   mu_stream_close (ostream);

@@ -118,7 +118,7 @@ sieve_action_pipe (mu_sieve_machine_t mach, mu_list_t args, mu_list_t tags)
 	  ONERR (rc, _("stream write failed"), NULL);
 	}
 
-      rc = mu_stream_copy (pstr, mstr, 0);
+      rc = mu_stream_copy (pstr, mstr, 0, NULL);
       ONERR (rc, _("command failed"), cmd);
     }
   while (0);

@@ -117,7 +117,7 @@ spamd_send_message (mu_stream_t stream, mu_message_t msg)
       return rc;
     }
 
-  rc = mu_stream_copy (stream, flt, 0);
+  rc = mu_stream_copy (stream, flt, 0, NULL);
 
   mu_stream_destroy (&mstr);
   mu_stream_destroy (&flt);

@@ -111,7 +111,7 @@ build_mime (mu_sieve_machine_t mach, mu_list_t tags, mu_mime_t *pmime,
     }
 
   mu_stream_seek (input, 0, MU_SEEK_SET, NULL);
-  rc = mu_stream_copy (stream, input, 0);
+  rc = mu_stream_copy (stream, input, 0, NULL);
   if (rc)
     {
       mu_sieve_error (mach,

@@ -1086,7 +1086,7 @@ append_message_to_stream (mu_stream_t ostr, mu_message_t msg,
       if (status)
 	return status;
     }
-  status = mu_stream_copy (ostr, istr, 0);
+  status = mu_stream_copy (ostr, istr, 0, NULL);
   mu_stream_destroy (&istr);
   if (status == 0)
     status = mu_stream_write (ostr, "\n", 1, NULL);
@@ -1264,7 +1264,7 @@ mbox_expunge_unlocked (mu_mailbox_t mailbox, size_t dirty, int remove_deleted,
 	      return status;
 	    }
 	  status = mu_stream_copy (tempstr, mailbox->stream,
-				   mum->body_end - mum->envel_from);
+				   mum->body_end - mum->envel_from, NULL);
 	  if (status)
 	    {
 	      mu_error (_("%s:%d: error copying: %s"),
@@ -1295,7 +1295,7 @@ mbox_expunge_unlocked (mu_mailbox_t mailbox, size_t dirty, int remove_deleted,
 	      return status;
 	    }
   
-	  status = mu_stream_copy (tempstr, mailbox->stream, len);
+	  status = mu_stream_copy (tempstr, mailbox->stream, len, NULL);
 	  if (status)
 	    {
 	      mu_error (_("%s:%d: error writing to temporary stream: %s"),
@@ -1342,7 +1342,7 @@ mbox_expunge_unlocked (mu_mailbox_t mailbox, size_t dirty, int remove_deleted,
       return status;
     }
 
-  status = mu_stream_copy (mailbox->stream, tempstr, size);
+  status = mu_stream_copy (mailbox->stream, tempstr, size, NULL);
   if (status)
     {
       mu_error (_("%s:%d: copying from the temporary stream: %s"),

@@ -41,6 +41,7 @@
 #include <mailutils/observer.h>
 #include <mailutils/property.h>
 #include <mailutils/stream.h>
+#include <mailutils/filter.h>
 #include <mailutils/url.h>
 #include <mailutils/secret.h>
 #include <mailutils/tls.h>
@@ -56,22 +57,24 @@
 #include <mailutils/sys/registrar.h>
 #include <mailutils/sys/url.h>
 
-#define _POP3_MSG_INBODY  0x01
-#define _POP3_MSG_SKIPHDR 0x02
-#define _POP3_MSG_SKIPBDY 0x04
+#define _POP3_MSG_CACHED  0x01      /* Message is already cached */
+#define _POP3_MSG_SIZE    0x02      /* Message size obtained */
+#define _POP3_MSG_SCANNED 0x04      /* Message has been scanned */
+#define _POP3_MSG_ATTRSET 0x08      /* Attributes has been set */
 
 struct _pop3_message
 {
   int flags;
-  size_t body_size;
-  size_t header_size;
-  size_t body_lines;
-  size_t header_lines;
-  size_t message_size;
-  size_t num;
-  char *uidl; /* Cache the uidl string.  */
-  int attr_flags;
-  mu_message_t message;
+  mu_off_t offset;          /* Offset in the message cache stream */
+  mu_off_t body_start;      /* Start of message, relative to offset */
+  mu_off_t body_end;        /* End of message, relative to offset */  
+  size_t header_lines;      /* Number of lines in the header */
+  size_t body_lines;        /* Number of lines in the body */
+  int attr_flags;           /* Message attributes */
+  size_t message_size;      /* Message size */ 
+  size_t num;               /* Message number */
+  char *uidl;               /* Cached uidl string.  */
+  mu_message_t message;     /* Pointer to the message structure */ 
   struct _pop3_mailbox *mpd; /* Back pointer.  */
 };
   
@@ -87,10 +90,16 @@ struct _pop3_mailbox
   size_t msg_max;             /* Actual size of the array */  
   mu_mailbox_t mbox;          /* MU mailbox corresponding to this one. */
 
-  char *user;     /* Temporary holders for user and passwd.  */
+  mu_stream_t cache;          /* Message cache stream */
+   /* Temporary holders for user and passwd: */
+  char *user;    
   mu_secret_t secret;
 };
 
+
+/* ------------------------------------------------------------------------- */
+/* Basic operations */
+
 static int
 pop_open (mu_mailbox_t mbox, int flags)
 {
@@ -193,6 +202,7 @@ pop_close (mu_mailbox_t mbox)
   if (status)
     mu_error ("mu_pop3_disconnect failed: %s", mu_strerror (status));
   mu_pop3_destroy (&mpd->pop3);
+  mu_stream_destroy (&mpd->cache);
   return 0;
 }
 
@@ -220,6 +230,7 @@ pop_destroy (mu_mailbox_t mbox)
 	free (mpd->user);
       if (mpd->secret)
 	mu_secret_unref (mpd->secret);
+      mu_stream_destroy (&mpd->cache);
    }
 }
 
@@ -319,6 +330,156 @@ pop_get_size (mu_mailbox_t mbox, mu_off_t *psize)
 }
 
 
+/* ------------------------------------------------------------------------- */
+/* POP3 message streams */
+
+static void
+pop_stream_drain (mu_stream_t str)
+{
+  char buf[2048];
+  size_t size;
+
+  while (mu_stream_read (str, buf, sizeof buf, &size) == 0 && size)
+    ;
+}
+     
+static int
+_pop_message_get_stream (struct _pop3_message *mpm, mu_stream_t *pstr)
+{
+  int status;
+  struct _pop3_mailbox *mpd = mpm->mpd;
+  
+  if (!(mpm->flags & _POP3_MSG_CACHED))
+    {
+      mu_stream_t stream;
+      mu_off_t size;
+      
+      status = mu_pop3_retr (mpd->pop3, mpm->num, &stream);
+      if (status)
+	return status;
+
+      do
+	{
+	  mu_stream_t flt;
+	  
+	  if (!mpd->cache)
+	    {
+	      status = mu_temp_file_stream_create (&mpd->cache, NULL);
+	      if (status)
+		/* FIXME: Try to recover first */
+		break;
+
+	      status = mu_stream_open (mpd->cache);
+	      if (status)
+		{
+		  mu_stream_destroy (&mpd->cache);
+		  break;
+		}
+	      mu_stream_set_buffer (mpd->cache, mu_buffer_full, 8192);
+	    }
+
+	  status = mu_stream_size (mpd->cache, &mpm->offset);
+	  if (status)
+	    break;
+
+	  status = mu_filter_create (&flt, stream, "CRLF", MU_FILTER_DECODE,
+				     MU_STREAM_READ);
+	  if (status)
+	    break;
+
+	  status = mu_stream_copy (mpd->cache, flt, 0, &size);
+
+	  mu_stream_destroy (&flt);
+	}
+      while (0);
+
+      if (status)
+	{
+	  pop_stream_drain (stream);
+	  mu_stream_unref (stream);
+	  return status;
+	}
+
+      mu_stream_unref (stream);
+
+      mpm->message_size = size; /* FIXME: Possible overflow. */
+
+      mpm->flags |= _POP3_MSG_CACHED | _POP3_MSG_SIZE;
+    }
+  return mu_streamref_create_abridged (pstr, mpd->cache,
+				       mpm->offset,
+				       mpm->offset + mpm->message_size - 1);
+}
+
+static int
+pop_message_get_stream (mu_message_t msg, mu_stream_t *pstr)
+{
+  struct _pop3_message *mpm = mu_message_get_owner (msg);
+  return _pop_message_get_stream (mpm, pstr);
+}
+        
+static int
+pop_scan_message (struct _pop3_message *mpm)
+{
+  int status;
+  mu_stream_t stream;
+  struct mu_message_scan scan;
+
+  if (mpm->flags & _POP3_MSG_SCANNED)
+    return 0;
+  
+  status = _pop_message_get_stream (mpm, &stream);
+  if (status)
+    return status;
+      
+  scan.flags = MU_SCAN_SEEK | MU_SCAN_SIZE;
+  scan.message_start = 0;
+  scan.message_size = mpm->message_size;
+  status = mu_stream_scan_message (stream, &scan);
+  mu_stream_unref (stream);
+
+  if (status == 0)
+    {
+      mpm->body_start = scan.body_start;
+      mpm->body_end = scan.body_end;
+      mpm->header_lines = scan.header_lines;
+      mpm->body_lines = scan.body_lines;
+      if (!(mpm->flags & _POP3_MSG_ATTRSET))
+	{
+	  mpm->attr_flags = scan.attr_flags;
+	  mpm->flags |= _POP3_MSG_ATTRSET;
+	}
+
+      mpm->flags |= _POP3_MSG_SCANNED;
+    }
+  
+  return status;
+}
+
+
+static int
+pop_message_size (mu_message_t msg, size_t *psize)
+{
+  struct _pop3_message *mpm = mu_message_get_owner (msg);
+  struct _pop3_mailbox *mpd = mpm->mpd;
+
+  if (mpm == NULL)
+    return EINVAL;
+
+  if (!(mpm->flags & _POP3_MSG_SIZE))
+    {
+      /* FIXME: The size obtained this way may differ from the actual one
+	 by the number of lines in the message. */
+      int status = mu_pop3_list (mpd->pop3, mpm->num, &mpm->message_size);
+      if (status)
+	return status;
+      mpm->flags |= _POP3_MSG_SIZE;
+    }
+  if (psize)
+    *psize = mpm->message_size;
+  return 0;
+}
+
 static int
 pop_create_message (struct _pop3_message *mpm, struct _pop3_mailbox *mpd)
 {
@@ -328,7 +489,9 @@ pop_create_message (struct _pop3_message *mpm, struct _pop3_mailbox *mpd)
   status = mu_message_create (&msg, mpm);
   if (status)
     return status;
-  // FIXME...
+
+  mu_message_set_get_stream (msg, pop_message_get_stream, mpm);
+  mu_message_set_size (msg, pop_message_size, mpm);
   mpm->message = msg;
   return 0;
 }
@@ -337,46 +500,34 @@ pop_create_message (struct _pop3_message *mpm, struct _pop3_mailbox *mpd)
 /* ------------------------------------------------------------------------- */
 /* Header */
 
-static int
-pop_header_fill (void *data, char **pbuf, size_t *plen)
+int
+pop_header_blurb (mu_stream_t stream, size_t maxlines,
+		  char **pbuf, size_t *plen)
 {
-  struct _pop3_message *mpm = data;
-  struct _pop3_mailbox *mpd = mpm->mpd;
-  mu_stream_t stream;
-  mu_opool_t opool;
   int status;
+  mu_opool_t opool;
+  size_t size = 0;
+  char *buf = NULL;
+  size_t n;
+  size_t nlines = 0;
   
   status = mu_opool_create (&opool, 0);
   if (status)
     return status;
-  
-  if (mu_pop3_capa_test (mpd->pop3, "TOP", NULL) == 0)
-    status = mu_pop3_top (mpd->pop3, mpm->num, 0, &stream);
-  else
-    status = mu_pop3_retr (mpd->pop3, mpm->num, &stream);
-
+      
+  while ((status = mu_stream_getline (stream, &buf, &size, &n)) == 0 && n > 0)
+    {
+      size_t len = mu_rtrim_cset (buf, "\r\n");
+      if (len == 0)
+	break;
+      mu_opool_append (opool, buf, len);
+      mu_opool_append_char (opool, '\n');
+      if (maxlines && ++nlines >= maxlines)
+	break;
+    }
+      
   if (status == 0)
     {
-      size_t size = 0;
-      char *buf = NULL;
-      size_t n;
-
-      while (mu_stream_getline (stream, &buf, &size, &n) == 0
-	     && n > 0)
-	{
-	  size_t len = mu_rtrim_cset (buf, "\r\n");
-	  if (len == 0)
-	    break;
-	  mu_opool_append (opool, buf, len);
-	  mu_opool_append_char (opool, '\n');
-	}
-
-      if (!mu_stream_eof (stream))
-	/* Drain the stream. */
-	while (mu_stream_getline (stream, &buf, &size, &n) == 0
-	       && n > 0);
-      mu_stream_destroy (&stream);
-
       n = mu_opool_size (opool);
       if (n > size)
 	{
@@ -384,19 +535,53 @@ pop_header_fill (void *data, char **pbuf, size_t *plen)
 	  if (!p)
 	    {
 	      free (buf);
-	      mu_opool_destroy (&opool);
-	      return ENOMEM;
+	      status = ENOMEM;
 	    }
-	  buf = p;
+	  else
+	    buf = p;
 	}
+    }
 
+  if (status == 0)
+    {
       mu_opool_copy (opool, buf, n);
       *pbuf = buf;
       *plen = n;
-      status = 0;
     }
+  else
+    free (buf);
   mu_opool_destroy (&opool);
-  
+
+  return 0;
+}
+
+static int
+pop_header_fill (void *data, char **pbuf, size_t *plen)
+{
+  struct _pop3_message *mpm = data;
+  struct _pop3_mailbox *mpd = mpm->mpd;
+  mu_stream_t stream;
+  int status;
+
+  if (mpm->flags & _POP3_MSG_SCANNED)
+    status = _pop_message_get_stream (mpm, &stream);
+  else
+    {
+      status = mu_pop3_top (mpd->pop3, mpm->num, 0, &stream);
+      if (status == 0)
+	{
+	  status = pop_header_blurb (stream, 0, pbuf, plen);
+	  if (!mu_stream_eof (stream))
+	    pop_stream_drain (stream);
+	  mu_stream_destroy (&stream);
+	  return status;
+	}
+      else
+	status = _pop_message_get_stream (mpm, &stream);
+    }
+
+  status = pop_header_blurb (stream, mpm->header_lines, pbuf, plen);
+  mu_stream_destroy (&stream);
   return status;
 }
 
@@ -437,7 +622,7 @@ pop_get_attribute (mu_attribute_t attr, int *pflags)
 
   if (mpm == NULL || pflags == NULL)
     return EINVAL;
-  if (mpm->attr_flags == 0)
+  if (!(mpm->flags & _POP3_MSG_ATTRSET))
     {
       hdr_status[0] = '\0';
 
@@ -458,6 +643,7 @@ pop_set_attribute (mu_attribute_t attr, int flags)
   if (mpm == NULL)
     return EINVAL;
   mpm->attr_flags |= flags;
+  mpm->flags |= _POP3_MSG_ATTRSET;
   return 0;
 }
 
@@ -469,6 +655,7 @@ pop_unset_attribute (mu_attribute_t attr, int flags)
   if (mpm == NULL)
     return EINVAL;
   mpm->attr_flags &= ~flags;
+  mpm->flags |= _POP3_MSG_ATTRSET;
   return 0;
 }
 
@@ -492,11 +679,58 @@ pop_create_attribute (struct _pop3_message *mpm)
 /* ------------------------------------------------------------------------- */
 /* Body */
 
+int
+pop_body_get_stream (mu_body_t body, mu_stream_t *pstr)
+{
+  struct _pop3_message *mpm = mu_body_get_owner (body);
+  struct _pop3_mailbox *mpd = mpm->mpd;
+  int status = pop_scan_message (mpm);
+  if (status)
+    return status;
+  return mu_streamref_create_abridged (pstr, mpd->cache,
+				       mpm->offset + mpm->body_start,
+				       mpm->offset + mpm->body_end - 1);
+}
+
+static int
+pop_body_size (mu_body_t body, size_t *psize)
+{
+  struct _pop3_message *mpm = mu_body_get_owner (body);
+  int status = pop_scan_message (mpm);
+  if (status)
+    return status;
+  *psize = mpm->body_end - mpm->body_start;
+  return 0;
+}
+
+static int
+pop_body_lines (mu_body_t body, size_t *plines)
+{
+  struct _pop3_message *mpm = mu_body_get_owner (body);
+  int status = pop_scan_message (mpm);
+  if (status)
+    return status;
+  *plines = mpm->body_lines;
+  return 0;
+}
+
 static int
 pop_create_body (struct _pop3_message *mpm)
 {
-  /* FIXME */
-  return ENOSYS;
+  int status;
+  mu_body_t body = NULL;
+
+  status = mu_body_create (&body, mpm);
+  if (status)
+    return status;
+
+  mu_body_set_get_stream (body, pop_body_get_stream, mpm);
+  mu_body_set_size (body, pop_body_size, mpm);
+  mu_body_set_lines (body, pop_body_lines, mpm);
+
+  mu_message_set_body (mpm->message, body, mpm);
+
+  return 0;
 }
 
 
@@ -568,9 +802,9 @@ pop_get_message (mu_mailbox_t mbox, size_t msgno, mu_message_t *pmsg)
       if (!mpd->msg)
 	return ENOMEM;
     }
-  if (mpd->msg[msgno])
+  if (mpd->msg[msgno - 1])
     {
-      *pmsg = mpd->msg[msgno]->message;
+      *pmsg = mpd->msg[msgno - 1]->message;
       return 0;
     }
 
@@ -613,13 +847,45 @@ pop_get_message (mu_mailbox_t mbox, size_t msgno, mu_message_t *pmsg)
 
   mu_message_set_uid (mpm->message, pop_uid, mpm);
 
-  mpd->msg[msgno] = mpm;
+  mpd->msg[msgno - 1] = mpm;
   mu_message_set_mailbox (mpm->message, mbox, mpm);
   *pmsg = mpm->message;
   return 0;
 }
 
 static int
+pop_expunge (mu_mailbox_t mbox)
+{
+  struct _pop3_mailbox *mpd = mbox->data;
+  int status = 0;
+  size_t i;
+  
+  if (mpd == NULL)
+    return EINVAL;
+
+  if (!mpd->msg)
+    return 0;
+  
+  for (i = 0; i < mpd->msg_count; i++)
+    {
+      struct _pop3_message *mpm = mpd->msg[i];
+
+      if (mpm &&
+	  (mpm->flags & _POP3_MSG_ATTRSET) &&
+	  (mpm->attr_flags & MU_ATTRIBUTE_DELETED))
+	{
+	  status = mu_pop3_dele (mpd->pop3, mpm->num);
+	  if (status)
+	    break;
+	}
+    }
+  return 0;
+}
+
+/* ------------------------------------------------------------------------- */
+/* Initialization */
+
+static int
 _pop3_mailbox_init (mu_mailbox_t mbox, int pops)
 {
   struct _pop3_mailbox *mpd;
@@ -649,13 +915,10 @@ _pop3_mailbox_init (mu_mailbox_t mbox, int pops)
   mbox->_message_unseen = pop_message_unseen;
   mbox->_get_size = pop_get_size;
   
-#if 0 //FIXME
   /* Messages.  */
   mbox->_get_message = pop_get_message;
   mbox->_expunge = pop_expunge;
 
-
-
   /* Set our properties.  */
   {
     mu_property_t property = NULL;
@@ -663,7 +926,6 @@ _pop3_mailbox_init (mu_mailbox_t mbox, int pops)
     mu_property_set_value (property, "TYPE", "POP3", 1);
   }
 
-#endif
   /* Hack! POP does not really have a folder.  */
   mbox->folder->data = mbox;
   return status;
@@ -682,6 +944,7 @@ _mailbox_pops_init (mu_mailbox_t mbox)
 }
 
 
+/* ------------------------------------------------------------------------- */
 /* Authentication */
 
 /* Extract the User from the URL or the ticket.  */

@@ -424,7 +424,7 @@ run_metamail (const char *mailcap_cmd, mu_message_t mesg)
 	      mu_error ("mu_stream_open: %s", mu_strerror (status));
 	      break;
 	    }
-	  mu_stream_copy (pstr, stream, 0);
+	  mu_stream_copy (pstr, stream, 0, NULL);
 	  mu_stream_close (pstr);
 	  mu_stream_destroy (&pstr);
 	  exit (0);

@@ -98,6 +98,8 @@ mail_print_msg (msgset_t *mspec, mu_message_t mesg, void *data)
   if (status)
     {
       mu_error (_("get_stream error: %s"), mu_strerror (status));
+      if (out != ofile)
+	pclose (out);
       return 0;
     }
 

@@ -108,6 +108,7 @@ libmailutils_la_SOURCES = \
  mimehdr.c\
  mkfilename.c\
  monitor.c\
+ msgscan.c\
  msrv.c\
  mu_auth.c\
  muctype.c\

@@ -161,36 +161,46 @@ _body_get_stream (mu_body_t body, mu_stream_t *pstream, int ref)
 
   if (body->stream == NULL)
     {
-      int status;
-      struct _mu_body_stream *str =
-	(struct _mu_body_stream *)
-	   _mu_stream_create (sizeof (*str),
-			      MU_STREAM_RDWR|MU_STREAM_SEEK);
-      if (!str)
-	return ENOMEM;
-      
-      /* Create the temporary file.  */
-      body->filename = mu_tempname (NULL);
-      status = mu_file_stream_create (&body->fstream, 
-				      body->filename, MU_STREAM_RDWR);
-      if (status != 0)
-	return status;
-      status = mu_stream_open (body->fstream);
-      if (status != 0)
-	return status;
-      str->stream.ctl = _body_ioctl;
-      str->stream.read = _body_read;
-      str->stream.write = _body_write;
-      str->stream.truncate = _body_truncate;
-      str->stream.size = _body_size;
-      str->stream.seek = _body_seek;
-      str->stream.flush = _body_flush;
-      str->body = body;
-      body->stream = (mu_stream_t) str;
-      /* Override the defaults.  */
-      body->_lines = _body_get_lines;
-      body->_size = _body_get_size;
+      if (body->_get_stream)
+	{
+	  int status = body->_get_stream (body, &body->stream);
+	  if (status)
+	    return status;
+	}
+      else
+	{
+	  int status;
+	  struct _mu_body_stream *str =
+	    (struct _mu_body_stream *)
+	    _mu_stream_create (sizeof (*str),
+			       MU_STREAM_RDWR|MU_STREAM_SEEK);
+	  if (!str)
+	    return ENOMEM;
+	  
+	  /* Create the temporary file.  */
+	  body->filename = mu_tempname (NULL);
+	  status = mu_file_stream_create (&body->fstream, 
+					  body->filename, MU_STREAM_RDWR);
+	  if (status != 0)
+	    return status;
+	  status = mu_stream_open (body->fstream);
+	  if (status != 0)
+	    return status;
+	  str->stream.ctl = _body_ioctl;
+	  str->stream.read = _body_read;
+	  str->stream.write = _body_write;
+	  str->stream.truncate = _body_truncate;
+	  str->stream.size = _body_size;
+	  str->stream.seek = _body_seek;
+	  str->stream.flush = _body_flush;
+	  str->body = body;
+	  body->stream = (mu_stream_t) str;
+	  /* Override the defaults.  */
+	  body->_lines = _body_get_lines;
+	  body->_size = _body_get_size;
+	}
     }
+  
   if (!ref)
     {
       *pstream = body->stream;
@@ -227,7 +237,21 @@ mu_body_set_stream (mu_body_t body, mu_stream_t stream, void *owner)
 }
 
 int
-mu_body_set_lines (mu_body_t body, int (*_lines) (mu_body_t, size_t *), void *owner)
+mu_body_set_get_stream (mu_body_t body,
+			int (*_getstr) (mu_body_t, mu_stream_t *),
+			void *owner)
+{
+  if (body == NULL)
+    return EINVAL;
+  if (body->owner != owner)
+    return EACCES;
+  body->_get_stream = _getstr;
+  return 0;
+}
+
+int
+mu_body_set_lines (mu_body_t body, int (*_lines) (mu_body_t, size_t *),
+		   void *owner)
 {
   if (body == NULL)
     return EINVAL;

@@ -597,7 +597,7 @@ mu_message_create_copy (mu_message_t *to, mu_message_t from)
       return status;
     }
 
-  status = mu_stream_copy (tmp, fromstr, 0);
+  status = mu_stream_copy (tmp, fromstr, 0, NULL);
   if (status == 0)
     {
       status = mu_message_create (to, NULL);
@@ -797,6 +797,8 @@ mu_message_set_stream (mu_message_t msg, mu_stream_t stream, void *owner)
 static int
 _message_get_stream (mu_message_t msg, mu_stream_t *pstream, int ref)
 {
+  int status;
+
   if (msg == NULL)
     return EINVAL;
   if (pstream == NULL)
@@ -804,25 +806,33 @@ _message_get_stream (mu_message_t msg, mu_stream_t *pstream, int ref)
 
   if (msg->stream == NULL)
     {
-      int status;
-      mu_header_t hdr;
-      mu_body_t body;
+      if (msg->_get_stream)
+	{
+	  status = msg->_get_stream (msg, &msg->stream);
+	  if (status)
+	    return status;
+	}
+      else
+	{
+	  mu_header_t hdr;
+	  mu_body_t body;
 
-      /* FIXME: Kind of a kludge: make sure the message has header
-	 and body initialized. */
-      status = mu_message_get_header (msg, &hdr);
-      if (status)
-	return status;
-      status = mu_message_get_body (msg, &body);
-      if (status)
-	return status;
-      
-      status = _message_stream_create (&msg->stream, msg, MU_STREAM_RDWR);
-      if (status)
-	return status;
-      msg->flags |= MESSAGE_INTERNAL_STREAM;
+	  /* FIXME: Kind of a kludge: make sure the message has header
+	     and body initialized. */
+	  status = mu_message_get_header (msg, &hdr);
+	  if (status)
+	    return status;
+	  status = mu_message_get_body (msg, &body);
+	  if (status)
+	    return status;
+	  
+	  status = _message_stream_create (&msg->stream, msg, MU_STREAM_RDWR);
+	  if (status)
+	    return status;
+	  msg->flags |= MESSAGE_INTERNAL_STREAM;
+	}
     }
-
+  
   if (!ref)
     {
       *pstream = msg->stream;
@@ -845,6 +855,19 @@ mu_message_get_streamref (mu_message_t msg, mu_stream_t *pstream)
 }
 
 int
+mu_message_set_get_stream (mu_message_t msg,
+			   int (*_getstr) (mu_message_t, mu_stream_t *),
+			   void *owner)
+{
+  if (msg == NULL)
+    return EINVAL;
+  if (msg->owner != owner)
+    return EACCES;
+  msg->_get_stream = _getstr;
+  return 0;
+}
+
+int
 mu_message_set_lines (mu_message_t msg, int (*_lines)
 		   (mu_message_t, size_t *), void *owner)
 {

+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2009, 2010 Free Software Foundation, Inc.
+
+   GNU Mailutils is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GNU Mailutils is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>. */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <string.h>
+#include <stdlib.h>
+#include <mailutils/types.h>
+#include <mailutils/stream.h>
+#include <mailutils/message.h>
+#include <mailutils/attribute.h>
+#include <mailutils/cstr.h>
+
+int
+mu_stream_scan_message (mu_stream_t stream, struct mu_message_scan *sp)
+{
+  char buf[1024];
+  mu_off_t off;
+  size_t n;
+  int status;
+  int in_header = 1;
+  size_t hlines = 0;
+  size_t blines = 0;
+  size_t body_start = 0;
+  int attr_flags = 0;
+  unsigned long uidvalidity = 0;
+  
+  if (sp->flags & MU_SCAN_SEEK)
+    {
+      status = mu_stream_seek (stream, sp->message_start, MU_SEEK_SET, NULL);
+      if (status)
+	return status;
+    }
+  
+  off = 0;
+  while (1)
+    {
+      size_t rdsize;
+      
+      status = mu_stream_readline (stream, buf, sizeof (buf), &n);
+      if (status || n == 0)
+	break;
+      
+      if (sp->flags & MU_SCAN_SIZE)
+	{
+	  rdsize = sp->message_size - off;
+	  if (n > rdsize)
+	    n = rdsize;
+	}
+      
+      if (in_header)
+	{
+	  if (buf[0] == '\n')
+	    {
+	      in_header = 0;
+	      body_start = off + 1;
+	    }
+	  if (buf[n - 1] == '\n')
+	    hlines++;
+	    
+	  /* Process particular attributes */
+	  if (mu_c_strncasecmp (buf, "status:", 7) == 0)
+	    mu_string_to_flags (buf, &attr_flags);
+	  else if (mu_c_strncasecmp (buf, "x-imapbase:", 11) == 0)
+	    {
+	      char *p;
+	      uidvalidity = strtoul (buf + 11, &p, 10);
+	      /* second number is next uid. Ignored */
+	    }
+	}
+      else
+	{
+	  if (buf[n - 1] == '\n')
+	    blines++;
+	}
+      off += n;
+    }
+
+  if (status == 0)
+    {
+      if (!body_start)
+	body_start = off;
+      sp->body_start = body_start;
+      sp->body_end = off;
+      sp->header_lines = hlines;
+      sp->body_lines = blines;
+      sp->attr_flags = attr_flags;
+      sp->uidvalidity = uidvalidity;
+    }
+  return status;
+}

@@ -48,7 +48,7 @@ rdcache_read (struct _mu_stream *str, char *buf, size_t size, size_t *pnbytes)
       status = mu_stream_seek (sp->cache, 0, MU_SEEK_END, NULL);
       if (status)
 	return status;
-      status = mu_stream_copy (sp->cache, sp->transport, left);
+      status = mu_stream_copy (sp->cache, sp->transport, left, NULL);
       if (status)
 	return status;
       sp->size = sp->offset;

@@ -153,9 +153,13 @@ _stream_flush_buffer (struct _mu_stream *stream, int all)
 		  
   if (stream->flags & _MU_STR_DIRTY)
     {
-      if ((stream->flags & MU_STREAM_SEEK)
-	  && (rc = mu_stream_seek (stream, stream->offset, MU_SEEK_SET, NULL)))
-	return rc;
+      if ((stream->flags & MU_STREAM_SEEK) && stream->seek)
+	{
+	  mu_off_t off;
+	  rc = stream->seek (stream, stream->offset, &off);
+	  if (rc)
+	    return rc;
+	}
 
       switch (stream->buftype)
 	{

@@ -31,22 +31,22 @@
 /* Copy SIZE bytes from SRC to DST.  If SIZE is 0, copy everything up to
    EOF. */
 int
-mu_stream_copy (mu_stream_t dst, mu_stream_t src, size_t size)
+mu_stream_copy (mu_stream_t dst, mu_stream_t src, mu_off_t size,
+		mu_off_t *pcsz)
 {
   int status;
   size_t bufsize, n;
   char *buf;
-
+  mu_off_t total = 0;
+  
+  if (pcsz)
+    *pcsz = 0;
   if (size == 0)
     {
-      mu_off_t strsize;
-      status = mu_stream_size (src, &strsize);
+      status = mu_stream_size (src, &size);
       switch (status)
 	{
 	case 0:
-	  size = strsize;
-	  if ((mu_off_t)size != strsize)
-	    return ERANGE;
 	  break;
 
 	case ENOSYS:
@@ -105,6 +105,7 @@ mu_stream_copy (mu_stream_t dst, mu_stream_t src, size_t size)
 	if (status)
 	  break;
 	size -= rdsize;
+	total += rdsize;
       }
   else
     while ((status = mu_stream_read (src, buf, bufsize, &n)) == 0
@@ -113,8 +114,11 @@ mu_stream_copy (mu_stream_t dst, mu_stream_t src, size_t size)
 	status = mu_stream_write (dst, buf, n, NULL);
 	if (status)
 	  break;
+	total += n;
       }
 
+  if (pcsz)
+    *pcsz = total;
   free (buf);
   return status;
 }

@@ -24,13 +24,15 @@
 #include <mailutils/errno.h>
 #include <mailutils/sys/streamref.h>
 
+#define _MU_STR_ERRMASK (_MU_STR_ERR|_MU_STR_EOF)
+
 static int
 streamref_return (struct _mu_streamref *sp, int rc)
 {
   if (rc)
     sp->stream.last_err = sp->transport->last_err;
-  sp->stream.flags = (sp->stream.flags & ~_MU_STR_INTERN_MASK) |
-                   (sp->transport->flags & _MU_STR_INTERN_MASK);
+  sp->stream.flags = (sp->stream.flags & ~_MU_STR_ERRMASK) |
+                   (sp->transport->flags & _MU_STR_ERRMASK);
   return rc;
 }
 
@@ -80,15 +82,15 @@ _streamref_readdelim (struct _mu_stream *str, char *buf, size_t bufsize,
   rc = mu_stream_seek (sp->transport, sp->offset, MU_SEEK_SET, &off);
   if (rc == 0)
     {
-      if (sp->end)
-	{
-	  size_t size = sp->end - off + 2; /* extra 1 to account for \0 */
-	  if (size < bufsize)
-	    bufsize = size;
-	}
       rc = mu_stream_readdelim (sp->transport, buf, bufsize, delim, &nread);
       if (rc == 0)
 	{
+	  if (sp->end)
+	    {
+	      size_t size = sp->end - off + 1;
+	      if (nread > size)
+		nread = size;
+	    }
 	  sp->offset += nread;
 	  *pnread = nread;
 	}

@@ -496,7 +496,7 @@ msg_copy (size_t num, const char *file)
   mu_mailbox_get_message (tmpbox, num, &msg);
   mu_message_get_streamref (msg, &istream);
   /* FIXME: Implement RFC 934 FSA? */
-  rc = mu_stream_copy (ostream, istream, 0);
+  rc = mu_stream_copy (ostream, istream, 0, NULL);
   if (rc)
     {
       mu_error (_("copy stream error: %s"), mu_strerror (rc));

@@ -171,7 +171,7 @@ copy_message (mu_mailbox_t mbox, size_t n, const char *file)
       return rc;
     }
 
-  rc = mu_stream_copy (out, in, 0);
+  rc = mu_stream_copy (out, in, 0, NULL);
   mu_stream_destroy (&in);
   mu_stream_close (out);
   mu_stream_destroy (&out);

@@ -1259,7 +1259,7 @@ show_internal (mu_message_t msg, msg_part_t part, char *encoding, mu_stream_t ou
 		         MU_FILTER_DECODE, MU_STREAM_READ);
   if (rc == 0)
     bstr = dstr;
-  rc = mu_stream_copy (out, bstr, 0);
+  rc = mu_stream_copy (out, bstr, 0, NULL);
   mu_stream_destroy (&bstr);
   return rc;
 }
@@ -2004,7 +2004,7 @@ finish_text_msg (struct compose_env *env, mu_message_t *msg, int ascii)
 			     MU_STREAM_READ);
       if (rc == 0)
 	{
-	  mu_stream_copy (output, fstr, 0);
+	  mu_stream_copy (output, fstr, 0, NULL);
 	  mu_stream_destroy (&fstr);
 	  mu_message_unref (*msg);
 	  *msg = newmsg;
@@ -2064,7 +2064,7 @@ edit_extern (char *cmd, struct compose_env *env, mu_message_t *msg, int level)
   free (id);
 
   mu_header_get_streamref (hdr2, &in);
-  mu_stream_copy (out, in, 0);
+  mu_stream_copy (out, in, 0, NULL);
   mu_stream_destroy (&in);
   mu_stream_close (out);
   mu_stream_destroy (&out);
@@ -2295,7 +2295,7 @@ edit_mime (char *cmd, struct compose_env *env, mu_message_t *msg, int level)
   
   mu_message_get_body (*msg, &body);
   mu_body_get_streamref (body, &out);
-  mu_stream_copy (out, fstr, 0);
+  mu_stream_copy (out, fstr, 0, NULL);
 
   mu_stream_close (out);
   mu_stream_destroy (&out);
@@ -2660,7 +2660,7 @@ mhn_compose ()
   mhn_header (message, msg);
   copy_header_to_stream (message, stream);
   mu_message_get_streamref (msg, &in);
-  mu_stream_copy (stream, in, 0);
+  mu_stream_copy (stream, in, 0, NULL);
   mu_stream_destroy (&in);
   mu_stream_destroy (&stream);
   

@@ -46,7 +46,7 @@ pop3d_retr (char *arg)
     return ERR_UNKNOWN;
   
   pop3d_outf ("+OK\n");
-  mu_stream_copy (iostream, stream, 0);
+  mu_stream_copy (iostream, stream, 0, NULL);
   mu_stream_destroy (&stream);
 
   if (!mu_attribute_is_read (attr))

@@ -62,7 +62,7 @@ pop3d_top (char *arg)
     return ERR_UNKNOWN;
   pop3d_outf ("+OK\n");
 
-  mu_stream_copy (iostream, stream, 0);
+  mu_stream_copy (iostream, stream, 0, NULL);
   pop3d_outf ("\n");
   mu_stream_destroy (&stream);