Reimplement POP3 I/O over the DOT and CRLFDOT filters.

As a side note, previous versions of pop3d did not do
byte-stuffing when sending RETR output.  This commit fixes
it.

* include/mailutils/filter.h (mu_dot_filter): New filter.
* mailbox/dot.c: New file.
* mailbox/Makefile.am (libmailutils_la_SOURCES): Add dot.c
* mailbox/crlfdot.c (_crlfdot_encoder): Bugfix. Do not
dereference iobuf until we know it is not NULL.
* mailbox/filter.c (mu_filter_get_list): Register mu_dot_filter.
* mailbox/fltstream.c (filter_wr_close, filter_rd_close): New
close methods, depending on the filter stream mode.
(mu_filter_stream_create): Set filter_rd_close for MU_STREAM_READ
and filter_wr_close for MU_STREAM_WRITE streams.
* mailbox/stream.c (mu_stream_readdelim): Update stream offset
when using stream->readdelim.

* pop3d/pop3d.h (pop3d_send_payload): New proto.
* pop3d/retr.c (pop3d_send_payload): Change function to be
optionally used in pop3d_top.  Use DOT filter on output.
* pop3d/top.c (pop3d_top): Rewrite using pop3d_send_payload.

* libproto/pop/pop3_retr.c (mu_pop3_retr): Default return
code is EINPROGRESS.
* libproto/pop/pop3_stream.c: Remove the filter stuff.
Use CRLFDOT instead.

* mailbox/amd.c (amd_body_stream_readdelim): Bugfix.  Increase
offset by nread bytes.

@@ -95,6 +95,7 @@ extern int mu_filter_get_list (mu_list_t *);
 extern mu_filter_record_t mu_crlf_filter;  
 extern mu_filter_record_t mu_rfc822_filter;
 extern mu_filter_record_t mu_crlfdot_filter;
+extern mu_filter_record_t mu_dot_filter;
 extern mu_filter_record_t mu_qp_filter; /* quoted-printable.  */
 extern mu_filter_record_t mu_base64_filter;
 extern mu_filter_record_t mu_binary_filter;

@@ -29,7 +29,7 @@
 int
 mu_pop3_retr (mu_pop3_t pop3, unsigned int msgno, mu_stream_t *pstream)
 {
-  int status;
+  int status = EINPROGRESS;
 
   if (pop3 == NULL || msgno == 0)
     return EINVAL;

@@ -41,145 +41,6 @@ struct mu_pop3_stream
   struct mu_buffer_query oldbuf;
 };
 
-enum pop3_decode_state
-  {
-    pds_init,  /* initial state */
-    pds_char,  /* Any character excepting [\r\n.] */
-    pds_cr,    /* prev. char was \r */
-    pds_crlf,  /* 2 prev. char were \r\n */
-    pds_dot,   /* 3 prev. chars were \r\n. */
-    pds_dotcr, /* 4 prev. chars were \r\n.\r */
-    pds_end    /* final state, a \r\n.\r\n seen. */
-  };
-
-static int
-newstate (int state, int c)
-{
-  switch (state)
-    {
-    case pds_init:
-      switch (c)
-	{
-	case '\r':
-	  return pds_cr;
-	case '.':
-	  return pds_dot;
-	}
-      break;
-      
-    case pds_char:
-      switch (c)
-	{
-	case '\r':
-	  return pds_cr;
-	}
-      break;
-      
-    case pds_cr:
-      switch (c)
-	{
-	case '\r':
-	  return pds_cr;
-	case '\n':
-	  return pds_crlf;
-	}
-      break;
-      
-    case pds_crlf:
-      switch (c)
-	{
-	case '\r':
-	  return pds_cr;
-	case '.':
-	  return pds_dot;
-	}
-      
-    case pds_dot:
-      switch (c)
-	{
-	case '\r':
-	  return pds_dotcr;
-	}
-      break;
-
-    case pds_dotcr:
-      switch (c)
-	{
-	case '\n':
-	  return pds_end;
-	}
-    }
-  return pds_char;
-}
-
-/* Move min(isize,osize) bytes from iptr to optr, replacing each \r\n
-   with \n. */
-static enum mu_filter_result
-_pop3_decoder (void *xd,
-	       enum mu_filter_command cmd,
-	       struct mu_filter_io *iobuf)
-{
-  int *pstate = xd;
-  size_t i, j;
-  const unsigned char *iptr;
-  size_t isize;
-  char *optr;
-  size_t osize;
-
-  switch (cmd)
-    {
-    case mu_filter_init:
-      *pstate = pds_init;
-      return mu_filter_ok;
-      
-    case mu_filter_done:
-      return mu_filter_ok;
-      
-    default:
-      break;
-    }
-  
-  iptr = (const unsigned char *) iobuf->input;
-  isize = iobuf->isize;
-  optr = iobuf->output;
-  osize = iobuf->osize;
-
-  for (i = j = 0; *pstate != pds_end && i < isize && j < osize; i++)
-    {
-      unsigned char c = *iptr++;
-
-      if (c == '\r')
-	{
-	  if (i + 1 == isize)
-	    break;
-	  *pstate = newstate (*pstate, c);
-	  if (*iptr == '\n')
-	    continue;
-	}
-      else if (c == '.' && (*pstate == pds_init || *pstate == pds_crlf))
-	{
-	  /* Make sure we have two more characters in the buffer */
-	  if (i + 2 == isize)
-	    break;
-	  *pstate = newstate (*pstate, c);
-	  if (*iptr != '\r')
-	    continue;
-	}
-      else
-	*pstate = newstate (*pstate, c);
-      optr[j++] = c;
-    }
-  
-  if (*pstate == pds_end)
-    {
-      j -= 2; /* remove the trailing .\n */
-      iobuf->eof = 1;
-    }
-  iobuf->isize = i;
-  iobuf->osize = j;
-  return mu_filter_ok;
-}
-
 static void
 _pop3_event_cb (mu_stream_t str, int ev, int flags)
 {
@@ -207,13 +68,9 @@ static int
 mu_pop3_filter_create (mu_stream_t *pstream, mu_stream_t stream)
 {
   int rc;
-  int *state = malloc (sizeof (*state));
-  if (!state)
-    return ENOMEM;
-  rc = mu_filter_stream_create (pstream, stream,
-				MU_FILTER_DECODE,
-				_pop3_decoder, state,
-				MU_STREAM_READ);
+
+  rc = mu_filter_create (pstream, stream, "CRLFDOT", MU_FILTER_DECODE,
+			 MU_STREAM_READ | MU_STREAM_AUTOCLOSE);
   if (rc == 0)
     {
       mu_stream_t str = *pstream;

@@ -74,6 +74,7 @@ libmailutils_la_SOURCES = \
  dbgsyslog.c\
  debug.c\
  diag.c\
+ dot.c\
  envelope.c\
  fgetpwent.c\
  file_stream.c\

@@ -1678,7 +1678,7 @@ amd_body_stream_readdelim (mu_stream_t is, char *buffer, size_t buflen,
 	  size_t rdsize = ((size_t)ln < buflen) ? (size_t)ln : buflen;
 	  status = mu_stream_readdelim (mhm->stream, buffer, rdsize,
 					delim, &nread);
-	  amdstr->off += rdsize;
+	  amdstr->off += nread;
 	}
 
       if (pnread)

@@ -219,11 +219,6 @@ _crlfdot_encoder (void *xd,
   size_t osize;
   int *state = xd;
   
-  iptr = (const unsigned char *) iobuf->input;
-  isize = iobuf->isize;
-  optr = iobuf->output;
-  osize = iobuf->osize;
-
   switch (cmd)
     {
     case mu_filter_init:
@@ -237,6 +232,11 @@ _crlfdot_encoder (void *xd,
       break;
     }
   
+  iptr = (const unsigned char *) iobuf->input;
+  isize = iobuf->isize;
+  optr = iobuf->output;
+  osize = iobuf->osize;
+
   for (i = j = 0; i < isize && j < osize; i++, iptr++)
     {
       unsigned char c = *iptr;

+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2003, 2007, 2010 Free Software Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General
+   Public License along with this library; if not, write to the
+   Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+   Boston, MA 02110-1301 USA */
+
+/* This file implements a DOT filter, useful for data I/O in
+   such protocols as POP3 and SMTP.  When encoding, this filter
+   "byte-stuffs" the input by outputting an additional '.' in front
+   of any '.' appearing at the beginning of a line.  Upon closing the
+   filter in this mode, it outputs additional ".\n".
+   
+   When decoding, the reverse is performed: any '.' appearing at the
+   beginning of a line is removed.  A single dot on a line by itself
+   marks end of the stream.
+
+   MU also provides a hairy version of this filter, called CRLFDOT.
+   In addition to byte-stuffing, CRLFDOT also performs CRLF/LF translation.
+ */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <mailutils/errno.h>
+#include <mailutils/filter.h>
+#include <mailutils/stream.h>
+
+enum dot_decode_state
+  {
+    dot_decode_init,  /* initial state */
+    dot_decode_char,  /* Any character excepting [\r\n.] */
+    dot_decode_lf,    /* prev. char was \n */
+    dot_decode_dot,   /* 2 prev. chars were \n. */
+    dot_decode_end   /* final state, a \n.\n seen. */
+  };
+
+static enum dot_decode_state
+new_decode_state (enum dot_decode_state state, int c)
+{
+  switch (state)
+    {
+    case dot_decode_init:
+      switch (c)
+	{
+	case '.':
+	  return dot_decode_dot;
+	}
+      break;
+      
+    case dot_decode_char:
+      switch (c)
+	{
+	case '\n':
+	  return dot_decode_lf;
+	}
+      break;
+      
+    case dot_decode_lf:
+      switch (c)
+	{
+	case '.':
+	  return dot_decode_dot;
+	}
+      
+    case dot_decode_dot:
+      switch (c)
+	{
+	case '\n':
+	  return dot_decode_end;
+	}
+      break;
+
+    case dot_decode_end:
+      return dot_decode_end;
+    }
+  return dot_decode_char;
+}
+
+/* Move min(isize,osize) bytes from iptr to optr, replacing each \r\n
+   with \n. */
+static enum mu_filter_result
+_dot_decoder (void *xd,
+	      enum mu_filter_command cmd,
+	      struct mu_filter_io *iobuf)
+{
+  int *pstate = xd;
+  size_t i, j;
+  const unsigned char *iptr;
+  size_t isize;
+  char *optr;
+  size_t osize;
+
+  switch (cmd)
+    {
+    case mu_filter_init:
+      *pstate = dot_decode_init;
+      return mu_filter_ok;
+      
+    case mu_filter_done:
+      return mu_filter_ok;
+      
+    default:
+      break;
+    }
+  
+  iptr = (const unsigned char *) iobuf->input;
+  isize = iobuf->isize;
+  optr = iobuf->output;
+  osize = iobuf->osize;
+
+  for (i = j = 0; *pstate != dot_decode_end && i < isize && j < osize; i++)
+    {
+      unsigned char c = *iptr++;
+      int curstate = *pstate;
+      
+      *pstate = new_decode_state (curstate, c);
+      if (!(c == '.' && (curstate == dot_decode_init ||
+			 curstate == dot_decode_lf)))
+	optr[j++] = c;
+    }
+  
+  if (*pstate == dot_decode_end)
+    iobuf->eof = 1;
+
+  iobuf->isize = i;
+  iobuf->osize = j;
+  return mu_filter_ok;
+}
+
+enum dot_encode_state
+  {
+    dot_encode_init,  /* initial state */
+    dot_encode_char,  /* Any character excepting \n */
+    dot_encode_lf,    /* prev. char was \n */
+  };    
+
+static enum dot_encode_state
+new_encode_state (int c)
+{
+  switch (c)
+    {
+    case '\n':
+      return dot_encode_lf;
+    }
+  return dot_encode_char;
+}
+
+/* Move min(isize,osize) bytes from iptr to optr, replacing each \n
+   with \r\n.  Any input \r\n sequences remain untouched. */
+static enum mu_filter_result
+_dot_encoder (void *xd,
+	      enum mu_filter_command cmd,
+	      struct mu_filter_io *iobuf)
+{
+  enum mu_filter_result result;
+  size_t i, j;
+  const unsigned char *iptr;
+  size_t isize;
+  char *optr;
+  size_t osize;
+  int *state = xd;
+  
+  switch (cmd)
+    {
+    case mu_filter_init:
+      *state = dot_encode_init;
+      return mu_filter_ok;
+      
+    case mu_filter_done:
+      return mu_filter_ok;
+
+    default:
+      break;
+    }
+  
+  iptr = (const unsigned char *) iobuf->input;
+  isize = iobuf->isize;
+  optr = iobuf->output;
+  osize = iobuf->osize;
+
+  for (i = j = 0; i < isize && j < osize; i++, iptr++)
+    {
+      unsigned char c = *iptr;
+      int curstate = *state;
+
+      if (c == '.' && (curstate == dot_encode_init ||
+		       curstate == dot_encode_lf))
+	{
+ 	  if (j + 2 > osize)
+	    {
+	      if (i == 0)
+		{
+		  iobuf->osize = 2;
+		  return mu_filter_moreoutput;
+		}
+	      break;
+	    }
+	  optr[j++] = '.';
+	  optr[j++] = '.';
+	}
+      else
+	optr[j++] = c;
+
+      *state = new_encode_state (c);
+    }
+
+  result = mu_filter_ok;
+  if (cmd == mu_filter_lastbuf)
+    {
+      switch (*state)
+	{
+	case dot_encode_lf:
+	  if (j + 2 > osize)
+	    result = mu_filter_again;
+	  break;
+	      
+	default:
+	  if (j + 3 > osize)
+	    result = mu_filter_again;
+	  else
+	    optr[j++] = '\n';
+	}
+
+      if (result == mu_filter_ok)
+	{
+	  optr[j++] = '.';
+	  optr[j++] = '\n';
+	}
+    }
+
+  iobuf->isize = i;
+  iobuf->osize = j;
+  return result;
+}
+
+static int
+alloc_state (void **pret, int mode MU_ARG_UNUSED, void *data MU_ARG_UNUSED)
+{
+  *pret = malloc (sizeof (int));
+  if (!*pret)
+    return ENOMEM;
+  return 0;
+}
+
+static struct _mu_filter_record _dot_filter = {
+  "DOT",
+  0,
+  alloc_state,
+  _dot_encoder,
+  _dot_decoder
+};
+
+mu_filter_record_t mu_dot_filter = &_dot_filter;

@@ -75,6 +75,7 @@ mu_filter_get_list (mu_list_t *plist)
       mu_list_append (filter_list, mu_rfc822_filter);
       mu_list_append (filter_list, mu_crlf_filter);
       mu_list_append (filter_list, mu_crlfdot_filter);
+      mu_list_append (filter_list, mu_dot_filter);
       mu_list_append (filter_list, mu_rfc_2047_Q_filter);
       mu_list_append (filter_list, mu_rfc_2047_B_filter);
       /* FIXME: add the default encodings?  */

@@ -401,11 +401,26 @@ filter_done (mu_stream_t stream)
 }
 
 static int
-filter_close (mu_stream_t stream)
+filter_wr_close (mu_stream_t stream)
+{
+  struct _mu_filter_stream *fs = (struct _mu_filter_stream *)stream;
+  if (!mu_stream_eof (stream))
+    {
+      size_t dummy;
+      int rc = filter_write_internal (stream, mu_filter_lastbuf, NULL, 0,
+				      &dummy);
+      if (rc)
+	return rc;
+    }
+  MBF_CLEAR (fs->inbuf);
+  MBF_CLEAR (fs->outbuf);
+  return mu_stream_close (fs->transport);
+}
+
+static int
+filter_rd_close (mu_stream_t stream)
 {
   struct _mu_filter_stream *fs = (struct _mu_filter_stream *)stream;
-  size_t dummy;
-  int rc = filter_write_internal (stream, mu_filter_lastbuf, NULL, 0, &dummy);
   MBF_CLEAR (fs->inbuf);
   MBF_CLEAR (fs->outbuf);
   return mu_stream_close (fs->transport);
@@ -469,6 +484,7 @@ mu_filter_stream_create (mu_stream_t *pflt,
     {
       fs->stream.read = filter_read;
       fs->stream.flush = filter_rd_flush;
+      fs->stream.close = filter_rd_close;
       if (flags & MU_STREAM_WRTHRU)
 	{
 	  flags |= MU_STREAM_WRITE;
@@ -479,6 +495,7 @@ mu_filter_stream_create (mu_stream_t *pflt,
     {
       fs->stream.write = filter_write;
       fs->stream.flush = filter_wr_flush;
+      fs->stream.close = filter_wr_close;
       if (flags & MU_STREAM_RDTHRU)
 	{
 	  flags |= MU_STREAM_READ;
@@ -486,7 +503,6 @@ mu_filter_stream_create (mu_stream_t *pflt,
 	}
     }
   fs->stream.done = filter_done;
-  fs->stream.close = filter_close;
   if (flags & MU_STREAM_SEEK)
     fs->stream.seek = filter_seek;
   fs->stream.ctl = filter_ctl;

@@ -737,7 +737,13 @@ mu_stream_readdelim (mu_stream_t stream, char *buf, size_t size,
     return EINVAL;
     
   if (stream->readdelim)
-    rc = stream->readdelim (stream, buf, size, delim, pread);
+    {
+      size_t nread;
+      rc = stream->readdelim (stream, buf, size, delim, &nread);
+      if (pread)
+	*pread = nread;
+      stream->offset += nread;
+    }
   else if (stream->buftype != mu_buffer_none)
     rc = _stream_scandelim (stream, buf, size, delim, pread);
   else

@@ -219,6 +219,9 @@ extern int pop3d_quit           (char *);
 extern int pop3d_retr           (char *);
 extern int pop3d_rset           (char *);
 
+void pop3d_send_payload (mu_stream_t stream, mu_stream_t linestr,
+			 size_t maxlines);
+
 extern void pop3d_bye           (void);
 extern int pop3d_abquit         (int);
 extern char *pop3d_apopuser     (const char *);

@@ -20,8 +20,9 @@
 size_t pop3d_output_bufsize = 64 * 1024;
 
 void
-pop3d_send_payload (mu_stream_t stream)
+pop3d_send_payload (mu_stream_t stream, mu_stream_t linestr, size_t maxlines)
 {
+  mu_stream_t flt;
   struct mu_buffer_query oldbuf, newbuf;
   int xscript_level = set_xscript_level (MU_XSCRIPT_PAYLOAD);
 
@@ -33,10 +34,28 @@ pop3d_send_payload (mu_stream_t stream)
   newbuf.bufsize = pop3d_output_bufsize;
   mu_stream_ioctl (iostream, MU_IOCTL_SET_TRANSPORT_BUFFER,
 		   &newbuf);
-
-  mu_stream_copy (iostream, stream, 0, NULL);
-  pop3d_outf (".\n");
-
+  /* FIXME: Return code */
+  mu_filter_create (&flt, iostream, "DOT", MU_FILTER_ENCODE,
+		    MU_STREAM_WRITE);
+  
+  mu_stream_copy (flt, stream, 0, NULL);
+
+  if (maxlines)
+    {
+      char *buf = NULL;
+      size_t size = 0, n;
+
+      mu_stream_write (flt, "\n", 1, NULL);
+      for (; maxlines > 0 &&
+	     mu_stream_getline (linestr, &buf, &size, &n) == 0 &&
+	     n > 0; maxlines--)
+	mu_stream_write (flt, buf, n, NULL);
+      free (buf);
+    }
+  
+  mu_stream_close (flt);
+  mu_stream_destroy (&flt);
+  
   mu_stream_ioctl (iostream, MU_IOCTL_SET_TRANSPORT_BUFFER,
 		   &oldbuf);
   set_xscript_level (xscript_level);
@@ -71,7 +90,7 @@ pop3d_retr (char *arg)
     return ERR_UNKNOWN;
   
   pop3d_outf ("+OK\n");
-  pop3d_send_payload (stream);
+  pop3d_send_payload (stream, NULL, 0);
   mu_stream_destroy (&stream);
   
   if (!mu_attribute_is_read (attr))

@@ -28,10 +28,8 @@ pop3d_top (char *arg)
   mu_attribute_t attr;
   mu_header_t hdr;
   mu_body_t body;
-  mu_stream_t stream;
+  mu_stream_t hstream, bstream;
   char *mesgc, *linesc, *p;
-  int xscript_level;
-  struct mu_buffer_query oldbuf, newbuf;
   
   if (strlen (arg) == 0)
     return ERR_BAD_ARGS;
@@ -60,47 +58,21 @@ pop3d_top (char *arg)
   
   /* Header.  */
   mu_message_get_header (msg, &hdr);
-  if (mu_header_get_streamref (hdr, &stream))
+  if (mu_header_get_streamref (hdr, &hstream))
     return ERR_UNKNOWN;
-  pop3d_outf ("+OK\n");
-
-  xscript_level = set_xscript_level (MU_XSCRIPT_PAYLOAD);
-  oldbuf.type = MU_TRANSPORT_OUTPUT;
-  mu_stream_ioctl (iostream, MU_IOCTL_GET_TRANSPORT_BUFFER,
-		   &oldbuf);
-  newbuf.type = MU_TRANSPORT_OUTPUT;
-  newbuf.buftype = mu_buffer_full;
-  newbuf.bufsize = pop3d_output_bufsize;
-  mu_stream_ioctl (iostream, MU_IOCTL_SET_TRANSPORT_BUFFER,
-		   &newbuf);  
-
-  mu_stream_copy (iostream, stream, 0, NULL);
-  pop3d_outf ("\n");
-  mu_stream_destroy (&stream);
-  
   mu_message_get_body (msg, &body);
-  if (mu_body_get_streamref (body, &stream) == 0)
+  if (mu_body_get_streamref (body, &bstream))
     {
-      char *buf = NULL;
-      size_t size = 0, n;
-      for (; lines > 0 &&
-	     mu_stream_getline (stream, &buf, &size, &n) == 0 &&
-	     n > 0; lines--)
-	{
-	  if (buf[0] == '.')
-	    pop3d_outf (".");
-	  pop3d_outf ("%s", buf);
-	}
-      mu_stream_destroy (&stream);
-      free (buf);
+      mu_stream_unref (hstream);
+      return ERR_UNKNOWN;
     }
+    
+  pop3d_outf ("+OK\n");
 
-  pop3d_outf (".\n");
+  pop3d_send_payload (hstream, bstream, lines);
 
-  mu_stream_ioctl (iostream, MU_IOCTL_SET_TRANSPORT_BUFFER,
-		   &oldbuf);
-  
-  set_xscript_level (xscript_level);
+  mu_stream_unref (hstream);
+  mu_stream_unref (bstream);
 
   return OK;
 }