Improve filter framework. Implement a CRLFDOT filter.

* include/mailutils/filter.h (mu_filter_command) <mu_filter_flush> New command.
(mu_filter_result) <mu_filter_again>: New result code.
(MU_FILTER_MAX_AGAIN): New constant.
(mu_crlfdot_filter): New filter.
* include/mailutils/sys/stream.h (_MU_STR_DIRTY, _MU_STR_WRT)
(_MU_STR_ERR, _MU_STR_EOF, _MU_STR_INTERN_MASK): Avoid conflicts with
MU_STREAM_ constants from mailutils/stream.h

* mailbox/crlfdot.c: New source.
* mailbox/Makefile.am (libmailutils_la_SOURCES): Add crlfdot.c.
* mailbox/filter.c (mu_filter_get_list): Register mu_crlfdot_filter.
* mailbox/fltstream.c (init_iobuf): Fix initialization of isize and
input.
(filter_read): Redo the loop, allowing xcode to request a new
iteration by returning mu_filter_again.
(filter_write_internal): Likewise.
(filter_wr_flush): Call filter_write_internal with mu_filter_flush, reserving
mu_filter_lastbuf for really last buffers.
(filter_close): Call filter_write_internal with mu_filter_lastbuf.
* mailbox/stream.c (mu_stream_skip_input_bytes): Clear MU_STREAM_SEEK bit
before actually doing anything.

* include/mailutils/sys/pop3.h [DMALLOC]: Remove unneeded cond.

@@ -50,7 +50,8 @@ enum mu_filter_command
     mu_filter_init,
     mu_filter_done,
     mu_filter_xcode,
-    mu_filter_lastbuf
+    mu_filter_lastbuf,
+    mu_filter_flush
   };
   
 enum mu_filter_result
@@ -58,9 +59,12 @@ enum mu_filter_result
     mu_filter_ok,
     mu_filter_falure,
     mu_filter_moreinput,
-    mu_filter_moreoutput
+    mu_filter_moreoutput,
+    mu_filter_again
   };
-  
+
+#define MU_FILTER_MAX_AGAIN 5
+
 typedef int (*mu_filter_new_data_t) (void **, int, void *);
 typedef enum mu_filter_result (*mu_filter_xcode_t) (void *data,
 						    enum mu_filter_command cmd,
@@ -88,8 +92,9 @@ extern int mu_filter_create (mu_stream_t *, mu_stream_t, const char*,
 extern int mu_filter_get_list (mu_list_t *);
 
 /* List of defaults.  */
-extern mu_filter_record_t mu_crlf_filter;
+extern mu_filter_record_t mu_crlf_filter;  
 extern mu_filter_record_t mu_rfc822_filter;
+extern mu_filter_record_t mu_crlfdot_filter;
 extern mu_filter_record_t mu_qp_filter; /* quoted-printable.  */
 extern mu_filter_record_t mu_base64_filter;
 extern mu_filter_record_t mu_binary_filter;

@@ -25,10 +25,6 @@
 #include <mailutils/errno.h>
 #include <mailutils/cstr.h>
 
-#ifdef DMALLOC
-# include <dmalloc.h>
-#endif
-
 #ifdef __cplusplus
 extern "C" {
 #endif

@@ -17,12 +17,12 @@
 #ifndef _MAILUTILS_SYS_STREAM_H
 #define _MAILUTILS_SYS_STREAM_H
 
-#define _MU_STR_DIRTY         0x1000    /* Buffer dirty */
-#define _MU_STR_WRT           0x2000    /* Unflushed write pending */
-#define _MU_STR_ERR           0x4000    /* Permanent error state */
-#define _MU_STR_EOF           0x8000    /* EOF encountered */
+#define _MU_STR_DIRTY         0x10000000    /* Buffer dirty */
+#define _MU_STR_WRT           0x20000000    /* Unflushed write pending */
+#define _MU_STR_ERR           0x40000000    /* Permanent error state */
+#define _MU_STR_EOF           0x80000000    /* EOF encountered */
 
-#define _MU_STR_INTERN_MASK   0xf000
+#define _MU_STR_INTERN_MASK   0xf0000000
 
 #define _MU_STR_EVENT_SET     1
 #define _MU_STR_EVENT_CLR     2

@@ -63,6 +63,7 @@ libmailutils_la_SOURCES = \
  cfg_format.c\
  cfg_lexer.c\
  cfg_parser.c\
+ crlfdot.c\
  crlfflt.c\
  cstrlower.c\
  cstrupper.c\

+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2003, 2007, 2010 Free Software Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General
+   Public License along with this library; if not, write to the
+   Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+   Boston, MA 02110-1301 USA */
+
+/* This source implements a CRLFDOT filter, useful for data I/O in
+   such protocols as POP3 and SMTP.  When encoding, this filter
+   replaces each '\n' not following '\r' by "\r\n" and "byte-stuffs"
+   the input by outputting an additional '.' in front of any '.' appearing
+   at the beginning of a line.  Upon closing the filter in this mode, it
+   outputs additional ".\r\n".
+   
+   When decoding, the reverse is performed: each "\r\n" is replaced by a
+   '\n', and additional '.' are removed from beginning of lines.  A single
+   dot on a line by itself marks end of the stream.
+ */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <mailutils/errno.h>
+#include <mailutils/filter.h>
+#include <mailutils/stream.h>
+
+enum crlfdot_decode_state
+  {
+    crlfdot_decode_init,  /* initial state */
+    crlfdot_decode_char,  /* Any character excepting [\r\n.] */
+    crlfdot_decode_cr,    /* prev. char was \r */
+    crlfdot_decode_crlf,  /* 2 prev. char were \r\n */
+    crlfdot_decode_dot,   /* 3 prev. chars were \r\n. */
+    crlfdot_decode_dotcr, /* 4 prev. chars were \r\n.\r */
+    crlfdot_decode_end    /* final state, a \r\n.\r\n seen. */
+  };
+
+static enum crlfdot_decode_state
+new_decode_state (enum crlfdot_decode_state state, int c)
+{
+  switch (state)
+    {
+    case crlfdot_decode_init:
+      switch (c)
+	{
+	case '\r':
+	  return crlfdot_decode_cr;
+	case '.':
+	  return crlfdot_decode_dot;
+	}
+      break;
+      
+    case crlfdot_decode_char:
+      switch (c)
+	{
+	case '\r':
+	  return crlfdot_decode_cr;
+	}
+      break;
+      
+    case crlfdot_decode_cr:
+      switch (c)
+	{
+	case '\r':
+	  return crlfdot_decode_cr;
+	case '\n':
+	  return crlfdot_decode_crlf;
+	}
+      break;
+      
+    case crlfdot_decode_crlf:
+      switch (c)
+	{
+	case '\r':
+	  return crlfdot_decode_cr;
+	case '.':
+	  return crlfdot_decode_dot;
+	}
+      
+    case crlfdot_decode_dot:
+      switch (c)
+	{
+	case '\r':
+	  return crlfdot_decode_dotcr;
+	}
+      break;
+
+    case crlfdot_decode_dotcr:
+      switch (c)
+	{
+	case '\n':
+	  return crlfdot_decode_end;
+	}
+
+    case crlfdot_decode_end:
+      break;
+    }
+  return crlfdot_decode_char;
+}
+
+/* Move min(isize,osize) bytes from iptr to optr, replacing each \r\n
+   with \n. */
+static enum mu_filter_result
+_crlfdot_decoder (void *xd,
+		  enum mu_filter_command cmd,
+		  struct mu_filter_io *iobuf)
+{
+  int *pstate = xd;
+  size_t i, j;
+  const unsigned char *iptr;
+  size_t isize;
+  char *optr;
+  size_t osize;
+
+  switch (cmd)
+    {
+    case mu_filter_init:
+      *pstate = crlfdot_decode_init;
+      return mu_filter_ok;
+      
+    case mu_filter_done:
+      return mu_filter_ok;
+      
+    default:
+      break;
+    }
+  
+  iptr = (const unsigned char *) iobuf->input;
+  isize = iobuf->isize;
+  optr = iobuf->output;
+  osize = iobuf->osize;
+
+  for (i = j = 0; *pstate != crlfdot_decode_end && i < isize && j < osize; i++)
+    {
+      unsigned char c = *iptr++;
+
+      if (c == '\r')
+	{
+	  if (i + 1 == isize)
+	    break;
+	  *pstate = new_decode_state (*pstate, c);
+	  if (*iptr == '\n')
+	    continue;
+	}
+      else if (c == '.' &&
+	       (*pstate == crlfdot_decode_init ||
+		*pstate == crlfdot_decode_crlf))
+	{
+	  /* Make sure we have two more characters in the buffer */
+	  if (i + 2 == isize)
+	    break;
+	  *pstate = new_decode_state (*pstate, c);
+	  if (*iptr != '\r')
+	    continue;
+	}
+      else
+	*pstate = new_decode_state (*pstate, c);
+      optr[j++] = c;
+    }
+  
+  if (*pstate == crlfdot_decode_end)
+    {
+      j -= 2; /* remove the trailing .\n */
+      iobuf->eof = 1;
+    }
+  iobuf->isize = i;
+  iobuf->osize = j;
+  return mu_filter_ok;
+}
+
+enum crlfdot_encode_state
+  {
+    crlfdot_encode_init,  /* initial state */
+    crlfdot_encode_char,  /* Any character excepting [\r\n] */
+    crlfdot_encode_cr,    /* prev. char was \r */
+    crlfdot_encode_lf,    /* prev. char was \n */
+  };    
+
+static enum crlfdot_encode_state
+new_encode_state (enum crlfdot_encode_state state, int c)
+{
+  switch (c)
+    {
+    case '\r':
+      return crlfdot_encode_cr;
+      
+    case '\n':
+      return crlfdot_encode_lf;
+    }
+  return crlfdot_encode_char;
+}
+
+/* Move min(isize,osize) bytes from iptr to optr, replacing each \n
+   with \r\n.  Any input \r\n sequences remain untouched. */
+static enum mu_filter_result
+_crlfdot_encoder (void *xd,
+		  enum mu_filter_command cmd,
+		  struct mu_filter_io *iobuf)
+{
+  enum mu_filter_result result;
+  size_t i, j;
+  const unsigned char *iptr;
+  size_t isize;
+  char *optr;
+  size_t osize;
+  int *state = xd;
+  
+  iptr = (const unsigned char *) iobuf->input;
+  isize = iobuf->isize;
+  optr = iobuf->output;
+  osize = iobuf->osize;
+
+  switch (cmd)
+    {
+    case mu_filter_init:
+      *state = crlfdot_encode_init;
+      return mu_filter_ok;
+      
+    case mu_filter_done:
+      return mu_filter_ok;
+
+    default:
+      break;
+    }
+  
+  for (i = j = 0; i < isize && j < osize; i++, iptr++)
+    {
+      unsigned char c = *iptr;
+      int curstate = *state;
+
+      if (c == '\n')
+	{
+	  if (curstate == crlfdot_encode_cr)
+	    optr[j++] = c;
+ 	  else if (j + 1 == osize)
+	    {
+	      if (i == 0)
+		{
+		  iobuf->osize = 2;
+		  return mu_filter_moreoutput;
+		}
+	      break;
+	    }
+	  else
+	    {
+	      optr[j++] = '\r';
+	      optr[j++] = '\n';
+	    }
+	}
+      else if (c == '.' &&
+	       (curstate == crlfdot_encode_init ||
+		curstate == crlfdot_encode_lf))
+	{
+ 	  if (j + 2 > osize)
+	    {
+	      if (i == 0)
+		{
+		  iobuf->osize = 2;
+		  return mu_filter_moreoutput;
+		}
+	      break;
+	    }
+	  optr[j++] = '.';
+	  optr[j++] = '.';
+	}
+      else
+	optr[j++] = c;
+
+      *state = new_encode_state (curstate, c);
+    }
+
+  result = mu_filter_ok;
+  if (cmd == mu_filter_lastbuf)
+    {
+      switch (*state)
+	{
+	case crlfdot_encode_lf:
+	  if (j + 3 > osize)
+	    result = mu_filter_again;
+	  break;
+	      
+	default:
+	  if (j + 5 > osize)
+	    result = mu_filter_again;
+	  else
+	    {
+	      optr[j++] = '\r';
+	      optr[j++] = '\n';
+	    }
+	}
+
+      if (result == mu_filter_ok)
+	{
+	  optr[j++] = '.';
+	  optr[j++] = '\r';
+	  optr[j++] = '\n';
+	}
+    }
+
+  iobuf->isize = i;
+  iobuf->osize = j;
+  return result;
+}
+
+static int
+alloc_state (void **pret, int mode MU_ARG_UNUSED, void *data MU_ARG_UNUSED)
+{
+  *pret = malloc (sizeof (int));
+  if (!*pret)
+    return ENOMEM;
+  return 0;
+}
+
+static struct _mu_filter_record _crlfdot_filter = {
+  "CRLFDOT",
+  0,
+  alloc_state,
+  _crlfdot_encoder,
+  _crlfdot_decoder
+};
+
+mu_filter_record_t mu_crlfdot_filter = &_crlfdot_filter;

@@ -74,6 +74,7 @@ mu_filter_get_list (mu_list_t *plist)
       mu_list_append (filter_list, mu_bit7_filter);
       mu_list_append (filter_list, mu_rfc822_filter);
       mu_list_append (filter_list, mu_crlf_filter);
+      mu_list_append (filter_list, mu_crlfdot_filter);
       mu_list_append (filter_list, mu_rfc_2047_Q_filter);
       mu_list_append (filter_list, mu_rfc_2047_B_filter);
       /* FIXME: add the default encodings?  */

@@ -45,8 +45,8 @@
 static void
 init_iobuf (struct mu_filter_io *io, struct _mu_filter_stream *fs)
 {
-  io->input = MFB_BASE (fs->inbuf);
-  io->isize = MFB_LEVEL (fs->inbuf);
+  io->input = MFB_CURPTR (fs->inbuf);
+  io->isize = MFB_RDBYTES (fs->inbuf);
   io->output = MFB_ENDPTR (fs->outbuf);
   io->osize = MFB_FREESIZE (fs->outbuf);
   io->errcode = 0;
@@ -111,14 +111,15 @@ static int
 filter_read (mu_stream_t stream, char *buf, size_t size, size_t *pret)
 {
   struct _mu_filter_stream *fs = (struct _mu_filter_stream *)stream;
-  enum mu_filter_command cmd = mu_filter_xcode;
   struct mu_filter_io iobuf;
   size_t min_input_level = MU_FILTER_BUF_SIZE;
   size_t min_output_size = MU_FILTER_BUF_SIZE;
+  enum mu_filter_command cmd = mu_filter_xcode;
   size_t total = 0;
   int stop = 0;
+  int again = 0;
   
-  while (!stop && total < size && cmd != mu_filter_lastbuf)
+  do
     {
       size_t rdsize;
 
@@ -127,7 +128,7 @@ filter_read (mu_stream_t stream, char *buf, size_t size, size_t *pret)
 	  enum mu_filter_result res;
 	  int rc;
 	  
-	  if (MFB_RDBYTES (fs->inbuf) < min_input_level)
+	  if (MFB_RDBYTES (fs->inbuf) < min_input_level && !again)
 	    {
 	      rc = MFB_require (&fs->inbuf, min_input_level);
 	      if (rc)
@@ -139,9 +140,10 @@ filter_read (mu_stream_t stream, char *buf, size_t size, size_t *pret)
 	      if (rc)
 		return rc;
 	      if (rdsize == 0 &&
-		  MFB_RDBYTES (fs->outbuf) == 0
-		  && MFB_RDBYTES (fs->inbuf) == 0)
-		break;
+		  MFB_RDBYTES (fs->outbuf) == 0 &&
+		  MFB_RDBYTES (fs->inbuf) == 0)
+		cmd = mu_filter_lastbuf;
+	      
 	      MFB_advance_level (&fs->inbuf, rdsize);
 	    }
 
@@ -153,16 +155,24 @@ filter_read (mu_stream_t stream, char *buf, size_t size, size_t *pret)
       
 	  init_iobuf (&iobuf, fs);
 
-	  cmd = mu_stream_eof (fs->transport) ?
-	    mu_filter_lastbuf : mu_filter_xcode;
+	  if (cmd != mu_filter_lastbuf)
+	    cmd = mu_stream_eof (fs->transport) ?
+	      mu_filter_lastbuf : mu_filter_xcode;
 	  res = fs->xcode (fs->xdata, cmd, &iobuf);
 	  switch (res)
 	    {
+	    case mu_filter_again:
+	      if (++again > MU_FILTER_MAX_AGAIN)
+		{
+		  /* FIXME: What filter? Need some id. */
+		  mu_error (_("filter returned `again' too many times"));
+		  again = 0;
+		}
+	      break;
+	      
 	    case mu_filter_ok:
-	      if (iobuf.isize > MFB_RDBYTES (fs->inbuf)
-		  || iobuf.osize > MFB_FREESIZE (fs->outbuf))
-		return MU_ERR_FAILURE; /* FIXME: special error code? */
-	      if (iobuf.eof)
+	      again = 0;
+	      if (cmd == mu_filter_lastbuf || iobuf.eof)
 		{
 		  _mu_stream_seteof (stream);
 		  stop = 1;
@@ -181,6 +191,10 @@ filter_read (mu_stream_t stream, char *buf, size_t size, size_t *pret)
 	      continue;
 	    }
       
+	  if (iobuf.isize > MFB_RDBYTES (fs->inbuf)
+	      || iobuf.osize > MFB_FREESIZE (fs->outbuf))
+	    return MU_ERR_FAILURE; /* FIXME: special error code? */
+	  
 	  /* iobuf.osize contains number of bytes written to output */
 	  MFB_advance_level (&fs->outbuf, iobuf.osize);
 	  
@@ -194,8 +208,10 @@ filter_read (mu_stream_t stream, char *buf, size_t size, size_t *pret)
       memcpy (buf + total, MFB_CURPTR (fs->outbuf), rdsize);
       MFB_advance_pos (&fs->outbuf, rdsize);
       total += rdsize;
-    }
 
+    }
+  while (!stop && (total < size || again));
+  
   *pret = total;
   return 0;
 }
@@ -217,6 +233,8 @@ filter_write_internal (mu_stream_t stream, enum mu_filter_command cmd,
   size_t min_output_size = MU_FILTER_BUF_SIZE;
   size_t total = 0;
   int rc = 0;
+  int again;
+  int stop = 0;
   
   do
     {
@@ -249,19 +267,22 @@ filter_write_internal (mu_stream_t stream, enum mu_filter_command cmd,
       res = fs->xcode (fs->xdata, cmd, &iobuf);
       switch (res)
 	{
+	case mu_filter_again:
+	  if (++again > MU_FILTER_MAX_AGAIN)
+	    {
+	      /* FIXME: What filter? Need some id. */
+	      mu_error (_("filter returned `again' too many times"));
+	      again = 0;
+	    }
+	  break;
+
 	case mu_filter_ok:
-	  if (iobuf.isize == 0 || iobuf.osize == 0)
+	  again = 0;
+	  if (cmd == mu_filter_lastbuf || iobuf.eof)
 	    {
-	      /* FIXME: Hack to handle eventual buggy filters */
-	      if (iobuf.isize == 0)
-		min_input_level++;
-	      if (iobuf.osize == 0)
-		min_output_size++;
-	      continue;
+	      _mu_stream_seteof (stream);
+	      stop = 1;
 	    }
-	  if (iobuf.isize > MFB_RDBYTES (fs->inbuf)
-	      || iobuf.osize > MFB_FREESIZE (fs->outbuf))
-	    return MU_ERR_FAILURE; /* FIXME: special error code? */
 	  break;
 	  
 	case mu_filter_falure:
@@ -276,24 +297,26 @@ filter_write_internal (mu_stream_t stream, enum mu_filter_command cmd,
 	  continue;
 	}
       
+      if (iobuf.isize > MFB_RDBYTES (fs->inbuf)
+	  || iobuf.osize > MFB_FREESIZE (fs->outbuf))
+	return MU_ERR_FAILURE; /* FIXME: special error code? */
+      
       /* iobuf.osize contains number of bytes written to output */
       MFB_advance_level (&fs->outbuf, iobuf.osize);
       
       /* iobuf.isize contains number of bytes read from input */
       MFB_advance_pos (&fs->inbuf, iobuf.isize);
       
-      rdsize = size - total;
-      if (rdsize > MFB_RDBYTES (fs->outbuf))
-	rdsize = MFB_RDBYTES (fs->outbuf);
-
       rc = mu_stream_write (fs->transport,
-			    MFB_CURPTR (fs->outbuf), MFB_RDBYTES (fs->outbuf),
+			    MFB_CURPTR (fs->outbuf),
+			    MFB_RDBYTES (fs->outbuf),
 			    &rdsize);
-      MFB_advance_pos (&fs->outbuf, rdsize);
-      if (rc)
+      if (rc == 0)
+	MFB_advance_pos (&fs->outbuf, rdsize);
+      else
 	break;
     }
-  while (MFB_RDBYTES (fs->outbuf));
+  while (!stop && (MFB_RDBYTES (fs->outbuf) || again));
   if (pret)
     *pret = total;
   else if (total < size && rc == 0)
@@ -311,7 +334,8 @@ static int
 filter_wr_flush (mu_stream_t stream)
 {
   struct _mu_filter_stream *fs = (struct _mu_filter_stream *)stream;
-  int rc = filter_write_internal (stream, mu_filter_lastbuf, NULL, 0, NULL);
+  size_t dummy;
+  int rc = filter_write_internal (stream, mu_filter_flush, NULL, 0, &dummy);
   if (rc == 0)
     rc = mu_stream_flush (fs->transport);
   return rc;
@@ -380,6 +404,8 @@ static int
 filter_close (mu_stream_t stream)
 {
   struct _mu_filter_stream *fs = (struct _mu_filter_stream *)stream;
+  size_t dummy;
+  int rc = filter_write_internal (stream, mu_filter_lastbuf, NULL, 0, &dummy);
   MBF_CLEAR (fs->inbuf);
   MBF_CLEAR (fs->outbuf);
   return mu_stream_close (fs->transport);

@@ -380,10 +380,10 @@ mu_stream_seek (mu_stream_t stream, mu_off_t offset, int whence,
    input operations.
    
    This function is designed to help implement seek method in otherwise
-   unseekable streams (such as filters).  Do not use it if you absolutely
+   unseekable streams (such as filters).  Do not use it unless you absolutely
    have to.  Using it on an unbuffered stream is a terrible waste of CPU. */
-int
-mu_stream_skip_input_bytes (mu_stream_t stream, mu_off_t count, mu_off_t *pres)
+static int
+_stream_skip_input_bytes (mu_stream_t stream, mu_off_t count, mu_off_t *pres)
 {
   mu_off_t pos;
   int rc;
@@ -439,6 +439,22 @@ mu_stream_skip_input_bytes (mu_stream_t stream, mu_off_t count, mu_off_t *pres)
   return rc;
 }
 
+/* A wrapper for the above function.  It is normally called from a
+   seek method implementation, so it makes sure the MU_STREAM_SEEK
+   is cleared while in _stream_skip_input_bytes, to avoid infitite
+   recursion that may be triggered by _stream_flush_buffer invoking
+   stream->seek. */
+int
+mu_stream_skip_input_bytes (mu_stream_t stream, mu_off_t count, mu_off_t *pres)
+{
+  int rc;
+  int seek_flag = stream->flags & MU_STREAM_SEEK;
+  stream->flags &= ~MU_STREAM_SEEK;
+  rc = _stream_skip_input_bytes (stream, count, pres);
+  stream->flags |= seek_flag;
+  return rc;
+}
+
 int
 mu_stream_set_buffer (mu_stream_t stream, enum mu_buffer_type type,
 		      size_t size)