mh: improve component reading (profile, context and whom).

* libmailutils/filter/header.c: New file.
* libmailutils/filter/Makefile.am (libfilter_la_SOURCES): Add header.c.
* libmailutils/filter/filter.c (mu_filter_get_list): Register
mu_header_filter.
* libmailutils/filter/inline-comment.c: (ilcmt_ws)
(ilcmt_rollback): New states.
(ILCMT_REMOVE_EMPTY_LINES,ILCMT_SQUEEZE_WS): New flags.
(ilcmt_data)<buf,size,level,replay>: New members.
(ilcmt_action): New enum
(new_ilcmt_state): Return enum ilcmt_action.
(_ilcmt_decoder): Optionally remove
empty lines and squeeze leading white space.
* libmailutils/property/mhprop.c (_mh_prop_read_stream): Remove
empty lines from the input.

* libmailutils/tests/Makefile.am (TESTSUITE_AT): Add hdrflt.at
and inline-comment.at.
* libmailutils/tests/testsuite.at: Include hdrflt.at and inline-comment.at.
* libmailutils/tests/fltst.c: Pass optional arguments to the
filter creation procedure.

* mh/mh_whom.c (read_header): New function.
(mh_whom): Rewrite using headers.

* include/mailutils/filter.h: Fix a typo.
* libmailutils/stream/fltstream.c: Likewise.
* libmu_auth/gsasl.c: Likewise.

@@ -56,7 +56,7 @@ enum mu_filter_command
 enum mu_filter_result
   {
     mu_filter_ok,
-    mu_filter_falure,
+    mu_filter_failure,
     mu_filter_moreinput,
     mu_filter_moreoutput,
     mu_filter_again
@@ -109,6 +109,7 @@ extern mu_filter_record_t mu_rfc_2047_Q_filter;
 extern mu_filter_record_t mu_rfc_2047_B_filter;
 extern mu_filter_record_t mu_from_filter;
 extern mu_filter_record_t mu_inline_comment_filter;
+extern mu_filter_record_t mu_header_filter;
 
 enum mu_iconv_fallback_mode
   {

@@ -27,6 +27,7 @@ libfilter_la_SOURCES =\
  filter.c\
  filter_iconv.c\
  fromflt.c\
+ header.c\
  inline-comment.c\
  linelenflt.c\
  qpflt.c

@@ -79,6 +79,7 @@ mu_filter_get_list (mu_list_t *plist)
       mu_list_append (filter_list, mu_rfc_2047_B_filter);
       mu_list_append (filter_list, mu_from_filter);
       mu_list_append (filter_list, mu_inline_comment_filter);
+      mu_list_append (filter_list, mu_header_filter);
       /* FIXME: add the default encodings?  */
     }
   *plist = filter_list;

+/* Message header filter for GNU Mailutils.
+   Copyright (C) 2010 Free Software Foundation, Inc.
+
+   GNU Mailutils is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GNU Mailutils is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>. */
+
+/* This filter reads the input data up to the first empty line (i.e. \n\n).
+   It is suitable to extract headers from RFC822 messages.
+ */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <stdlib.h>
+#include <string.h>
+#include <mailutils/errno.h>
+#include <mailutils/filter.h>
+#include <mailutils/cctype.h>
+
+enum hflt_state
+  {
+    hflt_initial,
+    hflt_newline,
+    hflt_copy
+  };
+
+static int
+new_state (int *pd, unsigned c)
+{
+  switch (*pd)
+    {
+    case hflt_initial:
+    case hflt_copy:
+      *pd = (c == '\n') ? hflt_newline : hflt_copy;
+      break;
+      
+    case hflt_newline:
+      if (c == '\n')
+	return 1;
+      *pd = hflt_copy;
+      break;
+    }
+  return 0;
+}
+
+static enum mu_filter_result
+_hflt_decoder (void *xd, enum mu_filter_command cmd,
+		struct mu_filter_io *iobuf)
+{
+  int *pd = xd;
+  size_t i, j;
+  const unsigned char *iptr;
+  size_t isize;
+  char *optr;
+  size_t osize;
+
+  switch (cmd)
+    {
+    case mu_filter_init:
+      *pd = hflt_initial;
+      return mu_filter_ok;
+      
+    case mu_filter_done:
+      return mu_filter_ok;
+      
+    default:
+      break;
+    }
+  
+  iptr = (const unsigned char *) iobuf->input;
+  isize = iobuf->isize;
+  optr = iobuf->output;
+  osize = iobuf->osize;
+
+  for (i = j = 0; i < isize && j < osize; i++)
+    {
+      unsigned char c = *iptr++;
+      if (new_state (pd, c))
+	{
+	  iobuf->eof = 1;
+	  break;
+	}
+      optr[j++] = c;
+    }
+
+  iobuf->isize = i;
+  iobuf->osize = j;
+  return mu_filter_ok;
+}
+  
+static int
+alloc_state (void **pret, int mode MU_ARG_UNUSED,
+	     int argc MU_ARG_UNUSED, const char **argv MU_ARG_UNUSED)
+{
+  int *pd = malloc (sizeof (*pd));
+  
+  if (!pd)
+    return ENOMEM;
+  *pret = pd;
+  return 0;
+}
+
+static struct _mu_filter_record _header_filter = {
+  "HEADER",
+  0,
+  alloc_state,
+  NULL,
+  _hflt_decoder
+};
+
+mu_filter_record_t mu_header_filter = &_header_filter;

@@ -36,22 +36,65 @@
 #include <string.h>
 #include <mailutils/errno.h>
 #include <mailutils/filter.h>
+#include <mailutils/cctype.h>
 
 enum ilcmt_state
   {
     ilcmt_initial,
     ilcmt_newline,
     ilcmt_copy,
-    ilcmt_comment
+    ilcmt_comment,
+    ilcmt_ws,
+    ilcmt_rollback
   };
 
+#define ILCMT_REMOVE_EMPTY_LINES 0x01
+#define ILCMT_SQUEEZE_WS         0x02
+
 struct ilcmt_data
 {
   enum ilcmt_state state;
   int cstart;
+  int flags;
+  char *buf;
+  size_t size;
+  size_t level;
+  size_t replay;
 };
 
+enum ilcmt_action
+  {
+    action_echo,
+    action_noecho,
+    action_error
+  };
+
+#define ILCMT_BUF_INIT 80
+#define ILCMT_BUF_INCR 16
+
 static int
+ilcmt_save (struct ilcmt_data *pd, int c)
+{
+  if (pd->level == pd->size)
+    {
+      size_t nsz;
+      char *np;
+      
+      if (pd->size == 0)
+	nsz = ILCMT_BUF_INIT;
+      else
+	nsz = pd->size + ILCMT_BUF_INCR;
+      np = realloc (pd->buf, nsz);
+      if (!np)
+	return 1;
+      pd->buf = np;
+      pd->size = nsz;
+    }
+  pd->buf[pd->level++] = c;
+  return 0;
+}
+
+static enum ilcmt_action
 new_ilcmt_state (struct ilcmt_data *pd, int c)
 {
   switch (pd->state)
@@ -61,12 +104,48 @@ new_ilcmt_state (struct ilcmt_data *pd, int c)
       if (c == pd->cstart)
 	{
 	  pd->state = ilcmt_comment;
-	  return 0;
+	  return action_noecho;
+	}
+      else if (c == '\n')
+	{
+	  if (pd->flags & ILCMT_REMOVE_EMPTY_LINES)
+	    return action_noecho;
+	}
+      else if (mu_isspace (c))
+	{
+	  if (pd->flags & ILCMT_REMOVE_EMPTY_LINES)
+	    {
+	      pd->state = ilcmt_ws;
+	      pd->level = 0;
+	      if (!(pd->flags & ILCMT_SQUEEZE_WS))
+		{
+		  if (ilcmt_save (pd, c))
+		    return action_error;
+		}
+	      return action_noecho;
+	    }
 	}
-      else
       pd->state = ilcmt_copy;
       break;
 
+    case ilcmt_ws:
+      if (c == '\n')
+	pd->state = ilcmt_newline;
+      else if (mu_isspace (c))
+	{
+	  if (!(pd->flags & ILCMT_SQUEEZE_WS))
+	    {
+	      if (ilcmt_save (pd, c))
+		return action_error;
+	    }
+	}
+      else
+	{
+	  pd->replay = 0;
+	  pd->state = ilcmt_rollback;
+	}
+      return action_noecho;
+      
     case ilcmt_copy:
       if (c == '\n')
 	pd->state = ilcmt_newline;
@@ -75,9 +154,13 @@ new_ilcmt_state (struct ilcmt_data *pd, int c)
     case ilcmt_comment:
       if (c == '\n')
 	pd->state = ilcmt_newline;
-      return 0;
+      return action_noecho;
+
+    default:
+      /* should not happen */
+      break;
     }
-  return 1;
+  return action_echo;
 }
 
 static enum mu_filter_result
@@ -109,11 +192,50 @@ _ilcmt_decoder (void *xd, enum mu_filter_command cmd,
   optr = iobuf->output;
   osize = iobuf->osize;
 
-  for (i = j = 0; i < isize && j < osize; i++)
+  i = j = 0;
+  if (pd->state == ilcmt_rollback)
+    {
+      if (pd->flags & ILCMT_SQUEEZE_WS)
+	{
+	  if (j == osize)
+	    {
+	      iobuf->osize = 1;
+	      return mu_filter_moreoutput;
+	    }
+	  optr[j++] = ' ';
+	  pd->state = ilcmt_copy;
+	}
+      else
+	while (j < osize)
+	  {
+	    if (pd->replay == pd->level)
+	      {
+		pd->state = ilcmt_copy;
+		break;
+	      }
+	    optr[j++] = pd->buf[pd->replay++];
+	  }
+      
+      if (pd->state == ilcmt_copy)
+	{
+	  /* Clear the buffer state. */
+	  pd->level = pd->replay = 0;
+	}
+    }
+    
+  for (; i < isize && j < osize; i++)
     {
       unsigned char c = *iptr++;
-      if (new_ilcmt_state (pd, c))
+      enum ilcmt_action action = new_ilcmt_state (pd, c);
+      if (action == action_echo)
 	optr[j++] = c;
+      else if (action == action_noecho)
+	{
+	  if (pd->state == ilcmt_rollback)
+	    break;
+	}
+      else
+	return mu_filter_failure;
     }
 
   iobuf->isize = i;
@@ -125,14 +247,39 @@ static int
 alloc_state (void **pret, int mode MU_ARG_UNUSED, int argc, const char **argv)
 {
   struct ilcmt_data *pd = malloc (sizeof (*pd));
+  int i;
   
   if (!pd)
     return ENOMEM;
 
-  if (argc == 2)
-    pd->cstart = argv[1][0];
-  else
   pd->cstart = ';';
+  pd->flags = 0;
+  pd->buf = NULL;
+  pd->size = pd->level = pd->replay = 0;
+
+  for (i = 1; i < argc; i++)
+    {
+      if (argv[i][1] == 0)
+	pd->cstart = argv[i][0];
+      else if (argv[i][0] == '-')
+	{
+	  switch (argv[i][1])
+	    {
+	    case 'r':
+	      pd->flags |= ILCMT_REMOVE_EMPTY_LINES;
+	      break;
+
+	    case 's':
+	      pd->flags |= ILCMT_SQUEEZE_WS;
+	      break;
+
+	    default:
+	      free (pd);
+	      return MU_ERR_PARSE;
+	    }
+	}
+    }
+
   *pret = pd;
   
   return 0;

@@ -100,7 +100,7 @@ _mh_prop_read_stream (mu_header_t *phdr, mu_stream_t stream)
 {
   int rc;
   mu_stream_t flt;
-  const char *argv[3];
+  const char *argv[4];
   mu_off_t size;
   size_t total;
   char *blurb;
@@ -111,7 +111,8 @@ _mh_prop_read_stream (mu_header_t *phdr, mu_stream_t stream)
   
   argv[0] = "INLINE-COMMENT";
   argv[1] = "#";
-  argv[2] = NULL;
+  argv[2] = "-r";
+  argv[3] = NULL;
   rc = mu_filter_create_args (&flt, stream, argv[0], 2, argv,
 			      MU_FILTER_DECODE, MU_STREAM_READ);
   if (rc)

@@ -60,7 +60,7 @@ filter_stream_init (struct _mu_filter_stream *fs)
     {
       struct mu_filter_io iobuf;
       memset (&iobuf, 0, sizeof (iobuf));
-      if (fs->xcode (fs->xdata, mu_filter_init, &iobuf) == mu_filter_falure)
+      if (fs->xcode (fs->xdata, mu_filter_init, &iobuf) == mu_filter_failure)
 	return iobuf.errcode;
     }
   return 0;
@@ -182,7 +182,7 @@ filter_read (mu_stream_t stream, char *buf, size_t size, size_t *pret)
 		}
 	      break;
 	  
-	    case mu_filter_falure:
+	    case mu_filter_failure:
 	      return iobuf.errcode;
 	      
 	    case mu_filter_moreinput:
@@ -288,7 +288,7 @@ filter_write_internal (mu_stream_t stream, enum mu_filter_command cmd,
 	    }
 	  break;
 	  
-	case mu_filter_falure:
+	case mu_filter_failure:
 	  return iobuf.errcode;
 	  
 	case mu_filter_moreinput:

@@ -68,6 +68,8 @@ TESTSUITE_AT = \
  decode2047.at\
  encode2047.at\
  fromflt.at\
+ hdrflt.at\
+ inline-comment.at\
  list.at\
  mailcap.at\
  prop.at\

@@ -99,7 +99,7 @@ usage (const char *diag)
     fp = stdout;
 
   fprintf (fp, "%s",
-	   "usage: fltst FILTER {encode|decode} {read|write} [shift=N] [linelen=N] [verbose] [printable] [nl]\n");
+	   "usage: fltst FILTER {encode|decode} {read|write} [shift=N] [linelen=N] [verbose] [printable] [nl] [-- args]\n");
   exit (diag ? 1 : 0);
 }
 
@@ -152,10 +152,18 @@ main (int argc, char * argv [])
 	printable++;
       else if (strcmp (argv[i], "nl") == 0)
 	newline_option++;
+      else if (strcmp (argv[i], "--") == 0)
+	{
+	  argv[i] = fltname;
+	  break;
+	}
       else
 	usage ("wrong option");
     }
 
+  argc -= i;
+  argv += i;
+  
   MU_ASSERT (mu_stdio_stream_create (&in, MU_STDIN_FD, 0));
   MU_ASSERT (mu_stdio_stream_create (&out, MU_STDOUT_FD, 0));
 
@@ -164,7 +172,9 @@ main (int argc, char * argv [])
   
   if (flags == MU_STREAM_READ)
     {
-      MU_ASSERT (mu_filter_create (&flt, in, fltname, mode,
+      MU_ASSERT (mu_filter_create_args (&flt, in, fltname,
+					argc, (const char **)argv,
+					mode,
 					MU_STREAM_READ|MU_STREAM_SEEK|
 					MU_STREAM_AUTOCLOSE));
       if (shift)
@@ -173,7 +183,9 @@ main (int argc, char * argv [])
     }
   else
     {
-      MU_ASSERT (mu_filter_create (&flt, out, fltname, mode,
+      MU_ASSERT (mu_filter_create_args (&flt, out, fltname,
+					argc, (const char **)argv,
+					mode,
 					MU_STREAM_WRITE));
       if (shift)
 	MU_ASSERT (mu_stream_seek (in, shift, MU_SEEK_SET, NULL));

+# This file is part of GNU Mailutils. -*- Autotest -*-
+# Copyright (C) 2010 Free Software Foundation, Inc.
+#
+# GNU Mailutils is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3, or (at
+# your option) any later version.
+#
+# GNU Mailutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>.
+
+AT_SETUP([header filter])
+AT_KEYWORDS([header-filter hdrflt])
+
+AT_DATA([input],
+[From: gray@gnu.org
+To: root@example.com
+Subject: test
+
+Mon beau pays par l'hiver soumis
+Quand reverrons-nous l'hirondelle
+Noire et blanche, noire et blanche
+Quand reverrons-nous l'hirondelle
+Blanche au ventre et noire aux ailes
+])
+
+AT_DATA([expout],[dnl
+From: gray@gnu.org
+To: root@example.com
+Subject: test
+])
+
+
+AT_CHECK([fltst header decode read < input],[0],[expout])
+AT_CHECK([fltst header decode write < input],[0],[expout])
+
+AT_CLEANUP
\ No newline at end of file

+# This file is part of GNU Mailutils. -*- Autotest -*-
+# Copyright (C) 2010 Free Software Foundation, Inc.
+#
+# GNU Mailutils is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3, or (at
+# your option) any later version.
+#
+# GNU Mailutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>.
+
+dnl -------------------------------------------------------------------
+dnl INLINECOM(NAME, [KW = `'], [OPTS = `'], [TEXT], [OUTPUT = `'])
+dnl -------------------------------------------------------------------
+m4_pushdef([INLINECOM],[
+AT_SETUP([inline-comment: $1])
+AT_KEYWORDS([filter inline-comment icmt $2])
+sed 's/\$.*//' > input <<EOT
+$4
+EOT
+sed 's/\$.*//' > expout <<EOT
+$5
+EOT
+
+AT_CHECK([fltst inline-comment decode read $3 < input],
+[0],
+[expout])
+AT_CHECK([fltst inline-comment decode write $3 < input],
+[0],
+[expout])
+AT_CLEANUP
+])
+
+dnl -------------------------------------------------------------------
+
+INLINECOM([default], [icmt00], [],
+[; comment 1
+text 1
+; comment 2
+text 2
+
+text 3
+; comment 3
+],
+[text 1
+text 2
+
+text 3
+])
+
+INLINECOM([change comment starter], [icmt02], [-- %],
+[% comment 1
+text 1
+% comment 2
+text 2
+
+text 3
+% comment 3
+],
+[text 1
+text 2
+
+text 3
+])
+
+INLINECOM([remove empty lines],[icmt03],[-- -r],
+[; comment 1
+text 1
+; comment 2
+; comment 3
+text 2
+
+	text 3
+
+	$
+text 4
+     text 5
+],
+[text 1
+text 2
+	text 3
+text 4
+     text 5[]dnl
+])	
+
+INLINECOM([remove empty lines/squeeze whitespace],[icmt04],[-- -r -s],
+[; comment 1
+text 1
+; comment 2
+; comment 3
+text 2
+
+	text 3
+
+	$
+text 4
+     text 5
+],
+[text 1
+text 2
+ text 3
+text 4
+ text 5[]dnl
+])	
+
+
+m4_popdef([INLINECOM])
+# End of inline-comment.at

@@ -66,4 +66,6 @@ m4_include([encode2047.at])
 m4_include([fromflt.at])
 m4_include([wicket.at])
 m4_include([prop.at])
+m4_include([inline-comment.at])
+m4_include([hdrflt.at])
 

@@ -82,7 +82,7 @@ _gsasl_encoder (void *xdata,
       if (status)
 	{
 	  flt->gsasl_err = status;
-	  return mu_filter_falure;
+	  return mu_filter_failure;
 	}
     }
     
@@ -141,7 +141,7 @@ _gsasl_decoder (void *xdata,
 	  
 	default:
 	  flt->gsasl_err = status;
-	  return mu_filter_falure;
+	  return mu_filter_failure;
 	}
     }
 

@@ -198,6 +198,64 @@ _print_local_recipient (void *item, void *data)
   return 0;
 }
 
+static mu_header_t
+read_header (mu_stream_t stream)
+{
+  int rc;
+  mu_stream_t flt;
+  mu_off_t size;
+  size_t total;
+  char *blurb;
+  mu_header_t hdr;
+  
+  rc = mu_stream_size (stream, &size);
+  if (rc)
+    {
+      mu_error (_("cannot get stream size: %s"), mu_strerror (rc));
+      exit (1);
+    }
+  
+  rc = mu_filter_create (&flt, stream, "HEADER",
+			 MU_FILTER_DECODE, MU_STREAM_READ);
+  if (rc)
+    {
+      mu_error (_("cannot open filter stream: %s"), mu_strerror (rc));
+      exit (1);
+    }
+
+  blurb = xmalloc (size + 1);
+
+  total = 0;
+  while (1)
+    {
+      size_t n;
+      
+      rc = mu_stream_read (flt, blurb + total, size - total, &n);
+      if (rc)
+	break;
+      if (n == 0)
+	break;
+      total += n;
+    }
+
+  mu_stream_destroy (&flt);
+  if (rc)
+    {
+      free (blurb);
+      mu_error (_("read error: %s"), mu_strerror (rc));
+      exit (1);
+    }
+
+  rc = mu_header_create (&hdr, blurb, total);
+  free (blurb);
+  if (rc)
+    {
+      mu_diag_funcall (MU_DIAG_ERROR, "mu_header_create", NULL, rc);
+      exit (1);
+    }
+  return hdr;
+}
+
 int
 mh_whom (const char *filename, int check)
 {
@@ -205,19 +263,35 @@ mh_whom (const char *filename, int check)
 
   if (access (filename, R_OK))
     {
-      mu_error ("%s: %s", filename, mu_strerror (rc));
+      mu_error ("%s: %s", filename, mu_strerror (errno));
       rc = -1;
     }
   else
     {
       size_t count = 0;
-      mu_property_t prop;
+      mu_header_t hdr;
+      mu_stream_t str;
+      int rc;
+      const char *val;
+
+      rc = mu_file_stream_create (&str, filename, MU_STREAM_READ);
+      if (rc)
+	{
+	  mu_diag_funcall (MU_DIAG_ERROR, "mu_file_stream_create",
+			   filename, rc);
+	  exit (1);
+	}
+      hdr = read_header (str);
+      mu_stream_unref (str);
 
       mh_read_aliases ();
-      prop = mh_read_property_file (xstrdup (filename), 1);
-      scan_addrs (mu_mhprop_get_value (prop, MU_HEADER_TO, NULL), 0);
-      scan_addrs (mu_mhprop_get_value (prop, MU_HEADER_CC, NULL), 0);
-      scan_addrs (mu_mhprop_get_value (prop, MU_HEADER_BCC, NULL), 1);
+
+      if (mu_header_sget_value (hdr, MU_HEADER_TO, &val) == 0)
+	scan_addrs (val, 0);
+      if (mu_header_sget_value (hdr, MU_HEADER_CC, &val) == 0)
+	scan_addrs (val, 0);
+      if (mu_header_sget_value (hdr, MU_HEADER_BCC, &val) == 0)
+	scan_addrs (val, 1);
 
       if (local_rcp)
 	{
@@ -236,7 +310,7 @@ mh_whom (const char *filename, int check)
 	  mu_error(_("no recipients"));
 	  rc = -1;
 	}
-      mu_property_destroy (&prop);
+      mu_header_destroy (&hdr);
     }
   destroy_addrs (&network_rcp);
   destroy_addrs (&local_rcp);