Finish imap mailbox implementation.

Some features are missing (notably, quick access), but overall the
implementation offers all necessary features and performs a lot
better than 2.x.

Additionally, this commit fixes a minor bug in the imap4d implementation:
the STORE command did not accept empty flag list.

* NEWS: Update.
* imap4d/store.c (store_thunk): Accept empty list as "items" argument.

* include/mailutils/imap.h (mu_imap_store_flags): New proto.
* include/mailutils/imaputil.h (mu_imap_format_flags): Add new argument.
* include/mailutils/sys/imap.h (_MU_IMAP_MSG_ATTRCHG): New flag.
* libmailutils/imapio/flags.c: Accept three arguments, last one being
a boolean flag specifying whether we want \Recent in the output.
* libproto/imap/storeflg.c: New flag.
* libproto/imap/Makefile.am (libmu_imap_la_SOURCES): Add storeflg.c.
* libproto/imap/mbox.c: Finish the basic implementation.

* imap4d/sync.c: Update calls to mu_imap_format_flags.
* imap4d/util.c: Likewise.
* mu/imap.c: Likewise.
* libmailutils/imapio/sendflg.c (mu_imapio_send_flags): Likewise.

-GNU mailutils NEWS -- history of user-visible changes. 2011-12-17
+GNU mailutils NEWS -- history of user-visible changes. 2011-12-23
 Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
 Free Software Foundation, Inc.
 See the end of file for copying conditions.
@@ -23,7 +23,6 @@ Due to the global nature of these changes, the rewrite of some parts
 of the package is not yet finished, while some others are considered
 experimental.  Not currently implemented are:
 
- - imap client
  - nntp client
  
 Experimental features are:
@@ -144,7 +143,7 @@ It is normally not needed to specify --with-gdbm, --with-berkeley-db
 or --with-ndbm explicitly.  Configuration will automatically pick up
 all available DBM libraries it can use.
 
-** Imap and nntp clients are not yet implemented
+** Nntp client is not yet implemented
 
 ** Link with GSASL by default
 

@@ -75,14 +75,14 @@ store_thunk (imap4d_parsebuf_t p)
   if (status)
     imap4d_parsebuf_exit (p, "Failed to parse message set");
 
-  if (p->token[0] != '(')
-    imap4d_parsebuf_exit (p, "Syntax error");
-  imap4d_parsebuf_next (p, 1);
-  
-  do
-    if (mu_imap_flag_to_attribute (p->token, &pclos->type))
-      imap4d_parsebuf_exit (p, "Unrecognized flag");
-  while (imap4d_parsebuf_next (p, 1) && p->token[0] != ')');
+  if (strcmp (p->token, "NIL"))
+    {
+      if (p->token[0] != '(')
+	imap4d_parsebuf_exit (p, "Syntax error");
+      while (imap4d_parsebuf_next (p, 1) && p->token[0] != ')')
+	if (mu_imap_flag_to_attribute (p->token, &pclos->type))
+	  imap4d_parsebuf_exit (p, "Unrecognized flag");
+    }
   return RESP_OK;
 }
 

@@ -100,7 +100,7 @@ notify (void)
 	      if (nflags != attr_table[i-1])
 		{
 		  io_sendf ("* %lu FETCH FLAGS (",  (unsigned long) i);
-		  mu_imap_format_flags (iostream, nflags);
+		  mu_imap_format_flags (iostream, nflags, 1);
 		  io_sendf (")\n");
 		  attr_table[i-1] = nflags;
 		}

@@ -499,7 +499,7 @@ util_print_flags (mu_attribute_t attr)
   int flags = 0;
 
   mu_attribute_get_flags (attr, &flags);
-  mu_imap_format_flags (iostream, flags);
+  mu_imap_format_flags (iostream, flags, 1);
 }
 
 int

@@ -71,6 +71,16 @@ int mu_imap_fetch (mu_imap_t imap, int uid, const char *msgset,
 int mu_imap_store (mu_imap_t imap, int uid, const char *msgset,
 		   const char *items);
 
+#define MU_IMAP_STORE_SET 0
+#define MU_IMAP_STORE_ADD 1
+#define MU_IMAP_STORE_CLR 2
+#define MU_IMAP_STORE_SILENT 0x10
+
+#define MU_IMAP_STORE_OPMASK 0xf
+
+int mu_imap_store_flags (mu_imap_t imap, int uid, const char *msgset,
+			 int op, int flags);
+  
 int mu_imap_delete (mu_imap_t imap, const char *mailbox);
 int mu_imap_rename (mu_imap_t imap, const char *mailbox,
 		    const char *new_mailbox);

@@ -26,7 +26,7 @@ extern "C" {
 int mu_imap_wildmatch (const char *pattern, const char *name, int delim);
 
 int mu_imap_flag_to_attribute (const char *item, int *attr);
-int mu_imap_format_flags (mu_stream_t str, int flags);
+  int mu_imap_format_flags (mu_stream_t str, int flags, int include_recent);
 
 #ifdef __cplusplus
 }

@@ -240,9 +240,10 @@ int _mu_imaps_url_init (mu_url_t url);
 
 int _mu_imap_mailbox_init (mu_mailbox_t mailbox);
 
-#define _MU_IMAP_MSG_SCANNED   0x01
-#define _MU_IMAP_MSG_CACHED    0x02
-#define _MU_IMAP_MSG_LINES     0x04
+#define _MU_IMAP_MSG_SCANNED   0x01 /* Message has already been scanned */
+#define _MU_IMAP_MSG_CACHED    0x02 /* Message is cached */
+#define _MU_IMAP_MSG_LINES     0x04 /* Number of lines is computed */
+#define _MU_IMAP_MSG_ATTRCHG   0x08 /* Message attributes has changed */
 
 struct _mu_imap_message
 {

@@ -60,7 +60,7 @@ mu_imap_flag_to_attribute (const char *item, int *attr)
 }
 
 int
-mu_imap_format_flags (mu_stream_t str, int flags)
+mu_imap_format_flags (mu_stream_t str, int flags, int include_recent)
 {
   int i;
   int delim = 0;
@@ -74,7 +74,7 @@ mu_imap_format_flags (mu_stream_t str, int flags)
 	delim = 1;
       }
 
-  if (MU_ATTRIBUTE_IS_UNSEEN (flags))
+  if (include_recent && MU_ATTRIBUTE_IS_UNSEEN (flags))
     {
       if (delim)
 	mu_stream_printf (str, " ");

@@ -29,7 +29,7 @@ mu_imapio_send_flags (struct _mu_imapio *io, int flags)
   rc = mu_stream_write (io->_imap_stream, "(", 1, NULL);
   if (rc)
     return rc;
-  rc = mu_imap_format_flags (io->_imap_stream, flags);
+  rc = mu_imap_format_flags (io->_imap_stream, flags, io->_imap_server);
   if (rc == 0)
     rc = mu_stream_write (io->_imap_stream, ")", 1, NULL);
   return rc;

@@ -60,6 +60,7 @@ libmu_imap_la_SOURCES = \
  status.c\
  starttls.c\
  store.c\
+ storeflg.c\
  subscribe.c\
  tag.c\
  trace.c\

@@ -83,67 +83,66 @@ struct save_closure
   struct _mu_imap_message *imsg;
 };
 
-static void
-_save_message (void *data, int code, size_t sdat, void *pdat)
+static int
+_save_message_parser (void *item, void *data)
 {
+  union mu_imap_fetch_response *resp = item;
   struct save_closure *clos = data;
-  struct _mu_imap_message *imsg = clos->imsg;
-  struct _mu_imap_mailbox *imbx = imsg->imbx;
-  mu_list_t list = pdat;
-  union mu_imap_fetch_response *resp;
-  int rc;
-  mu_stream_t istr, flt;
-  mu_off_t size;
-  
-  rc = mu_list_get (list, 0, (void**)&resp);
-  if (rc)
-    {
-      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-		(_("cannot get response item: %s"),
-		 mu_strerror (rc)));
-      imbx->last_error = rc;
-      return;
-    }
 
-  if (resp->type != MU_IMAP_FETCH_BODY)
+  if (resp->type == MU_IMAP_FETCH_BODY)
     {
-      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-		(_("fetch returned a not requested item %d"), resp->type));
-      imbx->last_error = MU_ERR_FAILURE;
-      return;
-    }
-  rc = mu_static_memory_stream_create (&istr, resp->body.text,
-				       strlen (resp->body.text));
-
-  if (rc)
-    {
-      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-		(_("mu_static_memory_stream_create: %s"),
-		 mu_strerror (rc)));
-      imbx->last_error = rc;
-      return;
-    }
+      struct _mu_imap_message *imsg = clos->imsg;
+      struct _mu_imap_mailbox *imbx = imsg->imbx;
+      int rc;
+      mu_stream_t istr, flt;
+      mu_off_t size;
+  
+      rc = mu_static_memory_stream_create (&istr, resp->body.text,
+					   strlen (resp->body.text));
+      
+      if (rc)
+	{
+	  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		    (_("mu_static_memory_stream_create: %s"),
+		     mu_strerror (rc)));
+	  imbx->last_error = rc;
+	  return 0;
+	}
 
-  rc = mu_filter_create (&flt, istr, "CRLF", MU_FILTER_DECODE, MU_STREAM_READ);
-  mu_stream_unref (istr);
-  if (rc)
-    {
-      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-		(_("mu_filter_create: %s"), mu_strerror (rc)));
-      imbx->last_error = rc;
-      return;
-    }
+      rc = mu_filter_create (&flt, istr, "CRLF", MU_FILTER_DECODE,
+			     MU_STREAM_READ);
+      mu_stream_unref (istr);
+      if (rc)
+	{
+	  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		    (_("mu_filter_create: %s"), mu_strerror (rc)));
+	  imbx->last_error = rc;
+	  return 0;
+	}
       
-  rc = mu_stream_copy (clos->save_stream, flt, 0, &size);
-  mu_stream_destroy (&flt);
-  if (rc)
-    {
-      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-		(_("copying to cache failed: %s"), mu_strerror (rc)));
-      imbx->last_error = rc;
-      return;
+      rc = mu_stream_copy (clos->save_stream, flt, 0, &size);
+      mu_stream_destroy (&flt);
+      if (rc)
+	{
+	  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		    (_("copying to cache failed: %s"), mu_strerror (rc)));
+	  imbx->last_error = rc;
+	  return 0;
+	}
+      clos->size = size;
     }
-  clos->size = size;
+  else
+    mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_TRACE0,
+	      (_("fetch returned a not requested item %d"),
+	       resp->type));
+  return 0;
+}
+
+static void
+_save_message (void *data, int code, size_t sdat, void *pdat)
+{
+  mu_list_t list = pdat;
+  mu_list_foreach (list, _save_message_parser, data);
 }
 
 static int
@@ -310,6 +309,7 @@ _imap_attr_set_flags (mu_attribute_t attr, int flags)
   if (!imsg)
     return EINVAL;
   imsg->attr_flags |= flags;
+  imsg->flags |= _MU_IMAP_MSG_ATTRCHG;
   return 0;
 }
 
@@ -321,6 +321,7 @@ _imap_attr_clr_flags (mu_attribute_t attr, int flags)
   if (!imsg)
     return EINVAL;
   imsg->attr_flags |= flags;
+  imsg->flags |= _MU_IMAP_MSG_ATTRCHG;
   return 0;
 }
 
@@ -511,6 +512,97 @@ _imap_msg_lines (mu_message_t msg, size_t *plines, int quick)
 }
 
 static int
+_copy_imapenvelope (struct mu_imapenvelope *env,
+		    struct mu_imapenvelope const *src)
+{  
+  env->date = src->date;
+  env->tz   = src->tz;
+  
+  if (src->subject && (env->subject = strdup (src->subject)) == NULL)
+    return ENOMEM;
+  if (src->from && (env->from = mu_address_dup (src->from)) == NULL)
+    return ENOMEM;
+  if (src->sender && (env->sender = mu_address_dup (src->sender)) == NULL)
+    return ENOMEM;
+  if (src->reply_to && (env->reply_to = mu_address_dup (src->reply_to))
+      == NULL)
+    return ENOMEM;
+  if (src->to && (env->to = mu_address_dup (src->to)) == NULL)
+    return ENOMEM;
+  if (src->cc && (env->cc = mu_address_dup (src->cc)) == NULL)
+    return ENOMEM;
+  if (src->bcc && (env->bcc = mu_address_dup (src->bcc)) == NULL)
+    return ENOMEM;
+  if (src->in_reply_to && (env->in_reply_to = strdup (src->in_reply_to))
+      == NULL)
+    return ENOMEM;
+  if (src->message_id && (env->message_id = strdup (src->message_id)) == NULL)
+    return ENOMEM;
+  return 0;
+}
+		   
+static int
+_imap_msg_imapenvelope (mu_message_t msg, struct mu_imapenvelope **penv)
+{
+  struct _mu_imap_message *imsg = mu_message_get_owner (msg);
+  int rc = 0;
+  struct mu_imapenvelope *env = calloc (1, sizeof (*env));
+
+  if (!env)
+    return ENOMEM;
+  rc = _copy_imapenvelope (env, imsg->env);
+  if (rc)
+    mu_message_imapenvelope_free (env);
+  else
+    *penv = env;
+  return rc;
+}
+
+static int
+fetch_bodystructure_parser (void *item, void *data)
+{
+  union mu_imap_fetch_response *resp = item;
+  struct mu_bodystructure **pbs = data;
+
+  if (resp->type == MU_IMAP_FETCH_BODYSTRUCTURE)
+    {
+      *pbs = resp->bodystructure.bs;
+      resp->bodystructure.bs = NULL;
+    }
+  else
+    mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_TRACE0,
+	      (_("fetch returned a not requested item %d"),
+	       resp->type));
+  return 0;
+}
+
+static void
+_imap_bodystructure_callback (void *data, int code, size_t sdat, void *pdat)
+{
+  mu_list_t list = pdat;
+  mu_list_foreach (list, fetch_bodystructure_parser, data);
+}
+  
+static int
+_imap_msg_bodystructure (mu_message_t msg, struct mu_bodystructure **pbs)
+{
+  struct _mu_imap_message *imsg = mu_message_get_owner (msg);
+  struct _mu_imap_mailbox *imbx = imsg->imbx;
+  mu_imap_t imap = imbx->mbox->folder->data;
+  int rc;
+  char *msgset;
+  
+  rc = mu_asprintf (&msgset, "%lu", _imap_msg_no (imsg));
+  if (rc)
+    return rc;
+
+  rc = _imap_fetch_with_callback (imap, msgset, "BODYSTRUCTURE",
+				  _imap_bodystructure_callback, pbs);
+  free (msgset);
+  return rc;
+}
+
+static int
 _imap_mbx_get_message (mu_mailbox_t mailbox, size_t msgno, mu_message_t *pmsg)
 {
   struct _mu_imap_mailbox *imbx = mailbox->data;
@@ -535,7 +627,9 @@ _imap_mbx_get_message (mu_mailbox_t mailbox, size_t msgno, mu_message_t *pmsg)
       mu_message_set_get_stream (msg, _imap_msg_get_stream, imsg);
       mu_message_set_size (msg, _imap_msg_size, imsg);
       mu_message_set_lines (msg, _imap_msg_lines, imsg);
-
+      mu_message_set_imapenvelope (msg, _imap_msg_imapenvelope, imsg);
+      mu_message_set_bodystructure (msg, _imap_msg_bodystructure, imsg);
+      
       do
 	{
 	  rc = _imap_msg_env_setup (imsg, msg);
@@ -621,6 +715,8 @@ _imap_mbx_open (mu_mailbox_t mbox, int flags)
   mu_url_t url;
   mu_imap_t imap;
   
+  mbox->flags = flags;
+  
   rc = mu_mailbox_get_url (mbox, &url);
   if (rc)
     return rc;
@@ -727,9 +823,86 @@ _imap_uidvalidity (mu_mailbox_t mbox, unsigned long *pn)
 static int
 _imap_mbx_expunge (mu_mailbox_t mbox)
 {
+  struct _mu_imap_mailbox *imbx = mbox->data;
   mu_folder_t folder = mbox->folder;
   mu_imap_t imap = folder->data;
-  return mu_imap_expunge (imap);
+  size_t i;
+  char *msgset;
+  int rc;
+  int delflg = 0;
+  
+  for (i = 0; i < imbx->msgs_cnt; i++)
+    {
+      if (imbx->msgs[i].flags & _MU_IMAP_MSG_ATTRCHG)
+	{
+	  rc = mu_asprintf (&msgset, "%lu", i + 1);
+	  if (rc)
+	    break;
+	  rc = mu_imap_store_flags (imap, 0, msgset,
+				    MU_IMAP_STORE_SET|MU_IMAP_STORE_SILENT,
+				    imbx->msgs[i].attr_flags);
+	  delflg |= imbx->msgs[i].attr_flags & MU_ATTRIBUTE_DELETED;
+	  free (msgset);
+	  if (rc)
+	    break;
+	}
+    }
+
+  if (rc)
+    return rc;
+
+  if (delflg)
+    rc = mu_imap_expunge (imap);
+  
+  return rc;
+}
+
+static int
+_imap_mbx_sync (mu_mailbox_t mbox)
+{
+  struct _mu_imap_mailbox *imbx = mbox->data;
+  mu_folder_t folder = mbox->folder;
+  mu_imap_t imap = folder->data;
+  size_t i;
+  char *msgset;
+  int rc;
+  
+  if (!imbx->msgs)
+    return 0; 
+
+  for (i = 0; i < imbx->msgs_cnt; i++)
+    {
+      if (imbx->msgs[i].flags & _MU_IMAP_MSG_ATTRCHG)
+	{
+	  rc = mu_asprintf (&msgset, "%lu", i + 1);
+	  if (rc)
+	    break;
+	  rc = mu_imap_store_flags (imap, 0, msgset,
+				    MU_IMAP_STORE_SET|MU_IMAP_STORE_SILENT,
+				    imbx->msgs[i].attr_flags);
+	  free (msgset);
+	  if (rc)
+	    break;
+	}
+    }
+  
+  return rc;
+}
+
+static int
+_imap_mbx_append_message (mu_mailbox_t mbox, mu_message_t msg)
+{
+  int rc;
+  mu_folder_t folder = mbox->folder;
+  mu_imap_t imap = folder->data;
+  mu_url_t url;
+  const char *mbox_name;
+  
+  rc = mu_mailbox_get_url (mbox, &url);
+  if (rc)
+    return rc;
+  rc = mu_url_sget_path (url, &mbox_name);
+  return mu_imap_append_message (imap, mbox_name, 0, NULL, NULL, msg);
 }
 
 static int _compute_lines (struct mu_bodystructure *bs, size_t *pcount);
@@ -934,13 +1107,9 @@ _mu_imap_mailbox_init (mu_mailbox_t mailbox)
   mailbox->_scan = _imap_mbx_scan;
   mailbox->_is_updated = _imap_mbx_is_updated;
   mailbox->_get_message = _imap_mbx_get_message;
+  mailbox->_sync = _imap_mbx_sync;
+  
+  mailbox->_append_message = _imap_mbx_append_message;
   
-  //FIXME
-#if 0
-
-  /* Messages.  */
-  mailbox->_append_message = _mu_imap_append_message;
-
-#endif
   return 0;
 }

+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2010, 2011 Free Software Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General
+   Public License along with this library.  If not, see
+   <http://www.gnu.org/licenses/>. */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <mailutils/errno.h>
+#include <mailutils/assoc.h>
+#include <mailutils/stream.h>
+#include <mailutils/imap.h>
+#include <mailutils/imaputil.h>
+#include <mailutils/sys/imap.h>
+
+int
+mu_imap_store_flags (mu_imap_t imap, int uid, const char *msgset,
+		     int op, int flags)
+{
+  int status;
+  static char *cmd[] = { "FLAGS", "+FLAGS", "-FLAGS" };
+  
+  if (imap == NULL || (op & MU_IMAP_STORE_OPMASK) >= MU_ARRAY_SIZE (cmd))
+    return EINVAL;
+  if (!imap->io)
+    return MU_ERR_NO_TRANSPORT;
+  if (imap->session_state < MU_IMAP_SESSION_SELECTED)
+    return MU_ERR_SEQ;
+
+  switch (imap->client_state)
+    {
+    case MU_IMAP_CLIENT_READY:
+      status = _mu_imap_tag_next (imap);
+      MU_IMAP_CHECK_EAGAIN (imap, status);
+      mu_imapio_printf (imap->io, "%s ", imap->tag_str);
+      if (uid)
+	mu_imapio_printf (imap->io, "UID ");
+      mu_imapio_printf (imap->io, "STORE %s %s", msgset,
+			cmd[op & MU_IMAP_STORE_OPMASK]);
+      if (op & MU_IMAP_STORE_SILENT)
+	mu_imapio_printf (imap->io, ".SILENT");
+      mu_imapio_send_flags (imap->io, flags);
+      mu_imapio_printf (imap->io, "\r\n");
+      status = mu_imapio_last_error (imap->io);
+      MU_IMAP_CHECK_ERROR (imap, status);
+      MU_IMAP_FCLR (imap, MU_IMAP_RESP);
+      imap->client_state = MU_IMAP_CLIENT_STORE_RX;
+      
+    case MU_IMAP_CLIENT_STORE_RX:
+      /* FIXME: Handle unsolicited responses */
+      status = _mu_imap_response (imap, NULL, NULL);
+      MU_IMAP_CHECK_EAGAIN (imap, status);
+      switch (imap->resp_code)
+	{
+	case MU_IMAP_OK:
+	  status = 0;
+	  break;
+	  
+	case MU_IMAP_NO:
+	  status = MU_ERR_FAILURE;
+	  break;
+
+	case MU_IMAP_BAD:
+	  status = MU_ERR_BADREPLY;
+	  break;
+	}
+      imap->client_state = MU_IMAP_CLIENT_READY;
+      break;
+
+    default:
+      status = EINPROGRESS;
+    }
+  return status;
+}

@@ -422,7 +422,7 @@ fetch_response_printer (void *item, void *data)
       
     case MU_IMAP_FETCH_FLAGS:
       mu_stream_printf (str, "  flags = ");
-      mu_imap_format_flags (str, resp->flags.flags);
+      mu_imap_format_flags (str, resp->flags.flags, 1);
       mu_stream_printf (str, "\n");
       break;
       
@@ -796,13 +796,13 @@ print_imap_stats (struct mu_imap_stat *st)
   if (st->flags & MU_IMAP_STAT_DEFINED_FLAGS)
     {
       mu_printf (_("Flags defined: "));
-      mu_imap_format_flags (mu_strout, st->defined_flags);
+      mu_imap_format_flags (mu_strout, st->defined_flags, 0);
       mu_printf ("\n");
     }
   if (st->flags & MU_IMAP_STAT_PERMANENT_FLAGS)
     {
       mu_printf (_("Flags permanent: "));
-      mu_imap_format_flags (mu_strout, st->permanent_flags);
+      mu_imap_format_flags (mu_strout, st->permanent_flags, 0);
       mu_printf ("\n");
     }