imap4d: speed up asynchronous mailbox notifications.

* imap4d/imap4d.h (util_format_attribute_flags): New prototype.
* imap4d/sync.c: Rewrite using flag table to keep attribute flags,
mu_mailbox_translate to convert uids to message numbers and
MU_EVT_MAILBOX_MESSAGE_EXPUNGE event to report expunged messages.
* imap4d/util.c  (_imap4d_attrlist): Remove \Recent
(util_attribute_to_type): Handle \Recent separately.
(util_format_attribute_flags): New function.
(util_print_flags): Rewrite as an extra entry point to the above.

mailboxes: New event MU_EVT_MAILBOX_MESSAGE_EXPUNGE.

* include/mailutils/observer.h: Rearrange and partially rename
event constants (all uses updated):
MU_EVT_MESSAGE_APPEND   =>   MU_EVT_MAILBOX_MESSAGE_APPEND
MU_EVT_AUTHORITY_FAILED =>   MU_EVT_FOLDER_AUTHORITY_FAILED
(MU_EVT_MAILBOX_MESSAGE_EXPUNGE): New event.
* libproto/mbox/mbox.c: Generate MU_EVT_MAILBOX_MESSAGE_EXPUNGE.
* libproto/pop/mbox.c: Likewise.
* libmailutils/base/amd.c: Likewise.

@@ -370,6 +370,7 @@ extern char *util_localname (void);
 
 extern int util_wcard_match (const char *string, const char *pattern,
 			     const char *delim);
+int util_format_attribute_flags (mu_stream_t str, int flags);
 void util_print_flags (mu_attribute_t attr);
 int util_attribute_to_type (const char *item, int *type);
 int util_type_to_attribute (int type, char **attr_str);

@@ -21,163 +21,24 @@
 /*
 
  */
-struct _uid_table
-{
-  size_t uid;
-  size_t msgno;
-  int notify;
-  mu_attribute_t attr;
-};
-
-static struct _uid_table *uid_table;
-static size_t uid_table_count;
-static int uid_table_loaded;
-
-static void
-add_flag (char **pbuf, const char *f)
-{
-  char *abuf = *pbuf;
-  abuf = realloc (abuf, strlen (abuf) + strlen (f) + 2);
-  if (abuf == NULL)
-    imap4d_bye (ERR_NO_MEM);
-  if (*abuf)
-    strcat (abuf, " ");
-  strcat (abuf, "\\Seen");
-  *pbuf = abuf;
-}
-
-static void
-notify_flag (size_t msgno, mu_attribute_t oattr)
-{
-  mu_message_t msg = NULL;
-  mu_attribute_t nattr = NULL;
-  int status ;
-  mu_mailbox_get_message (mbox, msgno, &msg);
-  mu_message_get_attribute (msg, &nattr);
-  status = mu_attribute_is_equal (oattr, nattr);
-
-  if (status == 0)
-    {
-      char *abuf = malloc (1);;
-      if (!abuf)
-	imap4d_bye (ERR_NO_MEM);
-      *abuf = '\0';
-      if (mu_attribute_is_seen (nattr) && mu_attribute_is_read (nattr))
-	if (!mu_attribute_is_seen (oattr) && !mu_attribute_is_read (oattr))
-	  {
-	    mu_attribute_set_seen (oattr);
-	    mu_attribute_set_read (oattr);
-	    add_flag (&abuf, "\\Seen");
-	  }
-      if (mu_attribute_is_answered (nattr))
-	if (!mu_attribute_is_answered (oattr))
-	  {
-	    mu_attribute_set_answered (oattr);
-	    add_flag (&abuf, "\\Answered");
-	  }
-      if (mu_attribute_is_flagged (nattr))
-	if (!mu_attribute_is_flagged (oattr))
-	  {
-	    mu_attribute_set_flagged (oattr);
-	    add_flag (&abuf, "\\Flagged");
-	  }
-      if (mu_attribute_is_deleted (nattr))
-	if (!mu_attribute_is_deleted (oattr))
-	  {
-	    mu_attribute_set_deleted (oattr);
-	    add_flag (&abuf, "\\Deleted");
-	  }
-      if (mu_attribute_is_draft (nattr))
-	if (!mu_attribute_is_draft (oattr))
-	  {
-	    mu_attribute_set_draft (oattr);
-	    add_flag (&abuf, "\\Draft");
-	  }
-      if (mu_attribute_is_recent (nattr))
-	if (!mu_attribute_is_recent (oattr))
-	  {
-	    mu_attribute_set_recent (oattr);
-	    add_flag (&abuf, "\\Recent");
-	  }
-      if (*abuf)
-	io_untagged_response (RESP_NONE, "%lu FETCH FLAGS (%s)",
-		                 (unsigned long) msgno, abuf);
-      free (abuf);
-    }
-}
-
-/* The EXPUNGE response reports that the specified message sequence
-   number has been permanently removed from the mailbox.  The message
-   sequence number for each successive message in the mailbox is
-   immediately decremented by 1, and this decrement is reflected in
-   message sequence numbers in subsequent responses (including other
-   untagged EXPUNGE responses). */
-static void
-notify_deleted (void)
-{
-  if (uid_table)
-    {
-      size_t i;
-      size_t decr = 0;
-      for (i = 0; i < uid_table_count; i++)
-	{
-	  if (!(uid_table[i].notify))
-	    {
-	      io_untagged_response (RESP_NONE, "%lu EXPUNGED",
-			               (unsigned long) uid_table[i].msgno-decr);
-	      uid_table[i].notify = 1;
-	      decr++;
-	    }
-	}
-    }
-}
-
-
-static int
-notify_uid (size_t uid)
-{
-  if (uid_table)
-    {
-      size_t i;
-      for (i = 0; i < uid_table_count; i++)
-	{
-	  if (uid_table[i].uid == uid)
-	    {
-	      notify_flag (uid_table[i].msgno, uid_table[i].attr);
-	      uid_table[i].notify = 1;
-	      return 1;
-	    }
-	}
-    }
-  return 0;
-}
+static int *attr_table;
+static size_t attr_table_count;
+static int attr_table_loaded;
 
 static void
 free_uids (void)
 {
-  if (uid_table)
+  if (attr_table)
     {
-      size_t i;
-      for (i = 0; i < uid_table_count; i++)
-	mu_attribute_destroy (&(uid_table[i].attr), NULL);
-      free (uid_table);
-      uid_table = NULL;
+      free (attr_table);
+      attr_table = NULL;
     }
-  uid_table_count = 0;
-  uid_table_loaded = 0;
-}
-
-static void
-reset_notify (void)
-{
-  size_t i;
-
-  for (i = 0; i < uid_table_count; i++)
-    uid_table[i].notify = 0;
+  attr_table_count = 0;
+  attr_table_loaded = 0;
 }
 
 static void
-reset_uids (void)
+reread_attributes (void)
 {
   size_t total = 0;
   size_t i;
@@ -185,26 +46,24 @@ reset_uids (void)
   free_uids ();
 
   mu_mailbox_messages_count (mbox, &total);
+  if (total > attr_table_count)
+    {
+      attr_table = realloc (attr_table, sizeof (*attr_table) * total);
+      if (!attr_table)
+	imap4d_bye (ERR_NO_MEM);
+      attr_table_count = total;
+    }
+  
   for (i = 1; i <= total; i++)
     {
       mu_message_t msg = NULL;
       mu_attribute_t attr = NULL;
-      size_t uid = 0;
-      uid_table = realloc (uid_table, sizeof (*uid_table) *
-			   (uid_table_count + 1));
-      if (!uid_table)
-	imap4d_bye (ERR_NO_MEM);
+
       mu_mailbox_get_message (mbox, i, &msg);
       mu_message_get_attribute (msg, &attr);
-      mu_message_get_uid (msg, &uid);
-      uid_table[uid_table_count].uid = uid;
-      uid_table[uid_table_count].msgno = i;
-      uid_table[uid_table_count].notify = 0;
-      mu_attribute_create (&(uid_table[uid_table_count].attr), NULL);
-      mu_attribute_copy (uid_table[uid_table_count].attr, attr);
-      uid_table_count++;
+      mu_attribute_get_flags (attr, &attr_table[i-1]);
     }
-  uid_table_loaded = 1;
+  attr_table_loaded = 1;
 }
 
 static void
@@ -216,25 +75,34 @@ notify (void)
   
   mu_mailbox_messages_count (mbox, &total);
 
-  if (!uid_table)
+  if (!attr_table)
     {
       reset = 1;
-      reset_uids ();
+      reread_attributes ();
+      mu_mailbox_messages_recent (mbox, &recent);
     }
-  
-  if (uid_table)
+  else if (attr_table)
     {
       size_t i;
 
       for (i = 1; i <= total; i++)
 	{
 	  mu_message_t msg = NULL;
-	  size_t uid = 0;
+	  mu_attribute_t nattr = NULL;
+	  int nflags;
+	  
 	  mu_mailbox_get_message (mbox, i, &msg);
-	  mu_message_get_uid (msg, &uid);
-	  notify_uid (uid);
+	  mu_message_get_attribute (msg, &nattr);
+	  mu_attribute_get_flags (nattr, &nflags);
+
+	  if (nflags != attr_table[i-1])
+	    {
+	      io_sendf ("* %lu FETCH FLAGS (",  (unsigned long) i);
+	      util_format_attribute_flags (iostream, nflags);
+	      io_sendf (")\n");
+	      attr_table[i-1] = nflags;
+	    }
 	}
-      notify_deleted ();
       mu_mailbox_messages_recent (mbox, &recent);
     }
 
@@ -242,35 +110,28 @@ notify (void)
   io_untagged_response (RESP_NONE, "%lu RECENT", (unsigned long) recent);
 
   if (!reset)
-    reset_uids ();
-  else
-    reset_notify ();
+    reread_attributes ();
 }
 
 size_t
 uid_to_msgno (size_t uid)
 {
-  size_t i;
-  for (i = 0; i < uid_table_count; i++)
-    if (uid_table[i].uid == uid)
-      return uid_table[i].msgno;
-  return 0;
+  size_t msgno;
+  mu_mailbox_translate (mbox, MU_MAILBOX_UID_TO_MSGNO, uid, &msgno);
+  return msgno;
 }
 
 int
 imap4d_sync_flags (size_t msgno)
 {
-  size_t i;
-  for (i = 0; i < uid_table_count; i++)
-    if (uid_table[i].msgno == msgno)
-      {
-	mu_message_t msg = NULL;
-	mu_attribute_t attr = NULL;
-	mu_mailbox_get_message (mbox, msgno, &msg);
-	mu_message_get_attribute (msg, &attr);
-	mu_attribute_copy (uid_table[i].attr, attr);
-	break;
-      }
+  if (attr_table)
+    {
+      mu_message_t msg = NULL;
+      mu_attribute_t attr = NULL;
+      mu_mailbox_get_message (mbox, msgno, &msg);
+      mu_message_get_attribute (msg, &attr);
+      mu_attribute_get_flags (attr, &attr_table[msgno-1]);
+    }
   return 0;
 }
 
@@ -288,6 +149,19 @@ action (mu_observer_t observer, size_t type, void *data, void *action_data)
     case MU_EVT_MAILBOX_DESTROY:
       mailbox_corrupt = 0;
       break;
+
+    case MU_EVT_MAILBOX_MESSAGE_EXPUNGE:
+      /* The EXPUNGE response reports that the specified message sequence
+	 number has been permanently removed from the mailbox.  The message
+	 sequence number for each successive message in the mailbox is
+	 immediately decremented by 1, and this decrement is reflected in
+	 message sequence numbers in subsequent responses (including other
+	 untagged EXPUNGE responses). */
+      {
+	size_t *exp = data;
+	io_untagged_response (RESP_NONE, "%lu EXPUNGED",
+			      (unsigned long) (exp[0] - exp[1]));
+      }
     }
   return 0;
 }
@@ -301,8 +175,11 @@ imap4d_set_observer (mu_mailbox_t mbox)
   mu_observer_create (&observer, mbox);
   mu_observer_set_action (observer, action, mbox);
   mu_mailbox_get_observable (mbox, &observable);
-  mu_observable_attach (observable, MU_EVT_MAILBOX_CORRUPT|MU_EVT_MAILBOX_DESTROY,
-		     observer);
+  mu_observable_attach (observable,
+			MU_EVT_MAILBOX_CORRUPT|
+			MU_EVT_MAILBOX_DESTROY|
+			MU_EVT_MAILBOX_MESSAGE_EXPUNGE,
+			observer);
   mailbox_corrupt = 0;
 }
 
@@ -314,7 +191,7 @@ imap4d_sync (void)
      If it was a close we do not send any notification.  */
   if (mbox == NULL)
     free_uids ();
-  else if (!uid_table_loaded || !mu_mailbox_is_updated (mbox))
+  else if (!attr_table_loaded || !mu_mailbox_is_updated (mbox))
     {
       if (mailbox_corrupt)
 	{
@@ -338,7 +215,7 @@ imap4d_sync (void)
     {
       size_t count = 0;
       mu_mailbox_messages_count (mbox, &count);
-      if (count != uid_table_count)
+      if (count != attr_table_count)
 	notify ();
     }
   return 0;

@@ -402,8 +402,7 @@ _imap4d_attrlist[] =
   { "\\Flagged", MU_ATTRIBUTE_FLAGGED },
   { "\\Deleted", MU_ATTRIBUTE_DELETED },
   { "\\Draft", MU_ATTRIBUTE_DRAFT },
-  { "\\Seen", MU_ATTRIBUTE_READ },
-  { "\\Recent", MU_ATTRIBUTE_RECENT },
+  { "\\Seen", MU_ATTRIBUTE_SEEN|MU_ATTRIBUTE_READ },
 };
 
 #define NATTR sizeof(_imap4d_attrlist)/sizeof(_imap4d_attrlist[0])
@@ -414,6 +413,13 @@ int
 util_attribute_to_type (const char *item, int *type)
 {
   int i;
+
+  if (mu_c_strcasecmp (item, "\\Recent") == 0)
+    {
+      *type = MU_ATTRIBUTE_RECENT;
+      return 0;
+    }
+  
   for (i = 0; i < _imap4d_nattr; i++)
     if (mu_c_strcasecmp (item, _imap4d_attrlist[i].name) == 0)
       {
@@ -423,68 +429,38 @@ util_attribute_to_type (const char *item, int *type)
   return 1;
 }
 
-/* Note: currently unused. Not needed, possibly? */
 int
-util_type_to_attribute (int type, char **attr_str)
+util_format_attribute_flags (mu_stream_t str, int flags)
 {
-  char *attr_list[NATTR];
-  int nattr = 0;
   int i;
-  size_t len = 0;
-
-  if (MU_ATTRIBUTE_IS_UNSEEN (type))
-    *attr_str = strdup ("\\Recent");
-  else
-    *attr_str = NULL;
-
+  int delim = 0;
+  
   for (i = 0; i < _imap4d_nattr; i++)
-    if (type & _imap4d_attrlist[i].flag)
+    if (flags & _imap4d_attrlist[i].flag)
       {
-	attr_list[nattr++] = _imap4d_attrlist[i].name;
-	len += 1 + strlen (_imap4d_attrlist[i].name);
+	if (delim)
+	  mu_stream_printf (str, " ");
+	mu_stream_printf (str, "%s", _imap4d_attrlist[i].name);
+	delim = 1;
       }
 
-  *attr_str = malloc (len + 1);
-  (*attr_str)[0] = 0;
-  if (*attr_str)
+  if (MU_ATTRIBUTE_IS_UNSEEN (flags))
     {
-      for (i = 0; i < nattr; i++)
-	{
-	  strcat (*attr_str, attr_list[i]);
-	  if (i != nattr - 1)
-	    strcat (*attr_str, " ");
-	}
+      if (delim)
+	mu_stream_printf (str, " ");
+      mu_stream_printf (str, "\\Recent");
     }
-
-  if (!*attr_str)
-    imap4d_bye (ERR_NO_MEM);
+  
   return 0;
 }
 
 void
 util_print_flags (mu_attribute_t attr)
 {
-  int i;
   int flags = 0;
-  int space = 0;
 
   mu_attribute_get_flags (attr, &flags);
-  for (i = 0; i < _imap4d_nattr; i++)
-    if (flags & _imap4d_attrlist[i].flag)
-      {
-	if (space)
-	  io_sendf (" ");
-	else
-	  space = 1;
-	io_sendf (_imap4d_attrlist[i].name);
-      }
-
-  if (MU_ATTRIBUTE_IS_UNSEEN (flags))
-    {
-      if (space)
-	io_sendf (" ");
-      io_sendf ("\\Recent");
-    }
+  util_format_attribute_flags (iostream, flags);
 }
 
 int

@@ -24,17 +24,26 @@
 #ifdef __cplusplus
 extern "C" {
 #endif
-                                          /* Call data type: */
-#define MU_EVT_MAILBOX_DESTROY     0x001  /*  mu_mailbox_t */
-#define MU_EVT_FOLDER_DESTROY      0x002  /*  mu_folder_t */ 
-#define MU_EVT_MAILER_DESTROY      0x004  /*  mu_mailer_t */
-#define MU_EVT_MESSAGE_DESTROY     0x008  /*  mu_message_t */
-#define MU_EVT_MESSAGE_ADD         0x010  /*  size_t *: FIXME */
-#define MU_EVT_MAILBOX_PROGRESS    0x020  /*  NULL: FIXME? */
-#define MU_EVT_AUTHORITY_FAILED    0x030  /*  NULL */
-#define MU_EVT_MAILBOX_CORRUPT     0x040  /*  mu_mailbox_t */
-#define MU_EVT_MAILER_MESSAGE_SENT 0x080  /*  mu_message_t */
-#define MU_EVT_MESSAGE_APPEND      0x100  /*  mu_message_qid_t: FIXME */ 
+  /* Mailbox events */
+                                               /* Call data type: */
+#define MU_EVT_MAILBOX_DESTROY          0x001  /*  mu_mailbox_t */
+#define MU_EVT_MAILBOX_PROGRESS         0x002  /*  NULL: FIXME? */
+#define MU_EVT_MAILBOX_CORRUPT          0x004  /*  mu_mailbox_t */
+#define MU_EVT_MAILBOX_MESSAGE_APPEND   0x008  /*  mu_message_qid_t: FIXME */ 
+#define MU_EVT_MAILBOX_MESSAGE_EXPUNGE  0x010  /*  size_t [2]
+					           (message number/number of 
+					            messages removed so far) */
+  /* Folder events */
+#define MU_EVT_FOLDER_DESTROY           0x020  /*  mu_folder_t */
+#define MU_EVT_FOLDER_AUTHORITY_FAILED  0x040  /*  NULL */
+
+  /* Message events */
+#define MU_EVT_MESSAGE_DESTROY          0x080  /*  mu_message_t */
+#define MU_EVT_MESSAGE_ADD              0x100  /*  size_t *: FIXME */
+
+  /* Mailer events */
+#define MU_EVT_MAILER_DESTROY           0x200  /*  mu_mailer_t */
+#define MU_EVT_MAILER_MESSAGE_SENT      0x400  /*  mu_message_t */
   
 #define MU_OBSERVER_NO_CHECK 1
 

@@ -900,7 +900,8 @@ amd_append_message (mu_mailbox_t mailbox, mu_message_t msg)
       char *qid;
       if (amd->cur_msg_file_name (mhm, &qid) == 0)
 	{
-	  mu_observable_notify (mailbox->observable, MU_EVT_MESSAGE_APPEND,
+	  mu_observable_notify (mailbox->observable, 
+	                        MU_EVT_MAILBOX_MESSAGE_APPEND,
 				qid);
 	  free (qid);
 	}
@@ -1148,6 +1149,7 @@ amd_expunge (mu_mailbox_t mailbox)
   struct _amd_message *mhm;
   size_t i;
   int updated = amd->has_new_msg;
+  size_t expcount = 0;
   
   if (amd == NULL)
     return EINVAL;
@@ -1175,9 +1177,15 @@ amd_expunge (mu_mailbox_t mailbox)
 
 	  if (amd->delete_msg)
 	    {
+	      size_t expevt[2] = { i + 1, expcount };
 	      rc = amd->delete_msg (amd, mhm);
 	      if (rc)
 		return rc;
+
+	      mu_observable_notify (mailbox->observable,
+				    MU_EVT_MAILBOX_MESSAGE_EXPUNGE,
+				    expevt);
+	      ++expcount;
 	    }
 	  else
 	    {

@@ -534,9 +534,7 @@ mu_message_destroy (mu_message_t *pmsg, void *owner)
 	 is very unfortunate.
 
 	 The `owner' stuff is a leftover from older mailutils versions.
-	 There is an ongoing attempt to remove it in the stream-cleanup
-	 branch. When it is ready, it will be merged to the HEAD and this
-	 will finally resolve this issue. */
+	 We are heading to removing it altogether. */
       if (msg->ref > 0)
 	msg->ref--;
       if ((msg->owner && msg->owner == owner)

@@ -2589,7 +2589,7 @@ imap_parse (f_imap_t f_imap)
 		{
 		  mu_observable_t observable = NULL;
 		  mu_folder_get_observable (f_imap->folder, &observable);
-		  mu_observable_notify (observable, MU_EVT_AUTHORITY_FAILED,
+		  mu_observable_notify (observable, MU_EVT_FOLDER_AUTHORITY_FAILED,
 					NULL);
 		  status = MU_ERR_AUTH_FAILURE;
 		}

@@ -1143,7 +1143,8 @@ mbox_append_message (mu_mailbox_t mailbox, mu_message_t msg)
     {
       char *buf = NULL;
       mu_asprintf (&buf, "%lu", (unsigned long) size);
-      mu_observable_notify (mailbox->observable, MU_EVT_MESSAGE_APPEND, buf);
+      mu_observable_notify (mailbox->observable, 
+                            MU_EVT_MAILBOX_MESSAGE_APPEND, buf);
       free (buf);
     }
   
@@ -1184,7 +1185,8 @@ mbox_expunge_unlocked (mu_mailbox_t mailbox, size_t dirty, int remove_deleted,
   size_t save_imapbase = 0;  /* uidvalidity is save in the first message.  */
   mu_off_t start_off;
   mu_off_t size;
-
+  size_t expcount = 0;
+  
   /* Set the marker position.  */
   start_off = mud->umessages[dirty]->envel_from;
 
@@ -1194,6 +1196,11 @@ mbox_expunge_unlocked (mu_mailbox_t mailbox, size_t dirty, int remove_deleted,
       
       if (remove_deleted && ATTRIBUTE_IS_DELETED (mum->attr_flags))
 	{
+	  size_t expevt[2] = { i + 1, expcount };
+	  mu_observable_notify (mailbox->observable,
+				MU_EVT_MAILBOX_MESSAGE_EXPUNGE,
+				expevt);
+	  expcount++;
 	  mu_message_destroy (&mum->message, mum);
 	  /* We save the uidvalidity in the first message, if it is being
 	     deleted we need to move the uidvalidity to the first available

@@ -864,6 +864,7 @@ pop_expunge (mu_mailbox_t mbox)
   struct _pop3_mailbox *mpd = mbox->data;
   int status = 0;
   size_t i;
+  size_t expcount = 0;
   
   if (mpd == NULL)
     return EINVAL;
@@ -879,9 +880,14 @@ pop_expunge (mu_mailbox_t mbox)
 	  (mpm->flags & _POP3_MSG_ATTRSET) &&
 	  (mpm->attr_flags & MU_ATTRIBUTE_DELETED))
 	{
+	  size_t expevt[2] = { i + 1, expcount };
 	  status = mu_pop3_dele (mpd->pop3, mpm->num);
 	  if (status)
 	    break;
+	  mu_observable_notify (mbox->observable,
+				MU_EVT_MAILBOX_MESSAGE_EXPUNGE,
+				&expevt);
+	  ++expcount;
 	}
     }
   return 0;

@@ -132,7 +132,7 @@ static const char *biff_user_name;
 static int
 notify_action (mu_observer_t obs, size_t type, void *data, void *action_data)
 {
-  if (type == MU_EVT_MESSAGE_APPEND && biff_user_name)
+  if (type == MU_EVT_MAILBOX_MESSAGE_APPEND && biff_user_name)
     {
       mu_message_qid_t qid = data;
       mu_mailbox_t mbox = mu_observer_get_owner (obs);
@@ -183,7 +183,8 @@ attach_notify (mu_mailbox_t mbox)
       mu_observer_create (&observer, mbox);
       mu_observer_set_action (observer, notify_action, mbox);
       mu_mailbox_get_observable (mbox, &observable);
-      mu_observable_attach (observable, MU_EVT_MESSAGE_APPEND, observer);
+      mu_observable_attach (observable, MU_EVT_MAILBOX_MESSAGE_APPEND, 
+                            observer);
     }
 }