Fix the "UNIX mailbox first message symptom".

UFMS, or "UNIX mailbox first message symptom", is a long-standing
bug that existed in all previous versions of Mailutils:  under
certain circumstances, a fragment of the message headers would get
prepended to the body of the very first message in a UNIX mailbox.
See the detailed description in testsuite/ufms.c.

Along with fixing this bug, this change also ensures a proper restoring
of UIDs from UNIX mailboxes.

* libproto/mbox/mbox.c (_msg_body_setup): Call mu_body_clear_modified
after constructing the body.
(new_message): Ditto for mu_message_clear_modified.
(mbox_reset): Rewrite.  Drop all cached messages and rescan entire
mailbox to avoid the "1st message symptom".
(mbox_expunge0): Change the call to mbox_reset.
* libproto/mbox/mboxscan.c (IS_X_UID, IS_X_IMAPBASE): New macros.
(mbox_scan_internal): Change handling of min_uid.
Attempt to get UID and imapbase from the corresponding message headers.
(mbox_scan0): Reflect the above changes.

* testsuite/.gitignore: Add ufms.
* testsuite/Makefile.am (noinst_PROGRAMS): Add ufms.
(TESTSUITE_AT): Add ufms.at.
* testsuite/testsuite.at: Include ufms.at.
* testsuite/ufms.at: New test case.
* testsuite/ufms.c: New test program.

* imap4d/testsuite/imap4d/expunge.exp: Fix UIDNEXT expectation.

@@ -63,7 +63,7 @@ imap4d_test "SELECT mbox1" \
 "1 EXISTS"\
 "0 RECENT"\
 -re {OK \[UIDVALIDITY [0-9]+\] UID valididy status}\
-"OK \[UIDNEXT 2\] Predicted next uid"\
+"OK \[UIDNEXT 6\] Predicted next uid"\
 "OK \[UNSEEN 1\] first unseen messsage"\
 "FLAGS (\\Answered \\Flagged \\Deleted \\Seen \\Draft)"\
 "OK \[PERMANENTFLAGS (\\Answered \\Deleted \\Seen)\] Permanent flags" \

@@ -349,6 +349,7 @@ _msg_body_setup (mu_message_t msg, mbox_message_t mum)
       mu_body_set_stream (body, stream, msg);
       mu_body_set_size (body, mbox_body_size, msg);
       mu_body_set_lines (body, mbox_body_lines, msg);
+      mu_body_clear_modified (body);
       mu_message_set_body (msg, body, mum);
     }
   return status;
@@ -571,6 +572,8 @@ new_message (mu_mailbox_t mailbox, mbox_message_t mum, mu_message_t *pmsg)
   mum->message = msg;
   mu_message_set_mailbox (msg, mailbox, mum);
 
+  mu_message_clear_modified (msg);
+
   *pmsg = msg;
   
   return 0;
@@ -1149,37 +1152,18 @@ mbox_append_message (mu_mailbox_t mailbox, mu_message_t msg)
 
 
 static void
-mbox_reset (mu_mailbox_t mailbox, size_t dirty, int remove_deleted)
+mbox_reset (mu_mailbox_t mailbox)
 {
   mbox_data_t mud = mailbox->data;
   size_t i;
-  size_t dlast;
 
   mu_monitor_wrlock (mailbox->monitor);
-  for (i = dirty, dlast = mud->messages_count - 1; i <= dlast; i++)
+  for (i = 0; i < mud->messages_count; i++)
     {
       /* Clear all the references */
       mbox_message_t mum = mud->umessages[i];
-      if (remove_deleted && ATTRIBUTE_IS_DELETED (mum->attr_flags))
-	{
-	  if ((i + 1) <= dlast)
-	    {
-	      /* Move all the pointers up.  So the message pointer
-		 part of mum will be at the right position.  */
-	      memmove (mud->umessages + i, mud->umessages + i + 1,
-		       (dlast - i) * sizeof (mum));
-	      memset (mum, 0, sizeof (*mum));
-	      /* We are not free()ing the useless mum, but instead
-		 we put it back in the pool, to be reused.  */
-	      mud->umessages[dlast] = mum;
-	      dlast--;
-	      /* Set mum to the new value after the memmove so it
-		 gets cleared to.  */
-	      mum = mud->umessages[i];
-	    }
-	  else
-	    memset (mum, 0, sizeof (*mum));
-	}
+      mu_message_destroy (&mum->message, mum);
+      memset (mum, 0, sizeof (*mum));
       mum->envel_from = mum->envel_from_end = 0;
       mum->body = mum->body_end = 0;
       mum->header_lines = mum->body_lines = 0;
@@ -1187,7 +1171,7 @@ mbox_reset (mu_mailbox_t mailbox, size_t dirty, int remove_deleted)
   mu_monitor_unlock (mailbox->monitor);
   /* This resets the messages_count, the last argument 0 means
      not to send event notification.  */
-  mbox_scan0 (mailbox, dirty, NULL, 0);
+  mbox_scan0 (mailbox, 1, NULL, 0);
 }
 
 static int
@@ -1434,7 +1418,7 @@ mbox_expunge0 (mu_mailbox_t mailbox, int remove_deleted)
       mu_stream_destroy (&tempstr);
       
       if (status == 0)
-	mbox_reset (mailbox, dirty, remove_deleted);
+	mbox_reset (mailbox);
     }
   if (mailbox->locker)
     mu_locker_unlock (mailbox->locker);

@@ -34,6 +34,8 @@
 
 #include <stdlib.h>
 #include <mbox0.h>
+#include <mailutils/cctype.h>
+#include <mailutils/cstr.h>
 
 /* Parsing.
    The approach is to detect the "From " as start of a new message, give the
@@ -279,6 +281,26 @@ do                                                                           \
  && (buf[5] == 'S' || buf[5] == 's')                                          \
  && (buf[6] == ':' || buf[6] == ' ' || buf[6] == '\t'))
 
+#define IS_X_UID(buf) (						\
+ (buf[0] == 'X' || buf[0] == 'x')		                \
+  && buf[1] == '-'						\
+  && (buf[2] == 'U' || buf[2] == 'u')				\
+  && (buf[3] == 'I' || buf[3] == 'i')				\
+  && (buf[4] == 'D' || buf[4] == 'd')				\
+  && (buf[5] == ':' || buf[5] == ' ' || buf[5] == '\t'))
+#define IS_X_IMAPBASE(buf) (					\
+ (buf[0] == 'X' || buf[0] == 'x')		                \
+  && buf[1] == '-'						\
+  && (buf[2] == 'I' || buf[2] == 'i')				\
+  && (buf[3] == 'M' || buf[3] == 'm')				\
+  && (buf[4] == 'A' || buf[4] == 'a')				\
+  && (buf[5] == 'P' || buf[5] == 'p')				\
+  && (buf[6] == 'B' || buf[6] == 'b')				\
+  && (buf[7] == 'A' || buf[7] == 'a')				\
+  && (buf[8] == 'S' || buf[8] == 's')				\
+  && (buf[9] == 'E' || buf[9] == 'e')				\
+  && (buf[10] == ':' || buf[10] == ' ' || buf[10] == '\t'))
+
 #define MBOX_SCAN_NOTIFY 0x1
 #define MBOX_SCAN_ONEMSG 0x2
 
@@ -298,7 +320,7 @@ mbox_scan_internal (mu_mailbox_t mailbox, mbox_message_t mum,
   int newline;
   size_t n = 0;
   mu_stream_t stream;
-  size_t min_uid = 0;
+  size_t min_uid = 1;
   int zn, isfrom = 0;
   char *temp;
   
@@ -345,9 +367,9 @@ mbox_scan_internal (mu_mailbox_t mailbox, mbox_message_t mum,
 		  mum->body_end = total - n - newline;
 		  mum->body_lines = --lines - newline;
 
-		  if (mum->uid <= min_uid)
+		  if (mum->uid < min_uid)
 		    {
-		      mum->uid = ++min_uid;
+		      mum->uid = min_uid++;
 		      /* Note that modification for when expunging.  */
 		      mum->attr_flags |= MU_ATTRIBUTE_MODIFIED;
 		    }
@@ -370,13 +392,32 @@ mbox_scan_internal (mu_mailbox_t mailbox, mbox_message_t mum,
 	      mum->attr_flags = 0;
 	      lines = 0;
 	    }
-	  else if (ISSTATUS(buf))
+	  else if (ISSTATUS (buf))
 	    {
 	      ATTRIBUTE_SET(buf, mum, 'r', 'R', MU_ATTRIBUTE_READ);
 	      ATTRIBUTE_SET(buf, mum, 'o', 'O', MU_ATTRIBUTE_SEEN);
 	      ATTRIBUTE_SET(buf, mum, 'a', 'A', MU_ATTRIBUTE_ANSWERED);
 	      ATTRIBUTE_SET(buf, mum, 'd', 'D', MU_ATTRIBUTE_DELETED);
 	    }
+	  else if (IS_X_UID (buf))
+	    {
+	      char *p;
+	      unsigned long n = strtoul (buf + 6, &p, 10);
+	      if (*p == 0 || mu_isspace (*p))
+		mum->uid = min_uid = n;
+	    }
+	  else if (mud->messages_count == 1 && IS_X_IMAPBASE (buf))
+	    {
+	      char *p;
+	      unsigned long n = strtoul (buf + 11, &p, 10);
+	      if (mu_isspace (*p))
+		mud->uidvalidity = n;
+	      n = strtoul (mu_str_skip_cset (p, " \t"), &p, 10);
+	      if (*p == 0 || mu_isspace (*p))
+		mud->uidnext = n;
+	      else
+		mud->uidvalidity = 0;
+	    }
 	}
 
       /* Body.  */
@@ -410,9 +451,9 @@ mbox_scan_internal (mu_mailbox_t mailbox, mbox_message_t mum,
       mum->body_end = total - newline;
       mum->body_lines = lines - newline;
 
-      if (mum->uid <= min_uid)
+      if (mum->uid < min_uid)
 	{
-	  mum->uid = ++min_uid;
+	  mum->uid = min_uid++;
 	  /* Note that modification for when expunging.  */
 	  mum->attr_flags |= MU_ATTRIBUTE_MODIFIED;
 	}
@@ -498,7 +539,7 @@ mbox_scan0 (mu_mailbox_t mailbox, size_t msgno, size_t *pcount, int do_notif)
 	}
     }
       
-  if (mud->messages_count > 0 && min_uid >= mud->uidnext)
+  if (mud->messages_count > 0 && min_uid > mud->uidnext)
     {
       mum = mud->umessages[0];
       mud->uidnext = min_uid + 1;

@@ -7,3 +7,4 @@ testsuite.log
 mbdel
 mimetest
 smtpsend
+ufms

@@ -42,7 +42,8 @@ INCLUDES = @MU_LIB_COMMON_INCLUDES@
 noinst_PROGRAMS = \
  mbdel\
  mimetest\
- smtpsend
+ smtpsend\
+ ufms
 
 LDADD = \
  ${MU_LIB_MBOX}\
@@ -68,6 +69,7 @@ smtpsend_LDADD = \
 TESTSUITE_AT = \
  mbdel.at\
  mime.at\
+ ufms.at\
  testsuite.at
 
 TESTSUITE = $(srcdir)/testsuite

@@ -20,3 +20,4 @@ AT_INIT
 
 m4_include([mime.at])
 m4_include([mbdel.at])
+m4_include([ufms.at])

+# This file is part of GNU Mailutils. -*- Autotest -*-
+# Copyright (C) 2010 Free Software Foundation, Inc.
+#
+# GNU Mailutils is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3, or (at
+# your option) any later version.
+#
+# GNU Mailutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>.
+
+# Check for "UNIX mbox 1st message symptom" bug.  See the description in
+# ufms.c, for a detailed description.
+
+AT_SETUP([UNIX mbox 1st message symptom])
+AT_KEYWORDS([mailbox mbox mbox-ufms ufms])
+AT_CHECK([
+MUT_MBCOPY($abs_top_srcdir/testsuite/spool/mbox1)
+ufms mbox:mbox1 || exit $?
+sed 's/^\(X-IMAPbase\): \([[0-9][0-9]*]\) \(.*\)/\1: UIDVALIDITY \3/' mbox1
+],
+[0],
+[From foobar@nonexistent.net  Fri Dec 28 22:18:09 2001
+Received: (from foobar@nonexistent.net)
+	by nonexistent.net id fBSKI8N04906
+	for bar@dontmailme.org; Fri, 28 Dec 2001 22:18:08 +0200
+Date: Fri, 28 Dec 2001 22:18:08 +0200
+From: Foo Bar <foobar@nonexistent.net>
+Message-Id: <200112282018.fBSKI8N04906@nonexistent.net>
+To: Bar <bar@dontmailme.org>
+Subject: Jabberwocky
+X-IMAPbase: UIDVALIDITY 6
+Status: R
+X-UID: 1
+
+`Twas brillig, and the slithy toves
+Did gyre and gimble in the wabe;
+All mimsy were the borogoves,
+And the mome raths outgrabe.
+
+`Beware the Jabberwock, my son!
+The jaws that bite, the claws that catch!
+Beware the Jujub bird, and shun
+The frumious Bandersnatch!'
+
+He took his vorpal sword in hand:
+Long time the manxome foe he sought --
+So rested he by the Tumtum gree,
+And stood awhile in thought.
+
+And as in uffish thought he stood,
+The Jabberwock, with eyes of flame,
+Came whiffling through the tulgey wook,
+And burbled as it came!
+
+One, two!  One, two!  And through and through
+The vorpal blade went snicker-snack!
+He left it dead, and with its head
+He went galumphing back.
+
+`And has thou slain the Jabberwock?
+Come to my arms, my beamish boy!
+O frabjous day!  Calloh!  Callay!
+He chortled in his joy.
+
+`Twas brillig, and the slithy toves
+Did gyre and gimble in the wabe;
+All mimsy were the borogoves,
+And the mome raths outgrabe.
+
+
+])
+
+AT_CLEANUP
+

+/* ufms.c - a test of "UNIX mailbox first message symptom"
+   Copyright (C) 2010 Free Software Foundation, Inc.
+
+   GNU Mailutils is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GNU Mailutils is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>. */
+
+#ifdef HAVE_CONFIG_H
+# include <config.h>
+#endif
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <mailutils/mailutils.h>
+
+/* UFMS, or "UNIX mailbox first message symptom", is a bug that 
+   existed in all versions of Mailutils up to 2.2.1 and 2.9.90
+   as of 2010-11-07.
+
+   Due to improperly designed expunge routine, the body of the very
+   first message in a UNIX mailbox would get corrupted under certain
+   circumstances.  The following three lines would appear at the top
+   of the body:
+
+   -IMAPbase: X Y
+   Status: S
+   X-UID: Z
+
+   where (X, Y, Z are decimal numbers and S is a status string).
+
+   Prerequisites for triggering the bug:
+
+   1. Mailbox contains more than one message.
+   2. The first message does not carry X-IMAPbase and X-UID headers.
+
+   Recipe for triggering the bug:
+
+   1. Delete some messages.
+   2. Expunge the mailbox.
+   3. Modify attributes of the first message.
+   4. Flush your changes.
+*/   
+   
+
+int
+main (int argc, char **argv)
+{
+  mu_mailbox_t mbox;
+  size_t i, count;
+  mu_message_t msg;
+  mu_attribute_t attr;
+  
+  if (argc != 2)
+    {
+      fprintf (stderr, "usage: %s MBOX\n", argv[0]);
+      return 1;
+    }
+
+  mu_registrar_record (mu_mbox_record);
+
+  /* Open the mailbox */
+  MU_ASSERT (mu_mailbox_create (&mbox, argv[1]));
+  MU_ASSERT (mu_mailbox_open (mbox, MU_STREAM_RDWR));
+  mu_mailbox_messages_count (mbox, &count);
+  if (count < 2)
+    {
+      fprintf (stderr,
+	       "%s: input mailbox should contain at least two messages\n",
+	       argv[0]);
+      exit (1);
+    }
+  /* Delete all records except the first and expunge the mailbox. */
+  for (i = 2; i <= count; i++)
+    {
+      MU_ASSERT (mu_mailbox_get_message (mbox, i, &msg));
+      MU_ASSERT (mu_message_get_attribute (msg, &attr));
+      MU_ASSERT (mu_attribute_set_deleted (attr));
+    }
+  mu_mailbox_expunge (mbox);
+
+  /* Modify attributes on the remaining message and synch */
+  MU_ASSERT (mu_mailbox_get_message (mbox, 1, &msg));
+  MU_ASSERT (mu_message_get_attribute (msg, &attr));
+  MU_ASSERT (mu_attribute_set_read (attr));
+  mu_mailbox_sync (mbox);
+
+  /* Cleanup & exit */
+  mu_mailbox_close (mbox);
+  mu_mailbox_destroy (&mbox);
+  return 0;
+}