Implemented mbox_append () first draft. Things are still very unstable.

+2001-09-26 Alain Magloire
+
+	* mailbox2/mbox/mbox_append.c: Implemented, first draft.
+
 2001-09-23 Alain Magloire
 
 	First draft of the mailbox/mbox2.  Not functional.

@@ -172,6 +172,7 @@ checkit
   format = va_arg (args, char *);
 #endif
   vasprintf (&result, format, args);
+  va_end (args);
   if (strlen (result) < global_total_width)
     printf ("PASS: ");
   else

@@ -70,16 +70,21 @@ extern int  mbox_unmark_deleted  __P ((mbox_t, unsigned int));
 extern int  mbox_expunge         __P ((mbox_t, int));
 extern int  mbox_changed_on_disk __P ((mbox_t));
 
-extern int  mbox_set_progress_cb __P ((mbox_t, int (*) __P ((int, void *)), void *));
-extern int  mbox_set_newmsg_cb __P ((mbox_t, int (*) __P ((int, void *)), void *));
+extern int  mbox_set_progress_cb
+__P ((mbox_t, int (*) __P ((int, void *)), void *));
+extern int  mbox_set_newmsg_cb
+__P ((mbox_t, int (*) __P ((int, void *)), void *));
 extern int  mbox_newmsg_cb     __P ((mbox_t, int));
 extern int  mbox_progress_cb   __P ((mbox_t, int));
 
 extern int  mbox_scan __P ((mbox_t, unsigned int, unsigned int *, int));
 extern int  mbox_messages_count __P ((mbox_t, unsigned int *));
 
-extern int  mbox_append         __P ((mbox_t, const char *, stream_t));
-extern int  mbox_append_hb __P ((mbox_t, const char *, stream_t, stream_t));
+extern int  mbox_append __P ((mbox_t, const char *, attribute_t, stream_t));
+extern int  mbox_append_hb
+__P ((mbox_t, const char *, attribute_t, stream_t, stream_t));
+extern int  mbox_append_hb0
+__P ((mbox_t, const char *, attribute_t, int, stream_t, stream_t));
 
 extern int  mbox_get_carrier __P ((mbox_t, stream_t *));
 extern int  mbox_set_carrier __P ((mbox_t, stream_t));
@@ -95,6 +100,8 @@ extern int  mbox_header_get_value __P ((mbox_t, unsigned int, const char *,
 
 extern int  stream_mbox_create __P ((stream_t *, mbox_t, unsigned int, int));
 extern int  attribute_mbox_create __P ((attribute_t *, mbox_t, unsigned int));
+extern int  mbox_attribute_to_status
+__P ((attribute_t, char *, size_t, size_t *));
 
 #ifdef __cplusplus
 }

@@ -49,9 +49,8 @@ struct _mbox
   stream_t carrier; /* File stream.  */
 
   off_t size; /* Size of the mailbox.  */
-  time_t mtime; /* Modified time.  */
   unsigned long uidvalidity;
-  size_t uidnext;
+  unsigned long uidnext;
   char *filename;
 
   struct _hcache hcache;
@@ -59,10 +58,11 @@ struct _mbox
   /* The variables below are use to hold the state when appending messages.  */
   enum mbox_state
   {
-    MBOX_NO_STATE = 0,
-    MBOX_STATE_APPEND_HEADER,
-    MBOX_STATE_APPEND_BODY
-  } state ;
+    MU_MBOX_NO_STATE = 0,
+    MU_MBOX_STATE_APPEND_SEPARATOR,
+    MU_MBOX_STATE_APPEND_HEADER,
+    MU_MBOX_STATE_APPEND_BODY
+  } state;
 
   lockfile_t  lockfile;
   struct

@@ -20,36 +20,332 @@
 #endif
 
 #include <stdlib.h>
+#include <string.h>
 
 #include <mailutils/error.h>
 #include <mailutils/sys/mbox.h>
 
-/* Save the uidvalidity
-   - if it is an empty mbox in the first message append
-   - if for the first message the uidvalidity is not the same
+#define IS_X_IMAPBASE(buf) (\
+(buf[0] == 'X' || buf[0] == 'x') \
+ && (buf[1] == '-') \
+ && (buf[2] == 'I' || buf[2] == 'i') \
+ && (buf[3] == 'M' || buf[3] == 'm') \
+ && (buf[4] == 'A' || buf[4] == 'a') \
+ && (buf[5] == 'P' || buf[5] == 'p') \
+ && (buf[6] == 'B' || buf[6] == 'b') \
+ && (buf[7] == 'A' || buf[7] == 'a') \
+ && (buf[8] == 'S' || buf[8] == 's') \
+ && (buf[9] == 'E' || buf[9] == 'e') \
+ && (buf[10] == ':' || buf[10] == ' ' || buf[10] == '\t'))
+
+#define IS_X_UID(buf) (\
+(buf[0] == 'X' || buf[0] == 'x') \
+ && (buf[1] == '-') \
+ && (buf[2] == 'U' || buf[2] == 'u') \
+ && (buf[3] == 'I' || buf[3] == 'i') \
+ && (buf[4] == 'D' || buf[4] == 'd') \
+ && (buf[5] == ':' || buf[5] == ' ' || buf[5] == '\t'))
+
+#define IS_CONTENT_LENGTH(buf) (\
+(buf[0] == 'C' || buf[0] == 'c') && \
+(buf[1] == 'O' || buf[1] == 'o') && \
+(buf[2] == 'N' || buf[2] == 'n') && \
+(buf[3] == 'T' || buf[3] == 't') && \
+(buf[4] == 'E' || buf[4] == 'e') && \
+(buf[5] == 'N' || buf[5] == 'n') && \
+(buf[6] == 'T' || buf[6] == 't') && \
+(buf[7] == '-') && \
+(buf[8] == 'L' || buf[8] == 'l') && \
+(buf[9] == 'E' || buf[9] == 'e') && \
+(buf[10] == 'N' || buf[10] == 'n') && \
+(buf[11] == 'G' || buf[11] == 'g') && \
+(buf[12] == 'T' || buf[12] == 't') && \
+(buf[13] == 'H' || buf[13] == 'h') && \
+(buf[14] == ':' || buf[14] == ' ' || buf[14] == '\t'))
+
+#define IS_STATUS(buf) (\
+(buf[0] == 'S' || buf[0] == 's') && \
+(buf[1] == 'T' || buf[1] == 't') && \
+(buf[2] == 'A' || buf[2] == 'a') && \
+(buf[3] == 'T' || buf[3] == 't') && \
+(buf[4] == 'U' || buf[4] == 'u') && \
+(buf[5] == 'S' || buf[5] == 's') && \
+(buf[6] == ':' || buf[6] == ' ' || buf[6] == '\t'))
+
+#define IS_FROM_(buf) (\
+(buf[0] == 'F' || buf[0] == 'f') && \
+(buf[1] == 'R' || buf[1] == 'r') && \
+(buf[2] == 'O' || buf[2] == 'o') && \
+(buf[3] == 'M' || buf[3] == 'm') && \
+(buf[4] == ' ' || buf[4] == '\t'))
+
+/* Save the uidvalidity:
+   + if it is an empty mbox in the first message append
+   + if for the first message the uidvalidity is not the same
    from the mbox->uidvalidity.
 
    - strip X-IMAPBASE, X-UID
    - add X-UID base on mbox->uidnext.
 
    - mangle any leading "From " in the body to ">From "
+
+   - update the size of the mailbox.
+
+   - Refuse to append if the mailbox is change on disk.
 */
 
-int
-mbox_append (mbox_t mbox, const char *sep, stream_t stream)
+/* Assuming that the file is lock.  */
+static int
+mbox_append_separator (mbox_t mbox, const char *sep)
 {
-  if (mbox == NULL || stream == NULL)
-    return MU_ERROR_INVALID_PARAMETER;
-  (void)sep;
-  return 0;
+  char separator[256];
+  size_t len;
+
+  if (sep == NULL)
+    {
+      time_t now;
+      struct tm *ptm;
+
+      now = time (NULL);
+      ptm = gmtime (&now);
+      len = strftime (separator, sizeof separator,
+		      "From unknown %a %b %d %H:%M:%S %Y\n", ptm);
+      if (len == 0)
+	{
+	  len = snprintf (separator, sizeof separator,
+			  "From unknown %s", ctime (&now));
+	}
+      sep = separator;
+    }
+  else
+    len = strlen (sep);
+
+  /* Write the separator.  */
+  return stream_write (mbox->carrier, sep, len, NULL);
+}
+
+/* Assuming that the file is lock.  */
+static int
+mbox_append_header (mbox_t mbox, attribute_t attribute, int save_uidvalidity,
+		    stream_t hstream)
+{
+  char buffer[1024];
+  size_t nread = 0;
+  int status = 0;
+  const char nl = '\n';
+
+  do
+    {
+      status = stream_readline (hstream, buffer, sizeof buffer, &nread);
+      if (status != 0)
+	return status;
+
+      /* A newline means the start of the body.  */
+      if (*buffer == '\n')
+	break;
+
+      if (IS_X_IMAPBASE (buffer))
+	{
+	  /* Skip the X-IMAPBase it has special meaning for us.  */
+	  continue;
+	}
+      else if (IS_X_UID (buffer))
+	{
+	  /* Skip the X-UID. A new one will be provided.  */
+	  continue;
+	}
+      else if (IS_STATUS (buffer))
+	{
+	  /* Skip, use the attribute.  */
+	  continue;
+	}
+      else if (IS_CONTENT_LENGTH (buffer))
+	{
+	  /* Ignore this, too often bad.  */
+	  continue;
+	}
+
+      status = stream_write (mbox->carrier, buffer, nread, NULL);
+      if (status != 0)
+	return status;
+    }
+  while (nread > 0);
+
+  /* Rewrite the X-IMAPbase marker If necesary. */
+  if (mbox->uidnext < 2 && save_uidvalidity)
+    {
+      nread = snprintf (buffer, sizeof buffer, "X-IMAPbase: %lu %lu\n",
+			mbox->uidvalidity, mbox->uidnext);
+      status = stream_write (mbox->carrier, buffer, nread, NULL);
+      if (status != 0)
+	return status;
+    }
+
+  /* Rewrite the  Status for the attribute.  */
+  if (attribute)
+    {
+      mbox_attribute_to_status (attribute, buffer, sizeof buffer, &nread);
+      status = stream_write (mbox->carrier, buffer, nread, NULL);
+      if (status != 0)
+	return status;
+    }
+
+  /* Rewrite the X-UID marker . */
+  nread = snprintf (buffer, sizeof buffer, "X-UID: %lu\n", mbox->uidnext);
+  status = stream_write (mbox->carrier, buffer, nread, NULL);
+  if (status != 0)
+    return status;
+
+  /* New line separator of the Header.  */
+  return stream_write (mbox->carrier, &nl , 1, NULL);
+}
+
+/* Assuming that the file is lock.  */
+static int
+mbox_append_body (mbox_t mbox, stream_t bstream)
+{
+  char buffer[1024];
+  char *buf;
+  int was_complete_line;
+  size_t nread = 0;
+  const char nl = '\n';
+  int status;
+
+  /* For "From " mangling.  */
+  *buffer = '>';
+  was_complete_line = 1; /* Say we start as complete line.  */
+  do
+    {
+      buf = buffer + 1;
+      status = stream_readline (bstream, buf, sizeof (buffer) - 1, &nread);
+      if (status != 0)
+	return status;
+
+      /* Unix Mbox:
+	 Since it's possibpe for a message to contain lines that looks
+	 like message separators, special care must be taken when adding
+	 a message to an mbox folder this is done by prepending a '>'
+	 character to any lines starting with zero or more '>' characters
+	 folowed by "From "
+	 p436 "Internet Email Protocols  a Debeloper's Guid"
+	 Kevin Johnson.
+      */
+      if (was_complete_line)
+	{
+	  char *s = buf;
+	  /* Eat all the '>'.  */
+	  while (*s == '>')
+	    s++;
+
+	  if (IS_FROM_ (s))
+	    {
+	      buf = buffer;
+	      nread++;
+	    }
+	}
+      status = stream_write (mbox->carrier, buf, nread, NULL);
+      if (status != 0)
+	return status ;
+
+      /* Register if we read a complete line.  */
+      was_complete_line =  (nread && buf[nread - 1] == '\n') ? 1 : 0;
+    }
+  while (nread > 0);
+
+  /* New line separator for the next message.  */
+  return stream_write (mbox->carrier, &nl, 1, NULL);
 }
 
 int
-mbox_append_hb (mbox_t mbox, const char *sep, stream_t hstream,
-		stream_t bstream)
+mbox_append_hb0 (mbox_t mbox, const char *sep, attribute_t attribute,
+		 int save_uidvalidity, stream_t hstream, stream_t bstream)
 {
+  int status = 0;
+
   if (mbox == NULL || hstream == NULL || bstream == NULL)
     return MU_ERROR_INVALID_PARAMETER;
-  (void)sep;
-  return 0;
+
+  switch (mbox->state)
+    {
+    case MU_MBOX_NO_STATE:
+      {
+	off_t size = 0;
+	unsigned long uidvalidity;
+	unsigned long uidnext;
+
+	/* Get the uidvalidity for this mbox.  */
+	mbox_get_uidvalidity (mbox, &uidvalidity);
+	mbox_get_uidnext (mbox, &uidnext);
+
+	/* Grab the lock.  */
+	status = lockfile_lock (mbox->lockfile);
+	if (status != 0)
+	  break;
+
+	/* Move to the end of the stream.  */
+	if ((status = stream_get_size (mbox->carrier, &size)) != 0
+	    || (status = stream_seek (mbox->carrier, size,
+				      MU_STREAM_WHENCE_SET) != 0))
+	  break;
+	mbox->state = MU_MBOX_STATE_APPEND_SEPARATOR;
+      }
+
+    case MU_MBOX_STATE_APPEND_SEPARATOR:
+      {
+	status = mbox_append_separator (mbox, sep);
+	if (status != 0)
+	  break;
+	mbox->state = MU_MBOX_STATE_APPEND_HEADER;
+      }
+
+    case MU_MBOX_STATE_APPEND_HEADER:
+      {
+	status = mbox_append_header (mbox, attribute, save_uidvalidity, hstream);
+	if (status != 0)
+	  break;
+        mbox->state = MU_MBOX_STATE_APPEND_BODY;
+      }
+
+    case MU_MBOX_STATE_APPEND_BODY:
+      {
+	status = mbox_append_body (mbox, bstream);
+	if (status != 0)
+	  break;
+        mbox->state = MU_MBOX_NO_STATE;
+      }
+
+    default:
+      break;
+    }
+
+  /* Maintain the lock if EAGAIN.  */
+  if (status != 0)
+    {
+      if (status != MU_ERROR_TRY_AGAIN)
+	{
+	  mbox->state = MU_MBOX_NO_STATE;
+	  lockfile_unlock (mbox->lockfile);
+	}
+    }
+  else
+    {
+      lockfile_unlock (mbox->lockfile);
+      mbox->uidnext++;
+    }
+
+  return status;
+}
+
+int
+mbox_append (mbox_t mbox, const char *sep, attribute_t attribute,
+	     stream_t stream)
+{
+  return mbox_append_hb (mbox, sep, attribute, stream, stream);
+}
+
+int
+mbox_append_hb (mbox_t mbox, const char *sep, attribute_t attribute,
+		stream_t hstream, stream_t bstream)
+{
+  return mbox_append_hb0 (mbox, sep, attribute, 1, hstream, bstream);
 }

@@ -20,6 +20,7 @@
 #endif
 
 #include <stdlib.h>
+#include <string.h>
 
 #include <mailutils/error.h>
 #include <mailutils/refcount.h>
@@ -197,3 +198,49 @@ mbox_get_attribute (mbox_t mbox, unsigned int msgno, attribute_t *pattribute)
     }
   return status;
 }
+
+int
+mbox_attribute_to_status (attribute_t attribute, char *buf, size_t buflen,
+			  size_t *pn)
+{
+  char Status[32];
+  char a[8];
+  size_t i;
+
+  *Status = *a = '\0';
+
+  if (attribute)
+    {
+      if (attribute_is_read (attribute))
+	strcat (a, "R");
+      if (attribute_is_seen (attribute))
+	strcat (a, "O");
+      if (attribute_is_answered (attribute))
+	strcat (a, "A");
+      if (attribute_is_deleted (attribute))
+	strcat (a, "d");
+      if (attribute_is_flagged (attribute))
+	strcat (a, "F");
+    }
+
+  if (*a != '\0')
+    {
+      strcpy (Status, "Status: ");
+      strcat (Status, a);
+      strcat (Status, "\n");
+    }
+
+  if (buf && buflen)
+    {
+      *buf = '\0';
+      strncpy (buf, Status, buflen - 1);
+      buf[buflen - 1] = '\0';
+      i = strlen (buf);
+    }
+  else
+    i = strlen (Status);
+
+  if (pn)
+    *pn = i;
+  return 0;
+}

@@ -28,8 +28,10 @@ int
 mbox_changed_on_disk (mbox_t mbox)
 {
   int changed = 0;
-  /* Check if the mtime stamp changed, random modifications can give
-     us back the same size.  */
+
+  /* If the modification time is greater then the access time, the file has
+     been modified since the last time it was accessed.  This typically means
+     new mail or someone tempered with the mailbox.  */
   if (mbox->carrier)
     {
       int fd = -1;
@@ -38,7 +40,7 @@ mbox_changed_on_disk (mbox_t mbox)
 	  struct stat statbuf;
 	  if (fstat (fd, &statbuf) == 0)
 	    {
-	      if (difftime (mbox->mtime, statbuf.st_mtime) != 0)
+	      if (difftime (statbuf.st_mtime, statbuf.st_atime) > 0)
 		changed = 1;
 	    }
 	}

@@ -65,7 +65,6 @@ mbox_close (mbox_t mbox)
   mbox->umessages = NULL;
   mbox->umessages_count = 0;
   mbox->size = 0;
-  mbox->mtime = 0;
   mbox->uidvalidity = 0;
   mbox->uidnext = 0;
   if (mbox->filename)

@@ -125,6 +125,7 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
   int tmp_fd;
   char *tmp_name = NULL;
   mbox_t tmp_mbox = NULL;
+  size_t save_uidvalidity = 0; /* uidvalidity is save in the first message.  */
 
   if (mbox == NULL)
     return MU_ERROR_INVALID_PARAMETER;
@@ -167,7 +168,7 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
 	return status;
       }
 
-    /* Must be flag CREATE if not the mailbox_open will try to mmap()
+    /* Must be flag CREATE if not the mailbox_open will/may try to mmap()
        the file.  */
     status = mbox_open (tmp_mbox, tmp_name, MU_STREAM_CREAT | MU_STREAM_RDWR);
     if (status != 0)
@@ -195,7 +196,19 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
       return status;
     }
 
-  /* Critical section, we can not allowed signal here.  */
+  /* _Must_ have and updated view of the size of the mailbox.  */
+  status = mbox_changed_on_disk (mbox);
+  if (status != 0)
+    {
+      mbox_close (tmp_mbox);
+      mbox_destroy (&tmp_mbox);
+      remove (tmp_name);
+      free (tmp_name);
+      /* mu_error ("Failed to grab the lock\n"); */
+      return status;
+    }
+
+  /* Critical section, can not allowed signal here.  */
   sigemptyset (&signalset);
   sigaddset (&signalset, SIGTERM);
   sigaddset (&signalset, SIGHUP);
@@ -216,16 +229,24 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
       /* Skip it, if mark for deletion.  */
       if (remove_deleted && ATTRIBUTE_IS_DELETED (mum->attr_flags))
 	{
+	  /* We save the uidvalidity in the first message, if it is being
+             deleted we need to move the uidvalidity to the first available
+             (non-deleted) message.  */
+          if (i == save_uidvalidity)
+            {
+              save_uidvalidity = i + 1;
+            }
 	  continue;
 	}
-
+      else
 	{
 	  stream_t hstream = NULL, bstream = NULL;
+	  attribute_t attribute = NULL;
 	  char *sep = NULL;
 	  /* The message was not instanciated, probably the dirty flag was
 	     set by mbox_scan(), create one here.  */
 	  if (mum->separator == NULL || mum->header.stream == NULL
-	    || mum->body.stream == NULL)
+	      || mum->body.stream == NULL || mum->attribute == NULL)
 	    {
 	      if (mbox_get_hstream (mbox, i + 1, &hstream) != 0
 		  || mbox_get_bstream (mbox, i + 1, &bstream) != 0
@@ -235,12 +256,14 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
 		  goto bailout0;
 		}
 	    }
-	status = mbox_append_hb (tmp_mbox, mum->separator, mum->header.stream,
+	  status = mbox_append_hb0 (tmp_mbox, mum->separator, mum->attribute,
+				    save_uidvalidity, mum->header.stream,
 				    mum->body.stream);
 	  if (sep)
 	    free (sep);
 	  stream_destroy (&hstream);
 	  stream_destroy (&bstream);
+	  attribute_destroy (&attribute);
 	  if (status != 0)
 	    {
 	      /* mu_error ("Error expunge:%d: %s", __LINE__,
@@ -266,7 +289,7 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
 	if (len > 0)
 	  {
 	    stream_seek (mbox->carrier, mbox->size, MU_STREAM_WHENCE_SET);
-	    stream_seek (mbox->carrier, tmp_mbox->size, MU_STREAM_WHENCE_SET);
+	    stream_seek (tmp_mbox->carrier, tmp_mbox->size, MU_STREAM_WHENCE_SET);
 	    while ((status = stream_read (mbox->carrier, buffer,
 					  sizeof buffer, &n)) == 0 && n > 0)
 	      {
@@ -345,11 +368,22 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
       size_t dlast;
       for (j = dirty, dlast = mbox->messages_count - 1; j <= dlast; j++)
 	{
-	  /* Clear all the references, any attach messages been already
-	     destroy above.  */
+	  /* Clear all the references to streams.  */
 	  mum = mbox->umessages[j];
 	  if (remove_deleted && ATTRIBUTE_IS_DELETED (mum->attr_flags))
 	    {
+	      if (mum->separator)
+		free (mum->separator);
+	      mum->separator = NULL;
+	      if (mum->attribute)
+		attribute_destroy (&mum->attribute);
+	      if (mum->header.stream)
+		stream_destroy (&mum->header.stream);
+	      if (mum->body.stream)
+		stream_destroy (&mum->body.stream);
+	      mbox_hcache_free (mbox, i + 1);
+	      /* memset (mum, 0, sizeof (*mum)); */
+
 	      if ((j + 1) <= dlast)
 		{
 		  /* Move all the pointers up.  So the message pointer
@@ -361,8 +395,6 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
 		  mum->body.start = mum->body.end = 0;
 		  mum->header.lines = mum->body.lines = 0;
 #endif
-		  mbox_hcache_free (mbox, i + 1);
-		  memset (mum, 0, sizeof (*mum));
 		  /* We are not free()ing the useless mum, but instead
 		     we put it back in the pool, to be reuse.  */
 		  mbox->umessages[dlast] = mum;
@@ -371,16 +403,10 @@ mbox_expunge (mbox_t mbox, int remove_deleted)
 		     gets cleared to.  */
 		  mum = mbox->umessages[j];
 		}
-	      else
-		{
-		  mbox_hcache_free (mbox, i + 1);
-		  memset (mum, 0, sizeof (*mum));
-		}
 	    }
 	  mum->from_ = mum->header.start = 0;
 	  mum->body.start = mum->body.end = 0;
 	  mum->header.lines = mum->body.lines = 0;
-	  mbox_hcache_free (mbox, i + 1);
 	}
       /* This is should reset the messages_count, the last argument 0 means
 	 not to send event notification.  */

@@ -150,7 +150,6 @@ mbox_scan (mbox_t mbox, unsigned int msgno, unsigned int *pcount, int do_notif)
 	     reality if expunge.  */
 	  /*	  mbox->size = statbuf.st_size; */
 	  file_size = statbuf.st_size;
-	  mbox->mtime = statbuf.st_mtime;
 	}
       else
 	status = MU_ERROR_IO;