Implement the mu_mailbox_get_uidls function and --uidl option in movemail.

* include/mailutils/mailbox.h (MU_UIDL_LENGH)
(MU_UIDL_BUFFER_SIZE): New defines.
(struct mu_uidl): New struct.
(mu_mailbox_get_uidls): New prototype.
* libproto/include/mailbox0.h (struct _mu_mailbox): New member
_get_uidls.
* libproto/pop/mbox.c: Implement _get_uidls.
* libproto/pop/folder.c: Include mailutils/cctype.h.

* mailbox/mailbox.c (mu_mailbox_get_uidls): New function.

* movemail/movemail.c: Implement --uidl option: use UIDLs to avoid
downloading same messages twice.  Based on the idea of Alfred M. Szmidt.
Implement --verbose option.

* libproto/imap/mbox.c: Fix comment.

* NEWS: Update.
* doc/texinfo/programs.texi: Update.

-GNU mailutils NEWS -- history of user-visible changes. 2009-05-25
+GNU mailutils NEWS -- history of user-visible changes. 2009-07-08
 Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 
 Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 
 2008, 2009 Free Software Foundation, Inc.
 2008, 2009 Free Software Foundation, Inc.
 See the end of file for copying conditions.
 See the end of file for copying conditions.
@@ -18,6 +18,15 @@ All MU client utilities make use of the user ticket file,
 New configuration file entities allow to modify user's personal
 New configuration file entities allow to modify user's personal
 namespace and visible home directory.
 namespace and visible home directory.
 
 
+* Movemail
+
+When called with the `--uidl' command line option, the utility tries
+to avoid copying the message if a message with the same UIDL already
+exists in the destination mailbox.
+
+The `--verbose' command line option enables outputting additional
+information.
+
 * API
 * API
 
 
 * Wicket/Ticket functions
 * Wicket/Ticket functions
@@ -50,6 +59,10 @@ The `remote+sendmail', `remote+prog' and `remote+smtp' mailbox formats
 are deprecated in favor of `sendmail', `prog' and `smtp',
 are deprecated in favor of `sendmail', `prog' and `smtp',
 correspondingly.  
 correspondingly.  
 
 
+** New functions
+
+- mu_mailbox_get_uidls
+
 ** Removed functions
 ** Removed functions
 
 
 - mu_sieve_get_ticket
 - mu_sieve_get_ticket

 @c This is part of the GNU Mailutils manual.
 @c This is part of the GNU Mailutils manual.
-@c Copyright (C) 1999,2000,2001,2002,2003,2004,2005,2006,2007,2008
+@c Copyright (C) 1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009
 @c Free Software Foundation, Inc.
 @c Free Software Foundation, Inc.
 @c See file mailutils.texi for copying conditions.
 @c See file mailutils.texi for copying conditions.
 @comment *******************************************************************
 @comment *******************************************************************
@@ -4034,6 +4034,15 @@ If @var{bool} is @samp{true}, reverse message sorting order.
 If @var{bool} is @samp{true}, output information used by Emacs rmail interface.
 If @var{bool} is @samp{true}, output information used by Emacs rmail interface.
 @end deffn
 @end deffn
 
 
+@deffn {Movemail Config} uidl @var{bool}
+Avoid copying the message if a message with the same UIDL already
+exists in the destination mailbox.
+@end deffn
+
+@deffn {Movemail Config} verbose @var{level}
+Set verbosity level.
+@end deffn
+
 @multitable @columnfractions 0.3 0.6
 @multitable @columnfractions 0.3 0.6
 @headitem Statement @tab Reference
 @headitem Statement @tab Reference
 @item debug         @tab @xref{Debug Statement}.
 @item debug         @tab @xref{Debug Statement}.
@@ -4147,9 +4156,6 @@ Preserve the source mailbox
 @itemx --reverse
 @itemx --reverse
 Reverse the sorting order
 Reverse the sorting order
 
 
-@item --license
-Print GPL license and exit
-
 @item --external-locker=@var{program}
 @item --external-locker=@var{program}
 Use given @var{program} as the external locker program.
 Use given @var{program} as the external locker program.
 
 
@@ -4177,6 +4183,13 @@ Use specified URL as a mailspool directory
 @item --tls[=@var{bool}]
 @item --tls[=@var{bool}]
 Enable (default) or disable TLS support
 Enable (default) or disable TLS support
 
 
+@item -u
+@item --uidl
+Use UIDLs to avoid downloading the same message twice.
+
+@item -v
+@item --verbose
+Increase verbosity level.
 @end table
 @end table
 
 
 @page
 @page

@@ -66,6 +66,16 @@ extern int  mu_mailbox_sync            (mu_mailbox_t);
 extern int  mu_mailbox_save_attributes (mu_mailbox_t)
 extern int  mu_mailbox_save_attributes (mu_mailbox_t)
                                        __attribute__ ((deprecated));
                                        __attribute__ ((deprecated));
 
 
+#define MU_UIDL_LENGTH 70
+#define MU_UIDL_BUFFER_SIZE (MU_UIDL_LENGTH+1)
+				       
+struct mu_uidl
+{
+  size_t msgno;
+  char uidl[MU_UIDL_BUFFER_SIZE];
+};
+extern int  mu_mailbox_get_uidls       (mu_mailbox_t, mu_list_t *);
+
 /* Update and scanning.  */
 /* Update and scanning.  */
 extern int  mu_mailbox_get_size        (mu_mailbox_t, mu_off_t *size);
 extern int  mu_mailbox_get_size        (mu_mailbox_t, mu_off_t *size);
 extern int  mu_mailbox_is_updated      (mu_mailbox_t);
 extern int  mu_mailbox_is_updated      (mu_mailbox_t);

@@ -719,14 +719,14 @@ imap_scan (mu_mailbox_t mailbox, size_t msgno, size_t *pcount)
   return imap_scan0 (mailbox, msgno, pcount , 1);
   return imap_scan0 (mailbox, msgno, pcount , 1);
 }
 }
 
 
-/* Usually when this function is call it is because there is an oberver
-   attach an the client is try to build some sort of list/tree header
-   as the scanning progress.  But doing this for each message can be
-   time consuming and inefficient.  So we bundle all the request
-   into one and ask the server for everything "FETCH 1:*".  The good
-   side is that everything will be faster and we do not do lot of small
-   transcation but rather a big one.  The bad thing is that every thing
-   will be cache in the structure using a lot of memory.  */
+/* Normally this function is called when an observer is trying to build
+   some sort of list/tree header as the scanning progresses.  But doing
+   this for each message can be time consuming and inefficient.  So we
+   bundle all requests into one and ask the server for everything:
+   "FETCH 1:*".  The good side is that everything will be faster and we
+   do not do lots of small transcations, but rather a big one.  The bad
+   side is that everything will be cached in the structure using a lot of
+   memory.  */
 static int
 static int
 imap_scan0 (mu_mailbox_t mailbox, size_t msgno, size_t *pcount, int notif)
 imap_scan0 (mu_mailbox_t mailbox, size_t msgno, size_t *pcount, int notif)
 {
 {

@@ -74,6 +74,7 @@ struct _mu_mailbox
   int  (*_get_size)        (mu_mailbox_t, mu_off_t *);
   int  (*_get_size)        (mu_mailbox_t, mu_off_t *);
 
 
   int  (*_quick_get_message) (mu_mailbox_t, mu_message_qid_t, mu_message_t *);
   int  (*_quick_get_message) (mu_mailbox_t, mu_message_qid_t, mu_message_t *);
+  int  (*_get_uidls) (mu_mailbox_t, mu_list_t);
 };
 };
 
 
 #ifdef __cplusplus
 #ifdef __cplusplus

@@ -34,7 +34,8 @@
 #include <mailutils/auth.h>
 #include <mailutils/auth.h>
 #include <mailutils/errno.h>
 #include <mailutils/errno.h>
 #include <mailutils/mailbox.h>
 #include <mailutils/mailbox.h>
-#include <mailutils/mutil.h>
+#include <mailutils/cstr.h>
+#include <mailutils/cctype.h>
 
 
 #include <folder0.h>
 #include <folder0.h>
 #include <registrar0.h>
 #include <registrar0.h>

@@ -55,6 +55,7 @@
 #include <mailutils/md5.h>
 #include <mailutils/md5.h>
 #include <mailutils/io.h>
 #include <mailutils/io.h>
 #include <mailutils/mutil.h>
 #include <mailutils/mutil.h>
+#include <mailutils/cstr.h>
 #include <mailutils/cctype.h>
 #include <mailutils/cctype.h>
 
 
 #include <folder0.h>
 #include <folder0.h>
@@ -417,6 +418,57 @@ pop_destroy (mu_mailbox_t mbox)
     }
     }
 }
 }
 
 
+static int
+pop_mbox_uidls (mu_mailbox_t mbox, mu_list_t list)
+{
+  pop_data_t mpd = mbox->data;
+  int status;
+
+  status = pop_writeline (mpd, "UIDL\r\n");
+  CHECK_ERROR (mpd, status);
+  MU_DEBUG (mbox->debug, MU_DEBUG_PROT, mpd->buffer);
+
+  status = pop_write (mpd);
+  CHECK_EAGAIN (mpd, status);
+
+  status = pop_read_ack (mpd);
+  CHECK_EAGAIN (mpd, status);
+  MU_DEBUG (mpd->mbox->debug, MU_DEBUG_PROT, mpd->buffer);
+
+  if (!mu_c_strncasecmp (mpd->buffer, "+OK", 3))
+    {
+      do
+	{
+	  char *p;
+	  size_t num;
+	  struct mu_uidl *uidl;
+	  
+	  status = pop_read_ack (mpd);
+	  MU_DEBUG (mpd->mbox->debug, MU_DEBUG_PROT, mpd->buffer);
+
+	  num = strtoul (mpd->buffer, &p, 10);
+	  if (*p == 0 || !mu_isblank (*p))
+	    continue; /* FIXME: or error? */
+	  p = mu_str_skip_class (p, MU_CTYPE_SPACE);
+	  mu_rtrim_cset (p, "\r\n");
+
+	  uidl = malloc (sizeof (uidl[0]));
+	  if (!uidl)
+	    {
+	      status = ENOMEM;
+	      break;
+	    }
+	  uidl->msgno = num;
+	  strncpy (uidl->uidl, p, MU_UIDL_BUFFER_SIZE);
+	  status = mu_list_append (list, uidl);
+	}
+      while (mpd->nl);
+    }
+  else
+    status = ENOSYS;
+  return status;
+}
+
 /*
 /*
   POP3 CAPA support.
   POP3 CAPA support.
  */
  */
@@ -454,12 +506,16 @@ pop_parse_capa (pop_data_t mpd)
 	    mpd->capa |= CAPA_STLS;
 	    mpd->capa |= CAPA_STLS;
 	}
 	}
       while (mpd->nl);
       while (mpd->nl);
+
+      if (mpd->capa & CAPA_UIDL)
+	mpd->mbox->_get_uidls = pop_mbox_uidls;
+  
       return status;
       return status;
     }
     }
   else
   else
     {
     {
       /* mu_error ("CAPA not implemented"); */ /* FIXME */
       /* mu_error ("CAPA not implemented"); */ /* FIXME */
-      return -1;
+      return ENOSYS;
     }
     }
 }
 }
 
 
@@ -485,7 +541,6 @@ pop_capa (mu_mailbox_t mbox)
   return pop_parse_capa (mpd);
   return pop_parse_capa (mpd);
 }
 }
 
 
-
 /* Simple User/pass authentication for pop. We ask for the info
 /* Simple User/pass authentication for pop. We ask for the info
    from the standard input.  */
    from the standard input.  */
 int
 int
@@ -1427,7 +1482,7 @@ pop_message_size (mu_message_t msg, size_t *psize)
   CLEAR_STATE (mpd);
   CLEAR_STATE (mpd);
 
 
   if (status != 0)
   if (status != 0)
-    status = MU_ERR_PARSE;
+    return MU_ERR_PARSE;
 
 
   /* The size of the message is with the extra '\r' octet for everyline.
   /* The size of the message is with the extra '\r' octet for everyline.
      Substract to get, hopefully, a good count.  */
      Substract to get, hopefully, a good count.  */
@@ -1570,7 +1625,7 @@ pop_uid (mu_message_t msg,  size_t *puid)
   return 0;
   return 0;
 }
 }
 
 
-/* Get the UIDL.  Client should be prepare since it may fail.  UIDL is
+/* Get the UIDL.  The client should be prepared, since it may fail.  UIDL is
    optional on many POP servers.
    optional on many POP servers.
    FIXME:  We should check the "mpd->capa & CAPA_UIDL" and fall back to
    FIXME:  We should check the "mpd->capa & CAPA_UIDL" and fall back to
    a md5 scheme ? Or maybe check for "X-UIDL" a la Qpopper ?  */
    a md5 scheme ? Or maybe check for "X-UIDL" a la Qpopper ?  */
@@ -1582,14 +1637,12 @@ pop_uidl (mu_message_t msg, char *buffer, size_t buflen, size_t *pnwriten)
   int status = 0;
   int status = 0;
   void *func = (void *)pop_uidl;
   void *func = (void *)pop_uidl;
   size_t num;
   size_t num;
-  /* According to the RFC uidl's are no longer then 70 chars.  Still playit
-     safe  */
-  char uniq[128];
+  char uniq[MU_UIDL_BUFFER_SIZE];
 
 
   if (mpm == NULL)
   if (mpm == NULL)
     return EINVAL;
     return EINVAL;
 
 
-  /* Is it cache ?  */
+  /* Is it cached ?  */
   if (mpm->uidl)
   if (mpm->uidl)
     {
     {
       size_t len = strlen (mpm->uidl);
       size_t len = strlen (mpm->uidl);

@@ -681,3 +681,66 @@ mu_mailbox_unlock (mu_mailbox_t mbox)
   return mu_locker_unlock (lock);
   return mu_locker_unlock (lock);
 }
 }
 
 
+static void
+free_uidl (void *item)
+{
+  free (item);
+}
+
+int
+mu_mailbox_get_uidls (mu_mailbox_t mbox, mu_list_t *plist)
+{
+  mu_list_t list;
+  int status;
+
+  if (mbox == NULL)
+    return MU_ERR_MBX_NULL;
+  if (plist == NULL)
+    return EINVAL;
+  status = mu_list_create (&list);
+  if (status)
+    return status;
+  mu_list_set_destroy_item (list, free_uidl);
+  if (mbox->_get_uidls)
+    status = mbox->_get_uidls (mbox, list);
+  else
+    {
+      size_t i, total;
+
+      status = mu_mailbox_messages_count (mbox, &total);
+      if (status)
+	return status;
+      for (i = 1; i <= total; i++)
+	{
+	  mu_message_t msg = NULL;
+	  char buf[MU_UIDL_BUFFER_SIZE];
+	  size_t n;
+	  struct mu_uidl *uidl;
+	  
+	  status = mu_mailbox_get_message (mbox, i, &msg);
+	  if (status)
+	    break;
+	  status = mu_message_get_uidl (msg, buf, sizeof (buf), &n);
+	  if (status)
+	    break;
+	  uidl = malloc (sizeof (uidl[0]));
+	  if (!uidl)
+	    {
+	      status = ENOMEM;
+	      break;
+	    }
+	  uidl->msgno = i;
+	  strncpy (uidl->uidl, strdup (buf), MU_UIDL_BUFFER_SIZE);
+	  status = mu_list_append (list, uidl);
+	  if (status)
+	    {
+	      free_uidl (uidl);
+	      break;
+	    }
+	}
+    }
+  *plist = list;
+  return status;
+}
+
+	  

@@ -47,6 +47,12 @@ static struct argp_option options[] = {
   { "copy-permissions", 'P', NULL, 0,
   { "copy-permissions", 'P', NULL, 0,
     N_("Copy original mailbox permissions and ownership when applicable"),
     N_("Copy original mailbox permissions and ownership when applicable"),
     0 },
     0 },
+  { "uidl", 'u', NULL, 0,
+    N_("Use UIDLs to avoid downloading the same message twice"),
+    0 },
+  { "verbose", 'v', NULL, 0,
+    N_("Increase verbosity level"),
+    0 },
   { NULL,      0, NULL, 0, NULL, 0 }
   { NULL,      0, NULL, 0, NULL, 0 }
 };
 };
 
 
@@ -54,6 +60,8 @@ static int reverse_order;
 static int preserve_mail; 
 static int preserve_mail; 
 static int emacs_mode;
 static int emacs_mode;
 static int copy_meta;
 static int copy_meta;
+static int uidl_option;
+static int verbose_option;
 
 
 static error_t
 static error_t
 parse_opt (int key, char *arg, struct argp_state *state)
 parse_opt (int key, char *arg, struct argp_state *state)
@@ -74,6 +82,14 @@ parse_opt (int key, char *arg, struct argp_state *state)
       copy_meta = 1;
       copy_meta = 1;
       break;
       break;
 
 
+    case 'u':
+      mu_argp_node_list_new (&lst, "uidl", "yes");
+      break;
+
+    case 'v':
+      verbose_option++;
+      break;
+      
     case OPT_EMACS:
     case OPT_EMACS:
       mu_argp_node_list_new (&lst, "emacs", "yes");
       mu_argp_node_list_new (&lst, "emacs", "yes");
       break;
       break;
@@ -109,6 +125,10 @@ struct mu_cfg_param movemail_cfg_param[] = {
     N_("Reverse message sorting order.") },
     N_("Reverse message sorting order.") },
   { "emacs", mu_cfg_bool, &emacs_mode, 0, NULL,
   { "emacs", mu_cfg_bool, &emacs_mode, 0, NULL,
     N_("Output information used by Emacs rmail interface.") },
     N_("Output information used by Emacs rmail interface.") },
+  { "uidl", mu_cfg_bool, &uidl_option, 0, NULL,
+    N_("Use UIDLs to avoid downloading the same message twice.") },
+  { "verbose", mu_cfg_int, &verbose_option, 0, NULL,
+    N_("Increase verbosity level.") },
   { NULL }
   { NULL }
 };
 };
 
 
@@ -227,13 +247,13 @@ move_message (mu_mailbox_t src, mu_mailbox_t dst, size_t msgno)
 
 
   if ((rc = mu_mailbox_get_message (src, msgno, &msg)) != 0)
   if ((rc = mu_mailbox_get_message (src, msgno, &msg)) != 0)
     {
     {
-      fprintf (stderr, _("Cannot read message %lu: %s\n"),
+      mu_error (_("Cannot read message %lu: %s"),
 		(unsigned long) msgno, mu_strerror (rc));
 		(unsigned long) msgno, mu_strerror (rc));
       return rc;
       return rc;
     }
     }
   if ((rc = mu_mailbox_append_message (dst, msg)) != 0)
   if ((rc = mu_mailbox_append_message (dst, msg)) != 0)
     {
     {
-      fprintf (stderr, _("Cannot append message %lu: %s\n"),
+      mu_error (_("Cannot append message %lu: %s\n"),
 		(unsigned long) msgno, mu_strerror (rc));
 		(unsigned long) msgno, mu_strerror (rc));
       return rc;
       return rc;
     }
     }
@@ -344,6 +364,36 @@ set_permissions (mu_mailbox_t mbox)
     exit (1);
     exit (1);
 }
 }
 
 
+static int
+_compare_uidls (const void *item, const void *value)
+{
+  const struct mu_uidl *a = item;
+  const struct mu_uidl *b = value;
+
+  return strcmp (a->uidl, b->uidl);
+}
+
+static int
+_compare_msgno (const void *item, const void *value)
+{
+  const struct mu_uidl *a = item;
+  const struct mu_uidl *b = value;
+
+  if (a->msgno < b->msgno)
+    return -1;
+  if (a->msgno > b->msgno)
+    return 1;
+  return 0;
+}
+
+static int
+msgno_in_list (mu_list_t list, size_t num)
+{
+  struct mu_uidl t;
+  t.msgno = num;
+  return mu_list_locate (list, &t, NULL) == 0;
+}  
+
 int
 int
 main (int argc, char **argv)
 main (int argc, char **argv)
 {
 {
@@ -352,6 +402,8 @@ main (int argc, char **argv)
   int rc = 0;
   int rc = 0;
   char *source_name, *dest_name;
   char *source_name, *dest_name;
   int flags;
   int flags;
+  mu_list_t src_uidl_list = NULL;
+  size_t msg_count = 0;
   
   
   /* Native Language Support */
   /* Native Language Support */
   MU_APP_INIT_NLS ();
   MU_APP_INIT_NLS ();
@@ -396,18 +448,90 @@ main (int argc, char **argv)
   
   
   open_mailbox (&dest, dest_name, MU_STREAM_RDWR | MU_STREAM_CREAT, NULL);
   open_mailbox (&dest, dest_name, MU_STREAM_RDWR | MU_STREAM_CREAT, NULL);
 
 
-  mu_mailbox_messages_count (source, &total);
+  rc = mu_mailbox_messages_count (source, &total);
+  if (rc)
+    {
+      mu_error(_("Cannot count messages: %s"), mu_strerror (rc));
+      exit (1);
+    }
+  
+  if (verbose_option)
+    mu_diag_output (MU_DIAG_INFO,
+		    _("Number of messages in source mailbox: %lu"),
+		    (unsigned long) total);
+
+  if (uidl_option)
+    {
+      mu_iterator_t itr;
+      mu_list_t dst_uidl_list = NULL;
+
+      rc = mu_mailbox_get_uidls (source, &src_uidl_list);
+      if (rc)
+	die (source, _("Cannot get UIDLs"), rc);
+      rc = mu_mailbox_get_uidls (dest, &dst_uidl_list);
+      if (rc)
+	die (dest, _("Cannot get UIDLs"), rc);
+
+      mu_list_set_comparator (src_uidl_list, NULL);
+      mu_list_set_comparator (dst_uidl_list, _compare_uidls);
+      
+      mu_list_get_iterator (src_uidl_list, &itr);
+      for (mu_iterator_first (itr); !mu_iterator_is_done (itr);
+	   mu_iterator_next (itr))
+	{
+	  struct mu_uidl *uidl;
+	      
+	  mu_iterator_current (itr, (void **)&uidl);
+	  if (mu_list_locate (dst_uidl_list, uidl, NULL) == 0)
+	    mu_list_remove (src_uidl_list, uidl);
+	}
+      mu_iterator_destroy (&itr);
+      mu_list_destroy (&dst_uidl_list);
+      mu_list_set_comparator (src_uidl_list, _compare_msgno);
+    }
+  
   if (reverse_order)
   if (reverse_order)
     {
     {
-      for (i = total; rc == 0 && i > 0; i--)
+      for (i = total; i > 0; i--)
+	{
+	  if (src_uidl_list && !msgno_in_list (src_uidl_list, i))
+	    {
+	      if (verbose_option > 1)
+		mu_diag_output (MU_DIAG_INFO, _("Ignoring message %lu"),
+				(unsigned long) i);
+	      continue;
+	    }
 	  rc = move_message (source, dest, i);
 	  rc = move_message (source, dest, i);
+	  if (rc == 0)
+	    msg_count++;
+	  else
+	    break;
+	}
     }
     }
   else
   else
     {
     {
-      for (i = 1; rc == 0 && i <= total; i++)
+      for (i = 1; i <= total; i++)
+	{
+	  if (src_uidl_list && !msgno_in_list (src_uidl_list, i))
+	    {
+	      if (verbose_option > 1)
+		mu_diag_output (MU_DIAG_INFO, _("Ignoring message %lu"),
+				(unsigned long) i);
+	      continue;
+	    }
 	  rc = move_message (source, dest, i);
 	  rc = move_message (source, dest, i);
+	  if (rc == 0)
+	    msg_count++;
+	  else
+	    break;
+	}
     }
     }
   
   
+  if (verbose_option)
+    mu_diag_output (MU_DIAG_INFO,
+		    _("Number of processed messages: %lu"),
+		    (unsigned long) msg_count);
+  
   if (rc)
   if (rc)
     return rc;
     return rc;