Implement the mu_mailbox_get_uidls function and --uidl option in movemail.

* include/mailutils/mailbox.h (MU_UIDL_LENGH)
(MU_UIDL_BUFFER_SIZE): New defines.
(struct mu_uidl): New struct.
(mu_mailbox_get_uidls): New prototype.
* libproto/include/mailbox0.h (struct _mu_mailbox): New member
_get_uidls.
* libproto/pop/mbox.c: Implement _get_uidls.
* libproto/pop/folder.c: Include mailutils/cctype.h.

* mailbox/mailbox.c (mu_mailbox_get_uidls): New function.

* movemail/movemail.c: Implement --uidl option: use UIDLs to avoid
downloading same messages twice.  Based on the idea of Alfred M. Szmidt.
Implement --verbose option.

* libproto/imap/mbox.c: Fix comment.

* NEWS: Update.
* doc/texinfo/programs.texi: Update.

-GNU mailutils NEWS -- history of user-visible changes. 2009-05-25
+GNU mailutils NEWS -- history of user-visible changes. 2009-07-08
 Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 
 2008, 2009 Free Software Foundation, Inc.
 See the end of file for copying conditions.
@@ -18,6 +18,15 @@ All MU client utilities make use of the user ticket file,
 New configuration file entities allow to modify user's personal
 namespace and visible home directory.
 
+* Movemail
+
+When called with the `--uidl' command line option, the utility tries
+to avoid copying the message if a message with the same UIDL already
+exists in the destination mailbox.
+
+The `--verbose' command line option enables outputting additional
+information.
+
 * API
 
 * Wicket/Ticket functions
@@ -50,6 +59,10 @@ The `remote+sendmail', `remote+prog' and `remote+smtp' mailbox formats
 are deprecated in favor of `sendmail', `prog' and `smtp',
 correspondingly.  
 
+** New functions
+
+- mu_mailbox_get_uidls
+
 ** Removed functions
 
 - mu_sieve_get_ticket

 @c This is part of the GNU Mailutils manual.
-@c Copyright (C) 1999,2000,2001,2002,2003,2004,2005,2006,2007,2008
+@c Copyright (C) 1999,2000,2001,2002,2003,2004,2005,2006,2007,2008,2009
 @c Free Software Foundation, Inc.
 @c See file mailutils.texi for copying conditions.
 @comment *******************************************************************
@@ -4034,6 +4034,15 @@ If @var{bool} is @samp{true}, reverse message sorting order.
 If @var{bool} is @samp{true}, output information used by Emacs rmail interface.
 @end deffn
 
+@deffn {Movemail Config} uidl @var{bool}
+Avoid copying the message if a message with the same UIDL already
+exists in the destination mailbox.
+@end deffn
+
+@deffn {Movemail Config} verbose @var{level}
+Set verbosity level.
+@end deffn
+
 @multitable @columnfractions 0.3 0.6
 @headitem Statement @tab Reference
 @item debug         @tab @xref{Debug Statement}.
@@ -4147,9 +4156,6 @@ Preserve the source mailbox
 @itemx --reverse
 Reverse the sorting order
 
-@item --license
-Print GPL license and exit
-
 @item --external-locker=@var{program}
 Use given @var{program} as the external locker program.
 
@@ -4177,6 +4183,13 @@ Use specified URL as a mailspool directory
 @item --tls[=@var{bool}]
 Enable (default) or disable TLS support
 
+@item -u
+@item --uidl
+Use UIDLs to avoid downloading the same message twice.
+
+@item -v
+@item --verbose
+Increase verbosity level.
 @end table
 
 @page

@@ -66,6 +66,16 @@ extern int  mu_mailbox_sync            (mu_mailbox_t);
 extern int  mu_mailbox_save_attributes (mu_mailbox_t)
                                        __attribute__ ((deprecated));
 
+#define MU_UIDL_LENGTH 70
+#define MU_UIDL_BUFFER_SIZE (MU_UIDL_LENGTH+1)
+				       
+struct mu_uidl
+{
+  size_t msgno;
+  char uidl[MU_UIDL_BUFFER_SIZE];
+};
+extern int  mu_mailbox_get_uidls       (mu_mailbox_t, mu_list_t *);
+
 /* Update and scanning.  */
 extern int  mu_mailbox_get_size        (mu_mailbox_t, mu_off_t *size);
 extern int  mu_mailbox_is_updated      (mu_mailbox_t);

@@ -719,14 +719,14 @@ imap_scan (mu_mailbox_t mailbox, size_t msgno, size_t *pcount)
   return imap_scan0 (mailbox, msgno, pcount , 1);
 }
 
-/* Usually when this function is call it is because there is an oberver
-   attach an the client is try to build some sort of list/tree header
-   as the scanning progress.  But doing this for each message can be
-   time consuming and inefficient.  So we bundle all the request
-   into one and ask the server for everything "FETCH 1:*".  The good
-   side is that everything will be faster and we do not do lot of small
-   transcation but rather a big one.  The bad thing is that every thing
-   will be cache in the structure using a lot of memory.  */
+/* Normally this function is called when an observer is trying to build
+   some sort of list/tree header as the scanning progresses.  But doing
+   this for each message can be time consuming and inefficient.  So we
+   bundle all requests into one and ask the server for everything:
+   "FETCH 1:*".  The good side is that everything will be faster and we
+   do not do lots of small transcations, but rather a big one.  The bad
+   side is that everything will be cached in the structure using a lot of
+   memory.  */
 static int
 imap_scan0 (mu_mailbox_t mailbox, size_t msgno, size_t *pcount, int notif)
 {

@@ -74,6 +74,7 @@ struct _mu_mailbox
   int  (*_get_size)        (mu_mailbox_t, mu_off_t *);
 
   int  (*_quick_get_message) (mu_mailbox_t, mu_message_qid_t, mu_message_t *);
+  int  (*_get_uidls) (mu_mailbox_t, mu_list_t);
 };
 
 #ifdef __cplusplus

@@ -34,7 +34,8 @@
 #include <mailutils/auth.h>
 #include <mailutils/errno.h>
 #include <mailutils/mailbox.h>
-#include <mailutils/mutil.h>
+#include <mailutils/cstr.h>
+#include <mailutils/cctype.h>
 
 #include <folder0.h>
 #include <registrar0.h>

@@ -55,6 +55,7 @@
 #include <mailutils/md5.h>
 #include <mailutils/io.h>
 #include <mailutils/mutil.h>
+#include <mailutils/cstr.h>
 #include <mailutils/cctype.h>
 
 #include <folder0.h>
@@ -417,6 +418,57 @@ pop_destroy (mu_mailbox_t mbox)
     }
 }
 
+static int
+pop_mbox_uidls (mu_mailbox_t mbox, mu_list_t list)
+{
+  pop_data_t mpd = mbox->data;
+  int status;
+
+  status = pop_writeline (mpd, "UIDL\r\n");
+  CHECK_ERROR (mpd, status);
+  MU_DEBUG (mbox->debug, MU_DEBUG_PROT, mpd->buffer);
+
+  status = pop_write (mpd);
+  CHECK_EAGAIN (mpd, status);
+
+  status = pop_read_ack (mpd);
+  CHECK_EAGAIN (mpd, status);
+  MU_DEBUG (mpd->mbox->debug, MU_DEBUG_PROT, mpd->buffer);
+
+  if (!mu_c_strncasecmp (mpd->buffer, "+OK", 3))
+    {
+      do
+	{
+	  char *p;
+	  size_t num;
+	  struct mu_uidl *uidl;
+	  
+	  status = pop_read_ack (mpd);
+	  MU_DEBUG (mpd->mbox->debug, MU_DEBUG_PROT, mpd->buffer);
+
+	  num = strtoul (mpd->buffer, &p, 10);
+	  if (*p == 0 || !mu_isblank (*p))
+	    continue; /* FIXME: or error? */
+	  p = mu_str_skip_class (p, MU_CTYPE_SPACE);
+	  mu_rtrim_cset (p, "\r\n");
+
+	  uidl = malloc (sizeof (uidl[0]));
+	  if (!uidl)
+	    {
+	      status = ENOMEM;
+	      break;
+	    }
+	  uidl->msgno = num;
+	  strncpy (uidl->uidl, p, MU_UIDL_BUFFER_SIZE);
+	  status = mu_list_append (list, uidl);
+	}
+      while (mpd->nl);
+    }
+  else
+    status = ENOSYS;
+  return status;
+}
+
 /*
   POP3 CAPA support.
  */
@@ -454,12 +506,16 @@ pop_parse_capa (pop_data_t mpd)
 	    mpd->capa |= CAPA_STLS;
 	}
       while (mpd->nl);
+
+      if (mpd->capa & CAPA_UIDL)
+	mpd->mbox->_get_uidls = pop_mbox_uidls;
+  
       return status;
     }
   else
     {
       /* mu_error ("CAPA not implemented"); */ /* FIXME */
-      return -1;
+      return ENOSYS;
     }
 }
 
@@ -485,7 +541,6 @@ pop_capa (mu_mailbox_t mbox)
   return pop_parse_capa (mpd);
 }
 
-
 /* Simple User/pass authentication for pop. We ask for the info
    from the standard input.  */
 int
@@ -1074,7 +1129,7 @@ pop_get_message (mu_mailbox_t mbox, size_t msgno, mu_message_t *pmsg)
   /* Set the UIDL call on the message. */
   if (mpd->capa & CAPA_UIDL)
     mu_message_set_uidl (msg, pop_uidl, mpm);
-
+  
   /* Set the UID on the message. */
   mu_message_set_uid (msg, pop_uid, mpm);
 
@@ -1427,7 +1482,7 @@ pop_message_size (mu_message_t msg, size_t *psize)
   CLEAR_STATE (mpd);
 
   if (status != 0)
-    status = MU_ERR_PARSE;
+    return MU_ERR_PARSE;
 
   /* The size of the message is with the extra '\r' octet for everyline.
      Substract to get, hopefully, a good count.  */
@@ -1570,7 +1625,7 @@ pop_uid (mu_message_t msg,  size_t *puid)
   return 0;
 }
 
-/* Get the UIDL.  Client should be prepare since it may fail.  UIDL is
+/* Get the UIDL.  The client should be prepared, since it may fail.  UIDL is
    optional on many POP servers.
    FIXME:  We should check the "mpd->capa & CAPA_UIDL" and fall back to
    a md5 scheme ? Or maybe check for "X-UIDL" a la Qpopper ?  */
@@ -1582,14 +1637,12 @@ pop_uidl (mu_message_t msg, char *buffer, size_t buflen, size_t *pnwriten)
   int status = 0;
   void *func = (void *)pop_uidl;
   size_t num;
-  /* According to the RFC uidl's are no longer then 70 chars.  Still playit
-     safe  */
-  char uniq[128];
+  char uniq[MU_UIDL_BUFFER_SIZE];
 
   if (mpm == NULL)
     return EINVAL;
 
-  /* Is it cache ?  */
+  /* Is it cached ?  */
   if (mpm->uidl)
     {
       size_t len = strlen (mpm->uidl);

@@ -287,7 +287,7 @@ mu_mailbox_flush (mu_mailbox_t mbox, int expunge)
 {
   size_t i, total = 0;
   int status = 0;
-
+  
   if (!mbox)
     return EINVAL;
   if (!(mbox->flags & (MU_STREAM_RDWR|MU_STREAM_WRITE|MU_STREAM_APPEND)))
@@ -680,4 +680,67 @@ mu_mailbox_unlock (mu_mailbox_t mbox)
   mu_mailbox_get_locker (mbox, &lock);
   return mu_locker_unlock (lock);
 }
-  
+
+static void
+free_uidl (void *item)
+{
+  free (item);
+}
+
+int
+mu_mailbox_get_uidls (mu_mailbox_t mbox, mu_list_t *plist)
+{
+  mu_list_t list;
+  int status;
+
+  if (mbox == NULL)
+    return MU_ERR_MBX_NULL;
+  if (plist == NULL)
+    return EINVAL;
+  status = mu_list_create (&list);
+  if (status)
+    return status;
+  mu_list_set_destroy_item (list, free_uidl);
+  if (mbox->_get_uidls)
+    status = mbox->_get_uidls (mbox, list);
+  else
+    {
+      size_t i, total;
+
+      status = mu_mailbox_messages_count (mbox, &total);
+      if (status)
+	return status;
+      for (i = 1; i <= total; i++)
+	{
+	  mu_message_t msg = NULL;
+	  char buf[MU_UIDL_BUFFER_SIZE];
+	  size_t n;
+	  struct mu_uidl *uidl;
+	  
+	  status = mu_mailbox_get_message (mbox, i, &msg);
+	  if (status)
+	    break;
+	  status = mu_message_get_uidl (msg, buf, sizeof (buf), &n);
+	  if (status)
+	    break;
+	  uidl = malloc (sizeof (uidl[0]));
+	  if (!uidl)
+	    {
+	      status = ENOMEM;
+	      break;
+	    }
+	  uidl->msgno = i;
+	  strncpy (uidl->uidl, strdup (buf), MU_UIDL_BUFFER_SIZE);
+	  status = mu_list_append (list, uidl);
+	  if (status)
+	    {
+	      free_uidl (uidl);
+	      break;
+	    }
+	}
+    }
+  *plist = list;
+  return status;
+}
+
+	  

@@ -47,6 +47,12 @@ static struct argp_option options[] = {
   { "copy-permissions", 'P', NULL, 0,
     N_("Copy original mailbox permissions and ownership when applicable"),
     0 },
+  { "uidl", 'u', NULL, 0,
+    N_("Use UIDLs to avoid downloading the same message twice"),
+    0 },
+  { "verbose", 'v', NULL, 0,
+    N_("Increase verbosity level"),
+    0 },
   { NULL,      0, NULL, 0, NULL, 0 }
 };
 
@@ -54,6 +60,8 @@ static int reverse_order;
 static int preserve_mail; 
 static int emacs_mode;
 static int copy_meta;
+static int uidl_option;
+static int verbose_option;
 
 static error_t
 parse_opt (int key, char *arg, struct argp_state *state)
@@ -73,6 +81,14 @@ parse_opt (int key, char *arg, struct argp_state *state)
     case 'P':
       copy_meta = 1;
       break;
+
+    case 'u':
+      mu_argp_node_list_new (&lst, "uidl", "yes");
+      break;
+
+    case 'v':
+      verbose_option++;
+      break;
       
     case OPT_EMACS:
       mu_argp_node_list_new (&lst, "emacs", "yes");
@@ -109,6 +125,10 @@ struct mu_cfg_param movemail_cfg_param[] = {
     N_("Reverse message sorting order.") },
   { "emacs", mu_cfg_bool, &emacs_mode, 0, NULL,
     N_("Output information used by Emacs rmail interface.") },
+  { "uidl", mu_cfg_bool, &uidl_option, 0, NULL,
+    N_("Use UIDLs to avoid downloading the same message twice.") },
+  { "verbose", mu_cfg_int, &verbose_option, 0, NULL,
+    N_("Increase verbosity level.") },
   { NULL }
 };
 
@@ -227,14 +247,14 @@ move_message (mu_mailbox_t src, mu_mailbox_t dst, size_t msgno)
 
   if ((rc = mu_mailbox_get_message (src, msgno, &msg)) != 0)
     {
-      fprintf (stderr, _("Cannot read message %lu: %s\n"),
-	       (unsigned long) msgno, mu_strerror (rc));
+      mu_error (_("Cannot read message %lu: %s"),
+		(unsigned long) msgno, mu_strerror (rc));
       return rc;
     }
   if ((rc = mu_mailbox_append_message (dst, msg)) != 0)
     {
-      fprintf (stderr, _("Cannot append message %lu: %s\n"),
-	       (unsigned long) msgno, mu_strerror (rc));
+      mu_error (_("Cannot append message %lu: %s\n"),
+		(unsigned long) msgno, mu_strerror (rc));
       return rc;
     }
   if (!preserve_mail)
@@ -344,6 +364,36 @@ set_permissions (mu_mailbox_t mbox)
     exit (1);
 }
 
+static int
+_compare_uidls (const void *item, const void *value)
+{
+  const struct mu_uidl *a = item;
+  const struct mu_uidl *b = value;
+
+  return strcmp (a->uidl, b->uidl);
+}
+
+static int
+_compare_msgno (const void *item, const void *value)
+{
+  const struct mu_uidl *a = item;
+  const struct mu_uidl *b = value;
+
+  if (a->msgno < b->msgno)
+    return -1;
+  if (a->msgno > b->msgno)
+    return 1;
+  return 0;
+}
+
+static int
+msgno_in_list (mu_list_t list, size_t num)
+{
+  struct mu_uidl t;
+  t.msgno = num;
+  return mu_list_locate (list, &t, NULL) == 0;
+}  
+
 int
 main (int argc, char **argv)
 {
@@ -352,6 +402,8 @@ main (int argc, char **argv)
   int rc = 0;
   char *source_name, *dest_name;
   int flags;
+  mu_list_t src_uidl_list = NULL;
+  size_t msg_count = 0;
   
   /* Native Language Support */
   MU_APP_INIT_NLS ();
@@ -395,19 +447,91 @@ main (int argc, char **argv)
     set_permissions (source);
   
   open_mailbox (&dest, dest_name, MU_STREAM_RDWR | MU_STREAM_CREAT, NULL);
+
+  rc = mu_mailbox_messages_count (source, &total);
+  if (rc)
+    {
+      mu_error(_("Cannot count messages: %s"), mu_strerror (rc));
+      exit (1);
+    }
+  
+  if (verbose_option)
+    mu_diag_output (MU_DIAG_INFO,
+		    _("Number of messages in source mailbox: %lu"),
+		    (unsigned long) total);
+
+  if (uidl_option)
+    {
+      mu_iterator_t itr;
+      mu_list_t dst_uidl_list = NULL;
+
+      rc = mu_mailbox_get_uidls (source, &src_uidl_list);
+      if (rc)
+	die (source, _("Cannot get UIDLs"), rc);
+      rc = mu_mailbox_get_uidls (dest, &dst_uidl_list);
+      if (rc)
+	die (dest, _("Cannot get UIDLs"), rc);
+
+      mu_list_set_comparator (src_uidl_list, NULL);
+      mu_list_set_comparator (dst_uidl_list, _compare_uidls);
+      
+      mu_list_get_iterator (src_uidl_list, &itr);
+      for (mu_iterator_first (itr); !mu_iterator_is_done (itr);
+	   mu_iterator_next (itr))
+	{
+	  struct mu_uidl *uidl;
+	      
+	  mu_iterator_current (itr, (void **)&uidl);
+	  if (mu_list_locate (dst_uidl_list, uidl, NULL) == 0)
+	    mu_list_remove (src_uidl_list, uidl);
+	}
+      mu_iterator_destroy (&itr);
+      mu_list_destroy (&dst_uidl_list);
+      mu_list_set_comparator (src_uidl_list, _compare_msgno);
+    }
   
-  mu_mailbox_messages_count (source, &total);
   if (reverse_order)
     {
-      for (i = total; rc == 0 && i > 0; i--)
-	rc = move_message (source, dest, i);
+      for (i = total; i > 0; i--)
+	{
+	  if (src_uidl_list && !msgno_in_list (src_uidl_list, i))
+	    {
+	      if (verbose_option > 1)
+		mu_diag_output (MU_DIAG_INFO, _("Ignoring message %lu"),
+				(unsigned long) i);
+	      continue;
+	    }
+	  rc = move_message (source, dest, i);
+	  if (rc == 0)
+	    msg_count++;
+	  else
+	    break;
+	}
     }
   else
     {
-      for (i = 1; rc == 0 && i <= total; i++)
-	rc = move_message (source, dest, i);
+      for (i = 1; i <= total; i++)
+	{
+	  if (src_uidl_list && !msgno_in_list (src_uidl_list, i))
+	    {
+	      if (verbose_option > 1)
+		mu_diag_output (MU_DIAG_INFO, _("Ignoring message %lu"),
+				(unsigned long) i);
+	      continue;
+	    }
+	  rc = move_message (source, dest, i);
+	  if (rc == 0)
+	    msg_count++;
+	  else
+	    break;
+	}
     }
   
+  if (verbose_option)
+    mu_diag_output (MU_DIAG_INFO,
+		    _("Number of processed messages: %lu"),
+		    (unsigned long) msg_count);
+  
   if (rc)
     return rc;