Fix imap4d input tokenizer.

The proper delimiters are ( and ), listed in RFC as atom_specials.
The characters "[]<>." are delimiters for FETCH, and "." is  a
delimiter for STORE.  Fix tokenizer to recognize these only in the
corresponding contexts.

In particular, it fixes parsing of input atoms containing dots, as
foo@bar.baz, which previous versions incorrectly splitted in three.

* imap4d/util.c (ISDELIM): Delimiters are only ( and )
* imap4d/imap4d.h (struct imap4d_parsebuf): new data type.
(imap4d_parsebuf_t): New data type.
(imap4d_parsebuf_exit, imap4d_parsebuf_peek)
(imap4d_parsebuf_next, imap4d_parsebuf_token)
(imap4d_parsebuf_data, imap4d_with_parsebuf): New prototypes.
* imap4d/parsebuf.c: New file.
* imap4d/Makefile.am: Add parsebuf.c
* imap4d/fetch.c, imap4d/store.c: Rewrite using parsebuf functions.

 ## Process this file with GNU Automake to create Makefile.in
 
 ##   Copyright (C) 1999, 2000, 2001, 2002, 2003, 2005,
-##   2007, 2008 Free Software Foundation, Inc.
+##   2007, 2008, 2009 Free Software Foundation, Inc.
 ##
 ##   GNU Mailutils is free software; you can redistribute it and/or
 ##   modify it under the terms of the GNU General Public License as
@@ -47,6 +47,7 @@ imap4d_SOURCES = \
  lsub.c\
  namespace.c\
  noop.c\
+ parsebuf.c\
  preauth.c\
  rename.c\
  search.c\

 /* GNU Mailutils -- a suite of utilities for electronic mail
    Copyright (C) 1999, 2001, 2002, 2003, 2004, 
-   2005, 2006, 2007, 2008 Free Software Foundation, Inc.
+   2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
 
    GNU Mailutils is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -218,7 +218,35 @@ int imap4d_getline (char **pbuf, size_t *psize, size_t *pnbytes);
 #define IMAP4_ARG_2       3
 #define IMAP4_ARG_3       4
 #define IMAP4_ARG_4       5
+
+  /* Specialized buffer parsing */
+struct imap4d_parsebuf
+{
+  imap4d_tokbuf_t tok;
+  int arg;            /* Argument number in tok */
+  char *tokptr;       /* Current token */
+  size_t tokoff;      /* Offset of next delimiter in tokptr */
+  int save_char;      /* Character saved from tokptr[tokoff] */
+  char *token;        /* Pointer to the current token */
+  const char *delim;  /* Array of delimiters */
+  char *peek_ptr;     /* Peeked token */
+  jmp_buf errjmp;
+  char *err_text;
+  void *data;
+};
+
+typedef struct imap4d_parsebuf *imap4d_parsebuf_t;
   
+void imap4d_parsebuf_exit (imap4d_parsebuf_t pb, char *text);
+char *imap4d_parsebuf_peek (imap4d_parsebuf_t pb);
+char *imap4d_parsebuf_next (imap4d_parsebuf_t pb, int req);
+int imap4d_with_parsebuf (imap4d_tokbuf_t tok, int arg,
+			  const char *delim,
+			  int (*thunk) (imap4d_parsebuf_t), void *data,
+			  char **err_text);
+#define imap4d_parsebuf_token(p) ((p)->token)
+#define imap4d_parsebuf_data(p) ((p)->data)
+
   /* Imap4 commands */
 extern int  imap4d_append (struct imap4d_command *, imap4d_tokbuf_t);
 extern int  imap4d_authenticate (struct imap4d_command *, imap4d_tokbuf_t);

+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2009 Free Software Foundation, Inc.
+
+   GNU Mailutils is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GNU Mailutils is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GNU Mailutils; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+   MA 02110-1301 USA */
+
+#include "imap4d.h"
+
+static void
+parsebuf_free (struct imap4d_parsebuf *p)
+{
+  if (p->peek_ptr)
+    free (p->peek_ptr);
+}
+  
+void
+imap4d_parsebuf_exit (struct imap4d_parsebuf *p, char *text)
+{
+  p->err_text = text;
+  longjmp (p->errjmp, 1);
+}
+
+static char *
+pbcopy (const char *str, size_t len)
+{
+  char *p = malloc (len + 1);
+  if (!p)
+    imap4d_bye (ERR_NO_MEM);
+  memcpy (p, str, len);
+  p[len] = 0;
+  return p;
+}
+
+static char *
+pbcopy_char (int c)
+{
+  char ch = c;
+  return pbcopy (&ch, 1);
+}
+
+char *
+imap4d_parsebuf_peek (struct imap4d_parsebuf *p)
+{
+  if (!p->peek_ptr)
+    {
+      if (!p->tokptr || p->save_char == 0)
+ 	{
+	  char *token = imap4d_tokbuf_getarg (p->tok, p->arg);
+	  if (!token)
+	    return NULL;
+	  if (p->delim)
+	    {
+	      if (strchr (p->delim, token[0]))
+		p->peek_ptr = pbcopy_char (token[0]);
+	      else
+		p->peek_ptr = pbcopy (token, strcspn (token, p->delim));
+	    }
+	  else
+	    p->peek_ptr = pbcopy (token, strlen (token));
+	}
+      else
+	{
+	  char *token = p->token + p->tokoff;
+
+	  if (strchr (p->delim, p->save_char))
+	    p->peek_ptr = pbcopy_char (p->save_char);
+	  else
+	    {
+	      size_t off = strcspn (token + 1, p->delim);
+	      p->peek_ptr = pbcopy (token, off + 1);
+	      p->peek_ptr[0] = p->save_char;
+	    }
+	}
+    }
+  return p->peek_ptr;
+}
+
+char *
+imap4d_parsebuf_next (struct imap4d_parsebuf *p, int req)
+{
+  if (p->peek_ptr)
+    {
+      free (p->peek_ptr);
+      p->peek_ptr = NULL;
+    }
+
+  if (!p->tokptr || p->save_char == 0)
+    {
+      p->tokptr = imap4d_tokbuf_getarg (p->tok, p->arg++);
+      p->token = p->tokptr;
+      p->tokoff = 0;
+      if (!p->token)
+	{
+	  if (req)
+	    imap4d_parsebuf_exit (p, "Too few arguments");
+	  return NULL;
+	}
+      if (p->delim)
+	{
+	  if (strchr (p->delim, p->token[0]))
+	    {
+	      p->save_char = p->token[1];
+	      p->tokoff = 1;
+	      p->token[1] = 0;
+	    }
+	  else
+	    {
+	      p->tokoff = strcspn (p->token, p->delim);
+	      if ((p->save_char = p->token[p->tokoff]))
+		p->token[p->tokoff] = 0;
+	    }
+	}
+      else
+	p->save_char = 0;
+    }
+  else
+    {
+      p->token[p->tokoff] = p->save_char;
+      p->token += p->tokoff;
+      if (strchr (p->delim, p->save_char))
+	{
+	  p->save_char = p->token[1];
+	  p->token[1] = 0;
+	  p->tokoff = 1;
+	}
+      else
+	{
+	  p->tokoff = strcspn (p->token, p->delim);
+	  if ((p->save_char = p->token[p->tokoff]))
+	    p->token[p->tokoff] = 0;
+	}
+    }
+  
+  return p->token;
+}
+
+int
+imap4d_with_parsebuf (imap4d_tokbuf_t tok, int arg, const char *delim,
+		      int (*thunk) (imap4d_parsebuf_t), void *data,
+		      char **err_text)
+{
+  struct imap4d_parsebuf pbuf;
+  int rc;
+  
+  memset (&pbuf, 0, sizeof pbuf);
+  pbuf.tok = tok;
+  pbuf.arg = arg;
+  pbuf.delim = delim;
+  pbuf.data = data;
+  pbuf.err_text = "Syntax error";
+  pbuf.data = data;
+  
+  if (setjmp (pbuf.errjmp))
+    {
+      *err_text = pbuf.err_text;
+      parsebuf_free (&pbuf);
+      return RESP_BAD;
+    }
+
+  rc = thunk (&pbuf);
+  parsebuf_free (&pbuf);
+  return rc;
+}

 /* GNU Mailutils -- a suite of utilities for electronic mail
-   Copyright (C) 1999, 2001, 2005, 2007, 2008 Free Software Foundation, Inc.
+   Copyright (C) 1999, 2001, 2005, 2007, 2008,
+   2009 Free Software Foundation, Inc.
 
    GNU Mailutils is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -18,165 +19,165 @@
 
 #include "imap4d.h"
 
-/*
- * Now you're messing with a sumbitch
- */
+enum value_type { STORE_SET, STORE_ADD, STORE_UNSET };
 
-int
-imap4d_store (struct imap4d_command *command, imap4d_tokbuf_t tok)
+struct store_parse_closure
 {
-  int rc;
-  char *err_text;
-  
-  rc = imap4d_store0 (tok, 0, &err_text);
-  return util_finish (command, rc, "%s", err_text);
-}
-
-struct parsebuf
-{
-  char *token;
-  imap4d_tokbuf_t tok;
-  int arg;
-  jmp_buf errjmp;
-  char *err_text;
+  enum value_type how;
+  int ack;
+  int type;
+  int isuid;
+  size_t *set;
+  size_t count;
 };
-
-static void
-parsebuf_exit (struct parsebuf *p, char *text)
-{
-  p->err_text = text;
-  longjmp (p->errjmp, 1);
-}
-
-static char *
-parsebuf_next (struct parsebuf *p, int req)
-{
-  p->token = imap4d_tokbuf_getarg (p->tok, p->arg++);
-  if (!p->token && req)
-    parsebuf_exit (p, "Too few arguments");
-  return p->token;
-}
-
-int
-imap4d_store0 (imap4d_tokbuf_t tok, int isuid, char **ptext)
+  
+static int
+store_thunk (imap4d_parsebuf_t p)
 {
+  struct store_parse_closure *pclos = imap4d_parsebuf_data (p);
   char *msgset;
-  int status;
-  int ack = 1;
-  size_t i;
-  int n = 0;
-  size_t *set = NULL;
-  enum value_type { STORE_SET, STORE_ADD, STORE_UNSET } how;
-  struct parsebuf pb;
   char *data;
-  int type = 0;
+  int status;
   
-  pb.tok = tok;
-  pb.arg = IMAP4_ARG_1 + !!isuid;
-  pb.err_text = NULL;
-  if (setjmp (pb.errjmp))
-    {
-      *ptext = pb.err_text;
-      free (set);
-      return RESP_BAD;
-    }
-      
-  msgset = parsebuf_next (&pb, 1);
-  data = parsebuf_next (&pb, 1);
+  msgset = imap4d_parsebuf_next (p, 1);
+  data = imap4d_parsebuf_next (p, 1);
 
   if (*data == '+')
     {
-      how = STORE_ADD;
+      pclos->how = STORE_ADD;
       data++;
     }
   else if (*data == '-')
     {
-      how = STORE_UNSET;
+      pclos->how = STORE_UNSET;
       data++;
     }
   else
-    how = STORE_SET;
+    pclos->how = STORE_SET;
   
   if (strcasecmp (data, "FLAGS"))
-    parsebuf_exit (&pb, "Bogus data item");
-  data = parsebuf_next (&pb, 1);
+    imap4d_parsebuf_exit (p, "Bogus data item");
+  data = imap4d_parsebuf_next (p, 1);
 
   if (*data == '.')
     {
-      data = parsebuf_next (&pb, 1);
+      data = imap4d_parsebuf_next (p, 1);
       if (strcasecmp (data, "SILENT") == 0)
 	{
-	  ack = 0;
-	  parsebuf_next (&pb, 1);
+	  pclos->ack = 0;
+	  imap4d_parsebuf_next (p, 1);
 	}
       else
-	parsebuf_exit (&pb, "Bogus data suffix");
+	imap4d_parsebuf_exit (p, "Bogus data suffix");
     }
 
   /* Get the message numbers in set[].  */
-  status = util_msgset (msgset, &set, &n, isuid);
-  if (status != 0)
+  status = util_msgset (msgset, &pclos->set, &pclos->count, pclos->isuid);
+  switch (status)
     {
+    case 0:
+      break;
+
+    case EINVAL:
       /* See RFC 3501, section 6.4.8, and a comment to the equivalent code
 	 in fetch.c */
-      *ptext = "Completed";
+      p->err_text = "Completed";
       return RESP_OK;
+
+    default:
+      p->err_text = "Failed to parse message set";
+      return RESP_NO;
     }      
 
-  if (pb.token[0] != '(')
-    parsebuf_exit (&pb, "Syntax error");
-  parsebuf_next (&pb, 1);
+  if (p->token[0] != '(')
+    imap4d_parsebuf_exit (p, "Syntax error");
+  imap4d_parsebuf_next (p, 1);
   
   do
     {
       int t;
-      if (!util_attribute_to_type (pb.token, &t))
-	type |= t;
+      if (!util_attribute_to_type (p->token, &t))
+	pclos->type |= t;
     }
-  while (parsebuf_next (&pb, 1) && pb.token[0] != ')');
+  while (imap4d_parsebuf_next (p, 1) && p->token[0] != ')');
+  return RESP_OK;
+}
+
+int
+imap4d_store0 (imap4d_tokbuf_t tok, int isuid, char **ptext)
+{
+  int rc;
+  struct store_parse_closure pclos;
 
-  for (i = 0; i < n; i++)
+  memset (&pclos, 0, sizeof pclos);
+  pclos.ack = 1;
+  pclos.isuid = isuid;
+  
+  rc = imap4d_with_parsebuf (tok,
+			     IMAP4_ARG_1 + !!isuid,
+			     ".",
+			     store_thunk, &pclos,
+			     ptext);
+  if (rc == RESP_OK)
     {
-      mu_message_t msg = NULL;
-      mu_attribute_t attr = NULL;
-      size_t msgno = isuid ? uid_to_msgno (set[i]) : set[i];
+      size_t i;
       
-      if (msgno)
+      for (i = 0; i < pclos.count; i++)
 	{
-	  mu_mailbox_get_message (mbox, msgno, &msg);
-	  mu_message_get_attribute (msg, &attr);
-
-	  switch (how)
-	    {
-	    case STORE_ADD:
-	      mu_attribute_set_flags (attr, type);
-	      break;
+	  mu_message_t msg = NULL;
+	  mu_attribute_t attr = NULL;
+	  size_t msgno = isuid ? uid_to_msgno (pclos.set[i]) : pclos.set[i];
       
-	    case STORE_UNSET:
-	      mu_attribute_unset_flags (attr, type);
-	      break;
+	  if (msgno)
+	    {
+	      mu_mailbox_get_message (mbox, msgno, &msg);
+	      mu_message_get_attribute (msg, &attr);
+	      
+	      switch (pclos.how)
+		{
+		case STORE_ADD:
+		  mu_attribute_set_flags (attr, pclos.type);
+		  break;
+		  
+		case STORE_UNSET:
+		  mu_attribute_unset_flags (attr, pclos.type);
+		  break;
       
-	    case STORE_SET:
-	      mu_attribute_unset_flags (attr, 0xffffffff); /* FIXME */
-	      mu_attribute_set_flags (attr, type);
+		case STORE_SET:
+		  mu_attribute_unset_flags (attr, 0xffffffff); /* FIXME */
+		  mu_attribute_set_flags (attr, pclos.type);
+		}
+	    }
+	  
+	  if (pclos.ack)
+	    {
+	      util_send ("* %d FETCH (", msgno);
+	      
+	      if (isuid)
+		util_send ("UID %lu ", (unsigned long) msgno);
+	      util_send ("FLAGS (");
+	      util_print_flags (attr);
+	      util_send ("))\r\n");
 	    }
+	  /* Update the flags of uid table.  */
+	  imap4d_sync_flags (pclos.set[i]);
 	}
 
-      if (ack)
-	{
-	  util_send ("* %d FETCH (", msgno);
-
-	  if (isuid)
-	    util_send ("UID %lu ", (unsigned long) msgno);
-	  util_send ("FLAGS (");
-	  util_print_flags (attr);
-	  util_send ("))\r\n");
-	}
-      /* Update the flags of uid table.  */
-      imap4d_sync_flags (set[i]);
+      *ptext = "Completed";
     }
-  free (set);
-  *ptext = "Completed";
-  return RESP_OK;
+
+  free (pclos.set);
+  
+  return rc;
+}
+
+int
+imap4d_store (struct imap4d_command *command, imap4d_tokbuf_t tok)
+{
+  int rc;
+  char *err_text;
+  
+  rc = imap4d_store0 (tok, 0, &err_text);
+  return util_finish (command, rc, "%s", err_text);
 }
 

 /* GNU Mailutils -- a suite of utilities for electronic mail
    Copyright (C) 1999, 2001, 2002, 2003, 2004, 
-   2005, 2006, 2007, 2008 Free Software Foundation, Inc.
+   2005, 2006, 2007, 2008, 2009 Free Software Foundation, Inc.
 
    GNU Mailutils is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -121,6 +121,7 @@ util_msgset (char *s, size_t ** set, int *n, int isuid)
 	      {
 		if (*set)
 		  free (*set);
+		*set = NULL;
 		*n = 0;
 		return EINVAL;
 	      }
@@ -133,6 +134,7 @@ util_msgset (char *s, size_t ** set, int *n, int isuid)
 		  }
 		if (*set)
 		  free (*set);
+		*set = NULL;
 		*n = 0;
 		return EINVAL;
 	      }
@@ -147,6 +149,7 @@ util_msgset (char *s, size_t ** set, int *n, int isuid)
 		    if (tmp < 0 || val == 0)
 		      {
 			free (*set);
+			*set = NULL;
 			*n = 0;
 			return EINVAL;
 		      }
@@ -206,6 +209,7 @@ util_msgset (char *s, size_t ** set, int *n, int isuid)
 	  done = 1;
 	  if (*set)
 	    free (*set);
+	  *set = NULL;
 	  *n = 0;
 	  return EINVAL;
 
@@ -238,6 +242,7 @@ util_msgset (char *s, size_t ** set, int *n, int isuid)
 	  if (tmp < 0 || val == 0)
 	    {
 	      free (*set);
+	      *set = NULL;
 	      *n = 0;
 	      return EINVAL;
 	    }
@@ -1221,7 +1226,7 @@ imap4d_tokbuf_expand (struct imap4d_tokbuf *tok, size_t size)
     }
 }
 
-#define ISDELIM(c) (strchr (".()[]<>", (c)) != NULL)
+#define ISDELIM(c) (strchr ("()", (c)) != NULL)
 #define ISWS(c) (c == ' ' || c == '\t')
 
 int
@@ -1257,7 +1262,7 @@ gettok (struct imap4d_tokbuf *tok, size_t off)
 	tok->argmax = 16;
       else
 	tok->argmax *= 2;
-      tok->argp = realloc(tok->argp, tok->argmax * sizeof (tok->argp[0]));
+      tok->argp = realloc (tok->argp, tok->argmax * sizeof (tok->argp[0]));
       if (!tok->argp)
 	imap4d_bye (ERR_NO_MEM);
     }
@@ -1278,7 +1283,7 @@ gettok (struct imap4d_tokbuf *tok, size_t off)
 	{
 	  size_t len;
 	  off++;
-	  len  = unquote(buf + off, p - (buf + off));
+	  len  = unquote (buf + off, p - (buf + off));
 	  buf[off + len] = 0;
 	  tok->argp[tok->argc++] = off;
 	  return p - buf + 1;