Port wordsplit from grecs 20616b88

The updated version supports tilde and pathname expansion, command
substitution and standard shell-like replacement constructs in
variable substitution, such as ${X:-V} etc.

* include/mailutils/wordsplit.h: Update.
* libmailutils/string/wordsplit.c: Update.
* libmailutils/tests/wsp.c: Update.
* libmailutils/tests/wordsplit.at: Update unescape test.

* libmailutils/imapio/create.c (mu_imapio_create): Initialize ws_escape
array.
* libmailutils/mime/mimehdr.c (_mime_header_parse): Likewise.
* libmailutils/tests/modmesg.c: Use mu_wordsplit with MU_WRDSF_NOSPLIT
to run expansions on the string.
* mu/shell.c (shell_prompt): Likewise.

@@ -29,7 +29,10 @@ mu_imapio_create (mu_imapio_t *iop, mu_stream_t str, int server)
   io->_imap_stream = str;
   mu_stream_ref (str);
   io->_imap_ws.ws_delim = " \t()[]";
-  io->_imap_ws.ws_escape = "\\\"";
+  io->_imap_ws.ws_escape[0] = NULL;
+  io->_imap_ws.ws_escape[1] = "\\\\\"\"";
+  MU_WRDSO_ESC_SET (&io->_imap_ws, 0, MU_WRDSO_BSKEEP);
+  MU_WRDSO_ESC_SET (&io->_imap_ws, 1, MU_WRDSO_BSKEEP);
   io->_imap_ws_flags = MU_WRDSF_DELIM |
                        MU_WRDSF_ESCAPE |
                        MU_WRDSF_NOVAR |
@@ -37,7 +40,8 @@ mu_imapio_create (mu_imapio_t *iop, mu_stream_t str, int server)
                        MU_WRDSF_DQUOTE |
                        MU_WRDSF_RETURN_DELIMS |
                        MU_WRDSF_WS |
-                       MU_WRDSF_APPEND;
+                       MU_WRDSF_APPEND |
+                       MU_WRDSF_OPTIONS;
   io->_imap_server = server;
   *iop = io;
   return 0;

@@ -293,12 +293,14 @@ _mime_header_parse (const char *text, char **pvalue,
   size_t i;
 
   ws.ws_delim = " \t\r\n;";
-  ws.ws_escape = "\\\"";
+  ws.ws_escape[0] = ws.ws_escape[1] = "\\\\\"\"";
+  MU_WRDSO_ESC_SET (&ws, 0, MU_WRDSO_BSKEEP);
+  MU_WRDSO_ESC_SET (&ws, 1, MU_WRDSO_BSKEEP);
   if (mu_wordsplit (text, &ws,
 		    MU_WRDSF_DELIM | MU_WRDSF_ESCAPE |
 		    MU_WRDSF_NOVAR | MU_WRDSF_NOCMD |
 		    MU_WRDSF_DQUOTE | MU_WRDSF_SQUEEZE_DELIMS |
-		    MU_WRDSF_RETURN_DELIMS | MU_WRDSF_WS))
+		    MU_WRDSF_RETURN_DELIMS | MU_WRDSF_WS | MU_WRDSF_OPTIONS))
     {
       mu_debug (MU_DEBCAT_MIME, MU_DEBUG_ERROR,
 		(_("wordsplit: %s"), mu_wordsplit_strerror (&ws)));

@@ -32,7 +32,6 @@ main (int argc, char **argv)
   mu_stream_t stream = NULL;
   mu_header_t hdr;
   mu_body_t body;
-  char *buf = NULL;
   
   mu_set_program_name (argv[0]);
 
@@ -78,15 +77,20 @@ main (int argc, char **argv)
 	}
       else if (strcmp (argv[i], "-t") == 0)
 	{
-	  size_t len;
+	  mu_wordsplit_t ws;
 	  i++;
 	  assert (argv[i] != NULL);
-	  len = strlen (argv[i]);
-	  buf = realloc (buf, len + 1);
-	  mu_wordsplit_c_unquote_copy (buf, argv[i], len);
-	  assert (buf != NULL);
-	  assert (mu_stream_write (stream, buf,
-				   strlen (buf), NULL) == 0);
+
+	  if (mu_wordsplit (argv[i], &ws,
+			    MU_WRDSF_NOSPLIT | MU_WRDSF_DEFFLAGS))
+	    {
+	      mu_error ("mu_wordsplit: %s", mu_wordsplit_strerror (&ws));
+	      exit (1);
+	    }
+	  else
+	    assert (mu_stream_write (stream, ws.ws_wordv[0],
+				     strlen (ws.ws_wordv[0]), NULL) == 0);
+	  mu_wordsplit_free (&ws);
 	}
       else
 	mu_error ("ignoring unknown argument %s", argv[i]);

@@ -347,7 +347,7 @@ TESTWSP([suppress ws trimming within quotes],[],
 4: "formatfield=In message %{text}, "
 ])
 
-TESTWSP([unescape],[],[-default novar nocmd quote escape '\"'],
+TESTWSP([unescape],[],[-default novar nocmd quote escape :+:'\\""'],
 [\Seen "quote \"" "bs \\"],
 [NF: 3
 0: \\Seen

@@ -22,6 +22,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <mailutils/wordsplit.h>
+#include <mailutils/alloc.h>
 #include <mailutils/kwd.h>
 #include <mailutils/errno.h>
 #include <mailutils/error.h>
@@ -29,6 +30,8 @@
 
 extern char **environ;
 
+char *progname;
+
 struct mu_kwd bool_keytab[] = {
   { "append", MU_WRDSF_APPEND },
   /*{ "reuse",  MU_WRDSF_REUSE },*/
@@ -50,6 +53,14 @@ struct mu_kwd bool_keytab[] = {
   { "default", MU_WRDSF_DEFFLAGS },
   { "env_kv", MU_WRDSF_ENV_KV },
   { "incremental", MU_WRDSF_INCREMENTAL },
+  { "pathexpand", MU_WRDSF_PATHEXPAND },
+  { NULL, 0 }
+};
+
+struct mu_kwd opt_keytab[] = {
+  { "nullglob", MU_WRDSO_NULLGLOB },
+  { "failglob", MU_WRDSO_FAILGLOB },
+  { "dotglob",  MU_WRDSO_DOTGLOB },
   { NULL, 0 }
 };
 
@@ -65,18 +76,27 @@ help ()
 {
   size_t i;
   
-  printf ("usage: wsp [options]\n");
+  printf ("usage: %s [options] [VAR=VALUE...]\n", progname);
   printf ("options are:\n");
   printf (" [-]trimnl\n");
   printf (" [-]plaintext\n");
+  printf (" -env\n");
+  printf (" env sys|none|null\n");
   putchar ('\n');
   for (i = 0; bool_keytab[i].name; i++)
     printf (" [-]%s\n", bool_keytab[i].name);
   putchar ('\n');
   for (i = 0; string_keytab[i].name; i++)
     {
-      printf (" -%s\n", bool_keytab[i].name);
-      printf (" %s ARG\n", bool_keytab[i].name);
+      printf (" -%s\n", string_keytab[i].name);
+      printf (" %s ARG\n", string_keytab[i].name);
+    }
+  printf (" escape-word ARG\n");
+  printf (" escape-quote ARG\n");
+  putchar ('\n');
+  for (i = 0; opt_keytab[i].name; i++)
+    {
+      printf (" [-]%s\n", opt_keytab[i].name);
     }
   putchar ('\n');
   printf (" -dooffs\n");
@@ -104,12 +124,7 @@ print_qword (const char *word, int plaintext)
   if (size >= qlen)
     {
       qlen = size + 1;
-      qbuf = realloc (qbuf, qlen);
-      if (!qbuf)
-	{
-	  mu_error ("not enough memory");
-	  abort ();
-	}
+      qbuf = mu_realloc (qbuf, qlen);
     }
   mu_wordsplit_c_quote_copy (qbuf, word, 0);
   qbuf[size] = 0;
@@ -131,50 +146,179 @@ make_env_kv ()
     ;
 
   size = (i - 1) * 2 + 1;
-  newenv = calloc (size, sizeof (newenv[0]));
-  if (!newenv)
-    {
-      mu_error ("not enough memory");
-      exit (1);
-    }
+  newenv = mu_calloc (size, sizeof (newenv[0]));
 
   for (i = j = 0; environ[i]; i++)
     {
       size_t len = strcspn (environ[i], "=");
-      char *p = malloc (len+1);
-      if (!p)
-	{
-	  mu_error ("not enough memory");
-	  exit (1);
-	}
+      char *p = mu_alloc (len+1);
       memcpy (p, environ[i], len);
       p[len] = 0;
       newenv[j++] = p;
-      p = strdup (environ[i] + len + 1);
-      if (!p)
-	{
-	  mu_error ("not enough memory");
-	  exit (1);
-	}
+      p = mu_strdup (environ[i] + len + 1);
       newenv[j++] = p;
     }
   newenv[j] = NULL;
   return newenv;
 }
-    
+
+static int
+wsp_getvar (char **ret, const char *vptr, size_t vlen, void *data)
+{
+  char **base = data;
+  int i;
+
+  for (i = 0; base[i]; i++)
+    {
+      size_t l = strcspn (base[i], "=");
+      if (l == vlen && memcmp (base[i], vptr, vlen) == 0)
+	{
+	  char *p = strdup (base[i] + vlen + 1);
+	  if (p == NULL)
+	    return MU_WRDSE_NOSPACE;
+	  *ret = p;
+	  return MU_WRDSE_OK;
+	}
+    }
+  return MU_WRDSE_UNDEF;
+}
+
+static int
+wsp_runcmd (char **ret, const char *str, size_t len, char **argv, void *closure)
+{
+  FILE *fp;
+  char *cmd;
+  int c, lastc;
+  char *buffer = NULL;
+  size_t bufsize = 0;
+  size_t buflen = 0;
+  
+  cmd = malloc (len + 1);
+  if (!cmd)
+    return MU_WRDSE_NOSPACE;
+  memcpy (cmd, str, len);
+  cmd[len] = 0;
+
+  fp = popen(cmd, "r");
+  if (!fp)
+    {
+      size_t size = 0;
+      ret = NULL;
+      if (mu_asprintf (ret, &size, "can't run %s: %s",
+		       cmd, strerror (errno)))
+	return MU_WRDSE_NOSPACE;
+      else
+	return MU_WRDSE_USERERR;
+    }
+
+  while ((c = fgetc (fp)) != EOF)
+    {
+      lastc = c;
+      if (c == '\n')
+	c = ' ';
+      if (buflen == bufsize)
+	{
+	  char *p;
+	  
+	  if (bufsize == 0)
+	    bufsize = 80;
+	  else
+	    bufsize *= 2;
+	  p = realloc (buffer, bufsize);
+	  if (!p)
+	    {
+	      free (buffer);
+	      free (cmd);
+	      return MU_WRDSE_NOSPACE;
+	    }
+	  buffer = p;
+	}
+      buffer[buflen++] = c;
+    }
+
+  if (buffer)
+    {
+      if (lastc == '\n')
+	--buflen;
+      buffer[buflen] = 0;
+    }
+  
+  pclose (fp);
+  free (cmd);
+
+  *ret = buffer;
+  return MU_WRDSE_OK;
+}
+
+enum env_type
+  {
+    env_none,
+    env_null,
+    env_sys
+  };
+
+struct mu_kwd env_keytab[] = {
+  { "none",   env_none },
+  { "null",   env_null },
+  { "sys",    env_sys },
+  { NULL }
+};
+
+static void
+set_escape_string (mu_wordsplit_t *ws, int *wsflags, int q, const char *str)
+{
+  if (*str == ':')
+    {
+      while (*++str != ':')
+	{
+	  int f;
+	  switch (*str)
+	    {
+	    case '+':
+	      f = MU_WRDSO_BSKEEP;
+	      break;
+
+	    case '0':
+	      f = MU_WRDSO_OESC;
+	      break;
+
+	    case 'x':
+	      f = MU_WRDSO_XESC;
+	      break;
+
+	    default:
+	      fprintf (stderr, "%s: invalid escape flag near %s\n",
+		       progname, str);
+	      abort ();
+	    }
+	  MU_WRDSO_ESC_SET (ws, q, f);
+	}
+      *wsflags |= MU_WRDSF_OPTIONS;
+      ++str;
+    }
+  ws->ws_escape[q] = str;
+}
+
 int
 main (int argc, char **argv)
 {
-  char buf[1024], *ptr;
+  char buf[1024], *ptr, *saved_ptr;
   int i, offarg = 0;
   int trimnl_option = 0;
   int plaintext_option = 0;
   int wsflags = (MU_WRDSF_DEFFLAGS & ~MU_WRDSF_NOVAR) |
                  MU_WRDSF_ENOMEMABRT |
-                 MU_WRDSF_ENV | MU_WRDSF_SHOWERR;
-  struct mu_wordsplit ws;
+                 MU_WRDSF_SHOWERR;
+  mu_wordsplit_t ws;
   int next_call = 0;
+  char *fenvbase[128];
+  size_t fenvidx = 0;
+  size_t fenvmax = sizeof (fenvbase) / sizeof (fenvbase[0]);
+  int use_env = env_sys;
+  
+  progname = argv[0];
 
+  ws.ws_options = 0;
   for (i = 1; i < argc; i++)
     {
       char *opt = argv[i];
@@ -212,7 +356,31 @@ main (int argc, char **argv)
 	  plaintext_option = !negate;
 	  continue;
 	}
-    
+
+      if (strcmp (opt, "env") == 0)
+	{
+	  if (negate)
+	    use_env = env_none;
+	  else
+	    {
+	      i++;
+	      if (i == argc)
+		{
+		  fprintf (stderr, "%s: missing argument for env\n",
+			   progname);
+		  exit (1);
+		}
+
+	      if (mu_kwd_xlat_name (env_keytab, argv[i], &use_env))
+		{
+		  fprintf (stderr, "%s: invalid argument for env\n",
+			   progname);
+		  exit (1);
+		}
+	    }
+	  continue;
+	}
+      
       if (mu_kwd_xlat_name (bool_keytab, opt, &flag) == 0)
 	{
 	  if (negate)
@@ -231,7 +399,8 @@ main (int argc, char **argv)
 	      i++;
 	      if (i == argc)
 		{
-		  mu_error ("%s missing argument", opt);
+		  fprintf (stderr, "%s: missing argument for %s\n",
+			   progname, opt);
 		  exit (1);
 		}
 	      
@@ -246,7 +415,8 @@ main (int argc, char **argv)
 		  break;
 
 		case MU_WRDSF_ESCAPE:
-		  ws.ws_escape = argv[i];
+		  set_escape_string (&ws, &wsflags, 0, argv[i]);
+		  set_escape_string (&ws, &wsflags, 1, argv[i]);
 		  break;
 		}
 	      
@@ -255,6 +425,27 @@ main (int argc, char **argv)
 	  continue;
 	}
 
+      if (strcmp (opt, "escape-word") == 0
+	  || strcmp (opt, "escape-quote") == 0)
+	{
+	  int q = opt[7] == 'q';
+	  
+	  i++;
+	  if (i == argc)
+	    {
+	      fprintf (stderr, "%s: missing argument for %s\n",
+		       progname, opt);
+	      exit (1);
+	    }
+	  if (!(wsflags & MU_WRDSF_ESCAPE))
+	    {
+	      wsflags |= MU_WRDSF_ESCAPE;
+	      ws.ws_escape[!q] = NULL;
+	    }
+	  set_escape_string (&ws, &wsflags, q, argv[i]);
+	  continue;
+	}
+
       if (strcmp (opt, "dooffs") == 0)
 	{
 	  if (negate)
@@ -267,20 +458,23 @@ main (int argc, char **argv)
 	      
 	      if (i == argc)
 		{
-		  mu_error ("%s missing arguments", opt);
+		  fprintf (stderr, "%s: missing arguments for %s\n",
+			   progname, opt);
 		  exit (1);
 		}
 	      ws.ws_offs = strtoul (argv[i], &p, 10);
 	      if (*p)
 		{
-		  mu_error ("invalid number: %s", argv[i]);
+		  fprintf (stderr, "%s: invalid number: %s\n",
+			   progname, argv[i]);
 		  exit (1);
 		}
 
 	      i++;
 	      if (i + ws.ws_offs > argc)
 		{
-		  mu_error ("%s: not enough arguments", opt);
+		  fprintf (stderr, "%s: not enough arguments for %s\n",
+			   progname, opt);
 		  exit (1);
 		}
 	      offarg = i;
@@ -290,15 +484,65 @@ main (int argc, char **argv)
 	  continue;
 	}
 
-      mu_error ("%s: unrecognized argument", opt);
+      if (mu_kwd_xlat_name (opt_keytab, opt, &flag) == 0)
+	{
+	  wsflags |= MU_WRDSF_OPTIONS;
+	  if (negate)
+	    ws.ws_options &= ~flag;
+	  else
+	    ws.ws_options |= flag;
+	  continue;
+	}
+      
+      if (strchr (opt, '='))
+	{
+	  if (fenvidx < fenvmax - 1)
+	    {
+	      fenvbase[fenvidx++] = opt;
+	      continue;
+	    }
+	  else
+	    {
+	      fprintf (stderr, "%s: environment too big\n", progname);
+	      exit (1);
+	    }
+	}
+      
+      fprintf (stderr, "%s: unrecognized argument: %s\n",
+	       progname, opt);
       exit (1);
     }
 
-  if (wsflags & MU_WRDSF_ENV_KV)
-    ws.ws_env = (const char **) make_env_kv ();
-  else
-    ws.ws_env = (const char **) environ;
+  if (fenvidx)
+    {
+      fenvbase[fenvidx] = NULL;
+      wsflags |= MU_WRDSF_GETVAR | MU_WRDSF_CLOSURE;
+      ws.ws_getvar = wsp_getvar;
+      ws.ws_closure = fenvbase;
+    }
+
+  switch (use_env)
+    {
+    case env_null:
+      wsflags |= MU_WRDSF_ENV;
+      ws.ws_env = NULL;
+      break;
 
+    case env_none:
+      break;
+
+    case env_sys:
+      wsflags |= MU_WRDSF_ENV;
+      if (wsflags & MU_WRDSF_ENV_KV)
+	ws.ws_env = (const char **) make_env_kv ();
+      else
+	ws.ws_env = (const char **) environ;
+      break;
+    }
+  
+  if (!(wsflags & MU_WRDSF_NOCMD))
+    ws.ws_command = wsp_runcmd;
+  
   if (wsflags & MU_WRDSF_INCREMENTAL)
     trimnl_option = 1;
   
@@ -309,7 +553,11 @@ main (int argc, char **argv)
       size_t i;
       
       if (trimnl_option)
-	mu_rtrim_cset (ptr, "\n");
+	{
+	  size_t len = strlen (ptr);
+	  if (len && ptr[len-1] == '\n')
+	    ptr[len-1] = 0;
+	}
       
       if (wsflags & MU_WRDSF_INCREMENTAL)
 	{
@@ -318,16 +566,12 @@ main (int argc, char **argv)
 	      if (*ptr == 0)
 		ptr = NULL;
 	      else
-		free ((void*)ws.ws_input);
+		free (saved_ptr);
 	    }
 	  else
 	    next_call = 1;
 	  if (ptr)
-	    {
-	      ptr = strdup (ptr);
-	      if (!ptr)
-		abort ();
-	    }
+	    ptr = saved_ptr = mu_strdup (ptr);
 	}
 	
       rc = mu_wordsplit (ptr, &ws, wsflags);
@@ -344,8 +588,8 @@ main (int argc, char **argv)
 	    ws.ws_wordv[i] = argv[offarg + i];
 	  offarg = 0;
 	}
-      
-      wsflags |= MU_WRDSF_REUSE;
+
+      wsflags |= MU_WRDSF_REUSE | (ws.ws_flags & MU_WRDSF_ENV);
       printf ("NF: %lu", (unsigned long) ws.ws_wordc);
       if (wsflags & MU_WRDSF_DOOFFS)
 	printf (" (%lu)", (unsigned long) ws.ws_offs);

@@ -252,12 +252,16 @@ shell_help (int argc, char **argv)
 static int
 shell_prompt (int argc, char **argv)
 {
-  size_t size;
+  mu_wordsplit_t ws;
   
-  free (mutool_shell_prompt);
-  size = strlen (argv[1]);
-  mutool_shell_prompt = mu_alloc (size + 1);
-  mu_wordsplit_c_unquote_copy (mutool_shell_prompt, argv[1], size);
+  if (mu_wordsplit (argv[1], &ws, MU_WRDSF_NOSPLIT | MU_WRDSF_DEFFLAGS))
+    mu_error ("mu_wordsplit: %s", mu_wordsplit_strerror (&ws));
+  else
+    {
+      free (mutool_shell_prompt);
+      mutool_shell_prompt = mu_strdup (ws.ws_wordv[0]);
+    }
+  mu_wordsplit_free (&ws);
   return 0;
 }
 
@@ -521,7 +525,7 @@ execute_line (char *line)
   int status = 0;
   
   ws.ws_comment = "#";
-  ws.ws_escape = "\\\"";
+  ws.ws_escape[0] = ws.ws_escape[1] = "\\\\\"\"";
   rc = mu_wordsplit (line, &ws,
 		     MU_WRDSF_DEFFLAGS|MU_WRDSF_COMMENT|MU_WRDSF_ESCAPE|
 		     MU_WRDSF_INCREMENTAL|MU_WRDSF_APPEND);