Improve mu_scan_datetime

* libmailutils/base/date.c (mu_scan_datetime): Handle optional
blocks (%[ %| %], alternatives %( %| %), "any character"
wildcards (%?) and strict character matches (%\C).  Return
MU_ERR_FORMAT on errors in format string.
Compute tm->tm_yday.
* libmailutils/diag/errors (MU_ERR_FORMAT): New error code.
* libmailutils/tests/scantime.c: Print yday.  Print input line with
diagnostic messages.
* libmailutils/tests/scantime.at: Add more tests.
* include/mailutils/util.h (MU_DATETIME_RFC822): Split into two
formats: MU_DATETIME_FORM_RFC822, for output formatting, and
MU_DATETIME_SCAN_RFC822 for input scanning.  All uses changed.

@@ -75,9 +75,12 @@ int mu_scan_datetime (const char *input, const char *fmt, struct tm *tm,
 #define MU_DATETIME_FROM         "%a %b %e %H:%M:%S %Y"
 #define MU_DATETIME_IMAP         "%d-%b-%Y %H:%M:%S %z"
 #define MU_DATETIME_INTERNALDATE "%d-%b-%Y%$ %H:%M:%S %z"
-  /* FIXME: [%a, ] part is actually optional */
-#define MU_DATETIME_RFC822       "%a, %e %b %Y %H:%M:%S %z"
 
+  /* RFC2822 date.  Scan format contains considerable allowances which would
+     stun formatting functions, therefore two distinct formats are provided:
+     one for outputting and one for scanning: */
+#define MU_DATETIME_FORM_RFC822  "%a, %e %b %Y %H:%M:%S %z"  
+#define MU_DATETIME_SCAN_RFC822  "%[%a, %]%e %b %Y %H:%M%[:%S%] %z"
   
   /* ----------------------- */
   /* File & path names.      */

@@ -22,6 +22,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <mailutils/diag.h>
 #include <mailutils/util.h>
 #include <mailutils/stream.h>
 #include <mailutils/errno.h>
@@ -53,8 +54,7 @@ static int  month_start[]=
          31  28  31   30   31   30   31   31   30   31   30   31
      */
 
-/* NOTE: ignore GCC warning. The precedence of operators is OK here */
-#define leap_year(y) ((y) % 4 == 0 && (y) % 100 != 0 || (y) % 400 == 0)
+#define leap_year(y) ((y) % 4 == 0 && ((y) % 100 != 0 || (y) % 400 == 0))
 
 static int
 dayofyear (int year, int month, int day)
@@ -625,6 +625,203 @@ get_num (const char *str, char **endp, int ndig, int minval, int maxval,
 #define DT_MIN   0x20
 #define DT_SEC   0x40
 
+#define ST_NON   -1
+#define ST_OPT   0
+#define ST_ALT   1
+
+struct save_input
+{
+  int state;
+  const char *input;
+};
+
+static int
+push_input (mu_list_t *plist, int state, const char *input)
+{
+  mu_list_t list = *plist;
+  struct save_input *inp = malloc (sizeof (*inp));
+  if (!inp)
+    return ENOMEM;
+  if (!list)
+    {
+      int rc = mu_list_create (&list);
+      if (rc)
+	{
+	  free (inp);
+	  return rc;
+	}
+      mu_list_set_destroy_item (list, mu_list_free_item);
+      *plist = list;
+    }
+  inp->state = state;
+  inp->input = input;
+  return mu_list_push (list, (void*)inp);
+}
+
+static int
+peek_state (mu_list_t list, int *state, const char **input)
+{
+  int rc;
+  struct save_input *inp;
+
+  rc = mu_list_tail (list, (void**)&inp);
+  if (rc)
+    return rc;
+  *state = inp->state;
+  if (input)
+    *input = inp->input;
+  return 0;
+}      
+
+static int
+change_top_input (mu_list_t list, const char *input)
+{
+  int rc;
+  struct save_input *inp;
+
+  rc = mu_list_tail (list, (void**)&inp);
+  if (rc)
+    return rc;
+  inp->input = input;
+  return 0;
+}
+
+static int
+pop_input (mu_list_t list, int *state, const char **input)
+{
+  int rc;
+  struct save_input *inp;
+
+  rc = mu_list_pop (list, (void**)&inp);
+  if (rc)
+    return rc;
+  *state = inp->state;
+  if (input)
+    *input = inp->input;
+  return 0;
+}
+
+static int
+bracket_to_state (int c)
+{
+  switch (c)
+    {
+    case '[':
+    case ']':
+      return ST_OPT;
+    case '(':
+    case ')':
+      return ST_ALT;
+    }
+  return ST_NON;
+}
+
+static int
+state_to_closing_bracket (int st)
+{
+  switch (st)
+    {
+    case ST_OPT:
+      return ']';
+    case ST_ALT:
+      return ')';
+    }
+  return '?';
+}
+
+static int
+scan_recovery (const char *fmt, mu_list_t *plist, int skip_alt,
+	       const char **endp,
+	       const char **input)
+{
+  int c, rc = 0;
+  int nesting_level = 1;
+  int st;
+  const char *p;
+  
+  while (*fmt)
+    {
+      c = *fmt++;
+      
+      if (c == '%')
+	{
+	  c = *fmt++;
+	  if (!c)
+	    {
+	      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+			("%s:%d: error in format: %% at the end of input",
+			 __FILE__, __LINE__));
+	      rc = MU_ERR_FORMAT;
+	      break;
+	    }
+	      
+	  switch (c)
+	    {
+	    case '[':
+	    case '(':
+	      nesting_level++;
+	      rc = push_input (plist, bracket_to_state (c), NULL);
+	      break;
+	      
+	    case ')':
+	    case ']':
+	      rc = pop_input (*plist, &st, &p);
+	      if (rc || st != bracket_to_state (c))
+		{
+		  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+			    ("%s:%d: error in format: %%%c out of context",
+			     __FILE__, __LINE__, c));
+		  rc = MU_ERR_FORMAT;
+		  break;
+		}
+	      if (--nesting_level == 0)
+		{
+		  *endp = fmt;
+		  if (skip_alt)
+		    return 0;
+		  *input = p;
+		  if (st == ST_ALT)
+		    {
+		      if (*fmt == '%' && (fmt[1] == '|' || fmt[1] == ']'))
+			return 0;
+		      return MU_ERR_PARSE; /* No match found */
+		    }
+		  return 0;
+		}
+	      break;
+
+	    case '|':
+	      if (skip_alt)
+		continue;
+	      if (nesting_level == 1)
+		{
+		  *endp = fmt;
+		  return peek_state (*plist, &st, input);
+		}
+	      break;
+
+	    case '\\':
+	      if (*++fmt == 0)
+		{
+		  peek_state (*plist, &st, NULL);
+		  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+			    ("%s:%d: error in format: missing closing %%%c",
+			     __FILE__, __LINE__,
+			     state_to_closing_bracket (st)));
+		  return MU_ERR_FORMAT;		  
+		}
+	    }
+	}
+    }
+  
+  peek_state (*plist, &st, NULL);
+  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+	    ("%s:%d: error in format: missing closing %%%c",
+	     __FILE__, __LINE__,
+	     state_to_closing_bracket (st)));
+  return MU_ERR_FORMAT;		  
+}
+
 int
 mu_scan_datetime (const char *input, const char *fmt,
 		  struct tm *tm, struct mu_timezone *tz, char **endp)
@@ -632,8 +829,12 @@ mu_scan_datetime (const char *input, const char *fmt,
   int rc = 0;
   char *p;
   int n;
+  int c;
+  int st;
+  int recovery = 0;
   int eof_ok = 0;
   int datetime_parts = 0;
+  mu_list_t save_input_list = NULL;
   
   memset (tm, 0, sizeof *tm);
 #ifdef HAVE_STRUCT_TM_TM_ISDST
@@ -661,7 +862,17 @@ mu_scan_datetime (const char *input, const char *fmt,
       
       if (*fmt == '%')
 	{
-	  switch (*++fmt)
+	  c = *++fmt;
+	  if (!c)
+	    {
+	      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+			("%s:%d: error in format: %% at the end of input",
+			 __FILE__, __LINE__));
+	      rc = MU_ERR_FORMAT;
+	      break;
+	    }
+	  
+	  switch (c)
 	    {
 	    case 'a':
 	      /* The abbreviated weekday name. */
@@ -852,27 +1063,110 @@ mu_scan_datetime (const char *input, const char *fmt,
 	      else
 		rc = MU_ERR_PARSE;
 	      break;
+
+	      rc = push_input (&save_input_list, ST_ALT, (void*)input);
+	      break;
+	      
+	    case '(':
+	    case '[':
+	      rc = push_input (&save_input_list, bracket_to_state (c),
+			       (void*)input);
+	      break;
+
+	    case ')':
+	    case ']':
+	      if (pop_input (save_input_list, &st, NULL))
+		{
+		  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+			    ("%s:%d: error in format: unbalanced %%%c near %s",
+			     __FILE__, __LINE__, c, fmt));
+		  rc = MU_ERR_FORMAT;
+		}
+	      else if (st != bracket_to_state (c))
+		{
+		  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+			    ("%s:%d: error in format: %%%c out of context",
+			     __FILE__, __LINE__, c));
+		  rc = MU_ERR_FORMAT;
+		}
+	      break;
+
+	    case '|':
+	      rc = scan_recovery (fmt, &save_input_list, 1, &fmt, NULL);
+	      if (rc == 0)
+		fmt--;
+	      break;
 	      
 	    case '$':
 	      eof_ok = 1;
 	      break;
+
+	    case '\\':
+	      c = *++fmt;
+	      if (!c)
+		{
+		  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+			    ("%s:%d: error in format: %% at the end of input",
+			     __FILE__, __LINE__));
+		  rc = MU_ERR_FORMAT;
+		}
+	      else if (c == *input)
+		input++;
+	      else
+		rc = MU_ERR_PARSE;
+	      break;
+
+	    case '?':
+	      input++;
+	      break;
+	      
+	    default:
+	      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+			("%s:%d: error in format: unrecognized conversion type"
+			 " near %s",
+			 __FILE__, __LINE__, fmt));
+	      rc = MU_ERR_FORMAT;
+	      break;
 	    }
+
 	  if (eof_ok && rc == 0 && *input == 0)
 	    break;
 	}
-      else if (*input != *fmt)
+      else if (!recovery && *input != *fmt)
 	rc = MU_ERR_PARSE;
       else
 	input++;
+
+      if (rc == MU_ERR_PARSE && !mu_list_is_empty (save_input_list))
+	{
+	  rc = scan_recovery (fmt, &save_input_list, 0, &fmt, &input);
+	  if (rc == 0)
+	    --fmt;
+	}
     }
 
+  if (!mu_list_is_empty (save_input_list))
+    {
+      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		("%s:%d: error in format: closing bracket missing",
+		 __FILE__, __LINE__));
+      rc = MU_ERR_FORMAT;
+    }
+  mu_list_destroy (&save_input_list);
+  
+  if (rc == 0 && recovery)
+    rc = MU_ERR_PARSE;
+  
   if (!eof_ok && rc == 0 && *input == 0 && *fmt)
     rc = MU_ERR_PARSE;
 
-  if (!(datetime_parts & DT_WDAY) &&
-      (datetime_parts & (DT_YEAR|DT_MONTH|DT_MDAY)) ==
+  if ((datetime_parts & (DT_YEAR|DT_MONTH|DT_MDAY)) ==
       (DT_YEAR|DT_MONTH|DT_MDAY))
-    tm->tm_wday = dayofweek (tm->tm_year + 1900, tm->tm_mon, tm->tm_mday);
+    {
+      if (!(datetime_parts & DT_WDAY))
+	tm->tm_wday = dayofweek (tm->tm_year + 1900, tm->tm_mon, tm->tm_mday);
+      tm->tm_yday = dayofyear (tm->tm_year + 1900, tm->tm_mon, tm->tm_mday);
+    }
   
   if (endp)
     *endp = (char*) input;

@@ -123,3 +123,5 @@ MU_ERR_PERM_DIR_IWGRP       _("File in group writable directory")
 MU_ERR_PERM_DIR_IWOTH       _("File in world writable directory")
 
 MU_ERR_DISABLED             _("Requested feature disabled in configuration")
+
+MU_ERR_FORMAT		    _("Error in format string")

@@ -30,34 +30,127 @@ m4_popdef([MU_TEST_GROUP])
 ])
 dnl ---------------------------------------------------------------------
 
-SCANTIME([RFC-822 time],[rfc822],
+SCANTIME([Envelope (From) time],[envelope],
 [%a %b %e %H:%M:%S %Y],
 [Tue May  3 13:25:26 2011
 Fri Nov 11 11:55:01 2011],
-[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,tz=0
-sec=1,min=55,hour=11,mday=11,mon=10,year=111,wday=5,tz=0
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=0
+sec=1,min=55,hour=11,mday=11,mon=10,year=111,wday=5,yday=315,tz=0
 ])
 
-SCANTIME([IMAP time format],[imap],
+SCANTIME([IMAP INTERNALDATE],[imap-internaldate],
 [%d-%b-%Y %H:%M:%S %z],
 [03-May-2011 13:25:26 +0100
 11-Nov-2011 11:55:01 +0100],
-[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,tz=3600
-sec=1,min=55,hour=11,mday=11,mon=10,year=111,wday=5,tz=3600
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=3600
+sec=1,min=55,hour=11,mday=11,mon=10,year=111,wday=5,yday=315,tz=3600
 ])
 
-SCANTIME([IMAP search time format],[imap-search],
+SCANTIME([IMAP INTERNALDATE (optional time)],[imap-search],
 [%d-%b-%Y%$ %H:%M:%S %z],
 [03-May-2011 13:25:26 +0100
 03-May-2011],
-[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,tz=3600
-sec=0,min=0,hour=0,mday=3,mon=4,year=111,wday=2,tz=0
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=3600
+sec=0,min=0,hour=0,mday=3,mon=4,year=111,wday=2,yday=123,tz=0
 ])
 
-SCANTIME([IMAP INTERNALDATE],[imap-internaldate],
+SCANTIME([RFC-822 Strict],[rfc822-strict],
 [%a, %d %b %Y %H:%M:%S %z],
 [Tue, 03 May 2011 13:25:26 +0200],
-[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,tz=7200
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+])
+
+SCANTIME([RFC-822],[rfc822],
+[[%[%a, %]%d %b %Y %H:%M%[:%S%] %z]],
+[Tue, 03 May 2011 13:25:26 +0200
+03 May 2011 13:25:26 +0200
+Tue, 03 May 2011 13:25 +0200
+03 May 2011 13:25 +0200],
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=0,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=0,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+])
+
+SCANTIME([Any char],[anychar],
+[%a%? %d %b %Y %H:%M:%S %z],
+[Tue, 03 May 2011 13:25:26 +0200
+Tue: 03 May 2011 13:25:26 +0200
+Tue; 03 May 2011 13:25:26 +0200
+Tue 03 May 2011 13:25:26 +0200],
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+])
+
+SCANTIME([Percent],[percent],
+[%d%%%b%%%Y %H:%M:%S %z],
+[03%May%2011 13:25:26 +0100],
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=3600
+])
+
+SCANTIME([Fixed WS],[fixws],
+[%d-%b-%Y%\ %H:%M:%S %z],
+[03-May-2011 13:25:26 +0100
+03-May-2011  13:25:26 +0100],
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=3600
+],
+[scantime: 2: parse failed near  13:25:26 +0100
+])
+
+SCANTIME([endp return],[endp],
+[%a, %d %b %Y %H:%M:%S %z],
+[Tue, 03 May 2011 13:25:26 +0100 other data
+],
+[# 1: stopped at  other data
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=3600
+])
+
+SCANTIME([Optional blocks],[opt],
+[[%[%a, %]%d %b %Y %H:%M:%S %z]],
+[Tue, 03 May 2011 13:25:26 +0200
+03 May 2011 13:25:26 +0200],
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+])
+
+SCANTIME([Nested optional blocks],[nested-opt],
+[[%[%a%[,%] %]%d %b %Y %H:%M:%S %z]],
+[Tue, 03 May 2011 13:25:26 +0200
+Tue 03 May 2011 13:25:26 +0200
+03 May 2011 13:25:26 +0200],
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+])
+
+SCANTIME([Optional alternatives],[opt-alt],
+[[%a%[,%|:%] %d %b %Y %H:%M:%S %z]],
+[Tue, 03 May 2011 13:25:26 +0200
+Tue: 03 May 2011 13:25:26 +0200
+Tue 03 May 2011 13:25:26 +0200
+Tue; 03 May 2011 13:25:26 +0200],
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+],
+[scantime: 4: parse failed near ; 03 May 2011 13:25:26 +0200
+])
+
+SCANTIME([Alternatives],[alt],
+[%a%(,%|:%|/%) %d %b %Y %H:%M:%S %z],
+[Tue, 03 May 2011 13:25:26 +0200
+Tue: 03 May 2011 13:25:26 +0200
+Tue/ 03 May 2011 13:25:26 +0200
+Tue 03 May 2011 13:25:26 +0200
+Tue; 03 May 2011 13:25:26 +0200],
+[sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+sec=26,min=25,hour=13,mday=3,mon=4,year=111,wday=2,yday=123,tz=7200
+],
+[scantime: 4: parse failed near  03 May 2011 13:25:26 +0200
+scantime: 5: parse failed near ; 03 May 2011 13:25:26 +0200
 ])
 
 m4_popdef([SCANTIME])

@@ -45,7 +45,8 @@ main (int argc, char **argv)
   char *buf = NULL;
   size_t size = 0;
   size_t n;
-
+  int line;
+  
   mu_set_program_name (argv[0]);
   
   mu_stdstream_setup (MU_STDSTREAM_RESET_NONE);
@@ -65,31 +66,34 @@ main (int argc, char **argv)
 	}
     }
 
+  line = 0;
   while ((rc = mu_stream_getline (mu_strin, &buf, &size, &n)) == 0 && n > 0)
     {
       char *endp;
       struct tm tm;
       struct mu_timezone tz;
 
+      line++;
+      mu_ltrim_class (buf, MU_CTYPE_BLANK);
       mu_rtrim_class (buf, MU_CTYPE_ENDLN);
-
+      if (!*buf)
+	continue;
       rc = mu_scan_datetime (buf, format, &tm, &tz, &endp);
       if (rc)
 	{
 	  if (*endp)
-	    mu_error ("parse failed near %s", endp);
+	    mu_error ("%d: parse failed near %s", line, endp);
 	  else
-	    mu_error ("parse failed at end of input");
+	    mu_error ("%d: parse failed at end of input", line);
 	  continue;
 	}
       if (*endp)
-	mu_printf ("# stopped at %s\n", endp);
-      /* FIXME: add tm_yday? */
-      mu_printf ("sec=%d,min=%d,hour=%d,mday=%d,mon=%d,year=%d,wday=%d,tz=%d\n",
+	mu_printf ("# %d: stopped at %s\n", line, endp);
+      mu_printf ("sec=%d,min=%d,hour=%d,mday=%d,mon=%d,year=%d,wday=%d,yday=%d,tz=%d\n",
 		 tm.tm_sec, tm.tm_min, tm.tm_hour, tm.tm_mday, tm.tm_mon,
-		 tm.tm_year, tm.tm_wday, tz.utc_offset);
+		 tm.tm_year, tm.tm_wday, tm.tm_yday, tz.utc_offset);
 		 
-      //mu_c_streamftime (mu_strout, "%c %z%n", &tm, &tz);
+      /*mu_c_streamftime (mu_strout, "%c %z%n", &tm, &tz);*/
     }
   
   if (rc)

@@ -472,7 +472,7 @@ _fill_response (void *item, void *data)
       else
 	{
 	  if (mu_scan_datetime (elt->v.string,
-				MU_DATETIME_RFC822,
+				MU_DATETIME_SCAN_RFC822,
 				&env->envelope->date,
 				&env->envelope->tz, NULL))
 	    rc = MU_ERR_FAILURE;