Rewrite sieve parser.

Three objectives:

1. Simplify code.
2. Produce optimized sieve code.
3. Improve error reporting.
4. Prepare for further extensions

* include/mailutils/sieve.h (mu_sieve_tag_checker_t): Change
signature (take mu_sieve_machine_t as the first arg).  All
uses changed.
(mu_sieve_require): Likewise.
(mu_sieve_yydebug): Remove global.

* libmu_sieve/sieve-priv.h (mu_locus_range): New struct.
(YYLTYPE): New define
(mu_sieve_state): New enum.
(mu_sieve_machine): New members: string_pool, state.
(mu_sieve_node_type): New enum.
(mu_sieve_node): New struct.
Remove unused prototypes.

* libmu_sieve/sieve.l: Keep track of code locations.  Use opool
for constructing string values.
* libmu_sieve/sieve.y: Rewrite.  First build the parse tree.  Then
optimize it.  Finally, generate code.
* libmu_sieve/tests.c (sieve_test_true,sieve_test_false): Remove.
True and false tests are always optimized away.
* libmu_sieve/util.c (mu_sv_compile_error): Remove.

* libmu_sieve/actions.c: Use mu_diag_at_locus to report errors
and mu_i_sv_error to mark sieve machine as being in error state.
* libmu_sieve/comparator.c: Likewise.
* libmu_sieve/prog.c (mu_sv_code): Replace with mu_i_sv_code.
(mu_sv_code_instr,mu_sv_code_handler)
(mu_sv_code_list,mu_sv_code_number)
(mu_sv_code_string,mu_sv_code_source)
(mu_sv_code_line,mu_sv_change_source)
(mu_sv_code_action,mu_sv_code_test)
(mu_sv_code_anyof,mu_sv_code_allof): Remove.
(mu_i_sv_locus,mu_i_sv_code_action)
(mu_i_sv_code_test): New function.
(mu_sv_code_command): Replace with a static function.
* libmu_sieve/require.c (mu_sieve_require): Take ptr to machine
as the first arg.
* libmu_sieve/runtime.c (mu_sieve_mailbox)
(mu_sieve_message): Refuse to run if the machine is in error state.

* sieve/sieve.c: Update.
* sieve/tests/i-numeric.at: Update expected error message.

* libmailutils/diag/diag.c (mu_diag_at_locus): Don't pass locus
if mu_file is NULL.
* libmu_auth/ldap.c (_mu_entry_to_auth_data): Remove leftover
mu_error.

@@ -47,7 +47,8 @@ typedef int (*mu_sieve_comparator_t) (const char *, const char *);
 typedef int (*mu_sieve_retrieve_t) (void *item, void *data, int idx,
 				    char **pval);
 typedef void (*mu_sieve_destructor_t) (void *data);
-typedef int (*mu_sieve_tag_checker_t) (const char *name,
+typedef int (*mu_sieve_tag_checker_t) (mu_sieve_machine_t mach,
+				       const char *name,
 				       mu_list_t tags, mu_list_t args);
 
 typedef enum
@@ -121,7 +122,6 @@ typedef struct
 #define MU_SIEVE_DEBUG_DISAS  0x0004
 #define MU_SIEVE_DRY_RUN      0x0008
 
-extern int mu_sieve_yydebug;
 extern mu_debug_handle_t mu_sieve_debug_handle;
 extern mu_list_t mu_sieve_include_path;
 extern mu_list_t mu_sieve_library_path;
@@ -190,13 +190,16 @@ int mu_sieve_str_to_relcmp (const char *str, mu_sieve_relcmp_t * test,
 mu_sieve_relcmp_t mu_sieve_get_relcmp (mu_sieve_machine_t mach,
 				       mu_list_t tags);
 
-void mu_sieve_require (mu_list_t slist);
+void mu_sieve_require (mu_sieve_machine_t mach, mu_list_t slist);
+
 int mu_sieve_tag_lookup (mu_list_t taglist, char *name,
 			 mu_sieve_value_t ** arg);
 int mu_sieve_load_ext (mu_sieve_machine_t mach, const char *name);
-int mu_sieve_match_part_checker (const char *name, mu_list_t tags,
+int mu_sieve_match_part_checker (mu_sieve_machine_t mach,
+				 const char *name, mu_list_t tags,
 				 mu_list_t args);
-int mu_sieve_match_part_checker (const char *name, mu_list_t tags,
+int mu_sieve_match_part_checker (mu_sieve_machine_t mach,
+				 const char *name, mu_list_t tags,
 				 mu_list_t args);
 /* Operations in value lists */
 mu_sieve_value_t *mu_sieve_value_get (mu_list_t vlist, size_t index);

@@ -61,10 +61,11 @@ mu_diag_at_locus (int level, struct mu_locus const *loc, const char *fmt, ...)
   va_list ap;
 
   va_start (ap, fmt);
-  mu_stream_printf (mu_strerr, "\033f<%d>%s\033l<%u>\033c<%u>",
-		    (unsigned) strlen (loc->mu_file), loc->mu_file,
-		    loc->mu_line,
-		    loc->mu_col);
+  if (loc && loc->mu_file)
+    mu_stream_printf (mu_strerr, "\033f<%d>%s\033l<%u>\033c<%u>",
+		      (unsigned) strlen (loc->mu_file), loc->mu_file,
+		      loc->mu_line,
+		      loc->mu_col);
   mu_diag_voutput (level, fmt, ap);
   va_end (ap);
 }

@@ -538,8 +538,6 @@ _mu_entry_to_auth_data (LDAP *ld, LDAPMessage *msg,
   
   rc = ldap_get_dn_ber (ld, msg, &ber, &bv);
   ufn = ldap_dn2ufn (bv.bv_val);
-  /* FIXME: Use debug or diag functions */
-  mu_error ("INFO: %s", ufn);
   ldap_memfree (ufn);
   
   mu_assoc_get_iterator (ldap_param.field_map, &itr);

@@ -17,6 +17,7 @@
 
 YLWRAP = $(SHELL) $(mu_aux_dir)/gylwrap
 AM_YFLAGS = -dtv
+#AM_LEXFLAGS=-dvp
 AM_CPPFLAGS =\
  @MU_LIB_COMMON_INCLUDES@\
  -DMU_SIEVE_MODDIR=\"@MU_SIEVE_MODDIR@\"

@@ -522,7 +522,8 @@ mu_sieve_data_type fileinto_args[] = {
 };
 
 static int
-perms_tag_checker (const char *name, mu_list_t tags, mu_list_t args)
+perms_tag_checker (mu_sieve_machine_t mach,
+		   const char *name, mu_list_t tags, mu_list_t args)
 {
   mu_iterator_t itr;
   int err = 0;
@@ -540,8 +541,9 @@ perms_tag_checker (const char *name, mu_list_t tags, mu_list_t args)
 	{
 	  if (mu_parse_stream_perm_string (&flag, t->arg->v.string, &p))
 	    {
-	      mu_sv_compile_error (&mu_sieve_locus, 
-				   _("invalid permissions (near %s)"), p);
+	      mu_diag_at_locus (MU_LOG_ERROR, &mach->locus,
+				_("invalid permissions (near %s)"), p);
+	      mu_i_sv_error (mach);
 	      err = 1;
 	    }
 	}

@@ -139,7 +139,9 @@ mu_sieve_get_comparator (mu_sieve_machine_t mach, mu_list_t tags)
 
 /* Compile time support */
 
-struct regex_data {
+struct regex_data
+{
+  mu_sieve_machine_t mach;
   int flags;
   mu_list_t list;
 };
@@ -158,7 +160,7 @@ _regex_compile (void *item, void *data)
 {
   struct regex_data *rd = data;
   int rc;
-  regex_t *preg = mu_sieve_malloc (mu_sieve_machine, sizeof (*preg));
+  regex_t *preg = mu_sieve_malloc (rd->mach, sizeof (*preg));
   
   rc = regcomp (preg, (char*)item, rd->flags);
   if (rc)
@@ -168,11 +170,13 @@ _regex_compile (void *item, void *data)
       if (errbuf)
 	{
 	  regerror (rc, preg, errbuf, size);
-	  mu_sv_compile_error (&mu_sieve_locus, _("regex error: %s"), errbuf);
+	  mu_diag_at_locus (MU_LOG_ERROR, &rd->mach->locus,
+			    _("regex error: %s"), errbuf);
 	  free (errbuf);
 	}
       else
-	mu_sv_compile_error (&mu_sieve_locus, _("regex error"));
+	mu_diag_at_locus (MU_LOG_ERROR, &rd->mach->locus, _("regex error"));
+      mu_i_sv_error (rd->mach);
       return rc;
     }
 
@@ -203,7 +207,8 @@ comp_false (const char *pattern, const char *text)
 }
 
 int
-mu_sieve_match_part_checker (const char *name, mu_list_t tags, mu_list_t args)
+mu_sieve_match_part_checker (mu_sieve_machine_t mach,
+			     const char *name, mu_list_t tags, mu_list_t args)
 {
   mu_iterator_t itr;
   mu_sieve_runtime_tag_t *match = NULL;
@@ -233,9 +238,10 @@ mu_sieve_match_part_checker (const char *name, mu_list_t tags, mu_list_t args)
 	{
 	  if (match)
 	    {
-	      mu_sv_compile_error (&mu_sieve_locus, 
-			     _("match type specified twice in call to `%s'"),
-				   name);
+	      mu_diag_at_locus (MU_LOG_ERROR, &mach->locus, 
+				_("match type specified twice in call to `%s'"),
+				name);
+	      mu_i_sv_error (mach);
 	      err = 1;
 	    }
 	  else
@@ -269,13 +275,14 @@ mu_sieve_match_part_checker (const char *name, mu_list_t tags, mu_list_t args)
 	  
 	  if (comp && strcmp (comp->arg->v.string, "i;ascii-numeric"))
 	    {
-	      mu_sv_compile_error (&mu_sieve_locus, 
-				   /* TRANSLATORS: Do not translate ':count'.
-				      It is the name of a Sieve tag */
-				   _("comparator %s is incompatible with "
-				     ":count in call to `%s'"),
-				   comp->arg->v.string,
-				   name);
+	      mu_diag_at_locus (MU_LOG_ERROR, &mach->locus,
+				/* TRANSLATORS: Do not translate ':count'.
+				   It is the name of a Sieve tag */
+				_("comparator %s is incompatible with "
+				  ":count in call to `%s'"),
+				comp->arg->v.string,
+				name);
+	      mu_i_sv_error (mach);
 	      return 1;
 	    }
 
@@ -288,16 +295,18 @@ mu_sieve_match_part_checker (const char *name, mu_list_t tags, mu_list_t args)
 	  mu_list_count (val->v.list, &count);
 	  if (count > 1)
 	    {
-	      mu_sv_compile_error (&mu_sieve_locus, 
+	      mu_diag_at_locus (MU_LOG_ERROR, &mach->locus, 
 			_("second argument must be a list of one element"));
+	      mu_i_sv_error (mach);
 	      return 1;
 	    }
 	  mu_list_get (val->v.list, 0, (void **) &str);
 	  count = strtoul (str, &str, 10);
 	  if (*str)
 	    {
-	      mu_sv_compile_error (&mu_sieve_locus, 
+	      mu_diag_at_locus (MU_LOG_ERROR, &mach->locus, 
 			   _("second argument cannot be converted to number"));
+	      mu_i_sv_error (mach);
 	      return 1;
 	    }
 	}
@@ -306,9 +315,10 @@ mu_sieve_match_part_checker (const char *name, mu_list_t tags, mu_list_t args)
 
       if (mu_sieve_str_to_relcmp (str, NULL, NULL))
 	{
-	  mu_sv_compile_error (&mu_sieve_locus, 
-			       _("invalid relational match `%s' in call to `%s'"),
-			       str, name);
+	  mu_diag_at_locus (MU_LOG_ERROR, &mach->locus, 
+			    _("invalid relational match `%s' in call to `%s'"),
+			    str, name);
+	  mu_i_sv_error (mach);
 	  return 1;
 	}
     }
@@ -316,18 +326,18 @@ mu_sieve_match_part_checker (const char *name, mu_list_t tags, mu_list_t args)
   if (!compfun)
     {
       compname = comp ? comp->arg->v.string : "i;ascii-casemap";
-      compfun = mu_sieve_comparator_lookup (mu_sieve_machine, compname, 
-                                            matchtype);
+      compfun = mu_sieve_comparator_lookup (mach, compname, matchtype);
       if (!compfun)
 	{
-	  mu_sv_compile_error (&mu_sieve_locus, 
-			   _("comparator `%s' is incompatible with match type `%s' in call to `%s'"),
-			       compname, match ? match->tag : "is", name);
+	  mu_diag_at_locus (MU_LOG_ERROR, &mach->locus, 
+			    _("comparator `%s' is incompatible with match type `%s' in call to `%s'"),
+			    compname, match ? match->tag : "is", name);
+	  mu_i_sv_error (mach);
 	  return 1;
 	}
     }
 
-  tmp = mu_sieve_malloc (mu_sieve_machine, sizeof (*tmp));
+  tmp = mu_sieve_malloc (mach, sizeof (*tmp));
   tmp->tag = TAG_COMPFUN;
   tmp->arg = mu_sieve_value_create (SVT_POINTER, compfun);
   mu_list_append (tags, tmp);
@@ -343,6 +353,7 @@ mu_sieve_match_part_checker (const char *name, mu_list_t tags, mu_list_t args)
       if (mu_list_get (args, 1, (void**)&val))
 	return 0;
 
+      rd.mach = mach;
       rd.flags = REG_EXTENDED;
       if (strcmp (compname, "i;ascii-casemap") == 0)
 	rd.flags |= REG_ICASE;
@@ -351,8 +362,7 @@ mu_sieve_match_part_checker (const char *name, mu_list_t tags, mu_list_t args)
       
       rc = mu_sieve_vlist_do (val, _regex_compile, &rd);
 
-      mu_sieve_machine_add_destructor (mu_sieve_machine, _free_reglist, 
-                                       rd.list);
+      mu_sieve_machine_add_destructor (rd.mach, _free_reglist, rd.list);
 
       if (rc)
 	return rc;

@@ -27,7 +27,7 @@
 #include <sieve-priv.h>
 
 void
-mu_sieve_require (mu_list_t slist)
+mu_sieve_require (mu_sieve_machine_t mach, mu_list_t slist)
 {
   int status;
   mu_iterator_t itr;
@@ -35,9 +35,10 @@ mu_sieve_require (mu_list_t slist)
   status = mu_list_get_iterator (slist, &itr);
   if (status)
     {
-      mu_sv_compile_error (&mu_sieve_locus,
-			   _("cannot create iterator: %s"),
-			   mu_strerror (status));
+      mu_diag_at_locus (MU_LOG_ERROR, &mach->locus,
+			_("cannot create iterator: %s"),
+			mu_strerror (status));
+      mu_i_sv_error (mach);
       return;
     }
 
@@ -73,12 +74,12 @@ mu_sieve_require (mu_list_t slist)
 	  text = _("required action");
 	}
 
-      if (reqfn (mu_sieve_machine, name))
+      if (reqfn (mach, name))
 	{
-	  mu_sv_compile_error (&mu_sieve_locus,
-			       _("source for the %s %s is not available"),
-			       text,
-			       name);
+	  mu_diag_at_locus (MU_LOG_ERROR, &mach->locus,
+			    _("source for the %s %s is not available"),
+			    text, name);
+	  mu_i_sv_error (mach);
 	}
     }
   mu_iterator_destroy (&itr);

@@ -339,6 +339,10 @@ mu_sieve_mailbox (mu_sieve_machine_t mach, mu_mailbox_t mbox)
   if (!mach || !mbox)
     return EINVAL;
 
+  if (mach->state != mu_sieve_state_compiled)
+    return EINVAL; /* FIXME: Error code */
+  mach->state = mu_sieve_state_running;
+  
   mu_observer_create (&observer, mach);
   mu_observer_set_action (observer, _sieve_action, mach);
   mu_mailbox_get_observable (mbox, &observable);
@@ -353,6 +357,7 @@ mu_sieve_mailbox (mu_sieve_machine_t mach, mu_mailbox_t mbox)
   mu_observable_detach (observable, observer);
   mu_observer_destroy (&observer, mach);
 
+  mach->state = mu_sieve_state_compiled;
   mach->mailbox = NULL;
   
   return rc;
@@ -366,10 +371,15 @@ mu_sieve_message (mu_sieve_machine_t mach, mu_message_t msg)
   if (!mach || !msg)
     return EINVAL;
 
+  if (mach->state != mu_sieve_state_compiled)
+    return EINVAL; /* FIXME: Error code */
+  mach->state = mu_sieve_state_running;
+
   mach->msgno = 1;
   mach->msg = msg;
   mach->mailbox = NULL;
   rc = sieve_run (mach);
+  mach->state = mu_sieve_state_compiled;
   mach->msg = NULL;
   
   return rc;

@@ -31,20 +31,38 @@ typedef union
   mu_sieve_value_t *val;
   mu_list_t list;
   long number;
-  const char *string;
   size_t pc;
   size_t line;
+  int inum;
+  char *string;
 } sieve_op_t;
 
+struct mu_locus_range
+{
+  struct mu_locus beg;
+  struct mu_locus end;
+};
+  
+#define YYLTYPE struct mu_locus_range
+
+enum mu_sieve_state
+  {
+    mu_sieve_state_init,
+    mu_sieve_state_error,
+    mu_sieve_state_compiled,
+    mu_sieve_state_running
+  };
+
 struct mu_sieve_machine
 {
   /* Static data */
-  struct mu_locus locus;    /* Approximate location in the code */
+  struct mu_locus locus;     /* Approximate location in the code */
 
   mu_list_t memory_pool;     /* Pool of allocated memory objects */
   mu_list_t destr_list;      /* List of destructor functions */
 
   /* Symbol space: */
+  mu_opool_t string_pool;    /* String constants */
   mu_list_t test_list;       /* Tests */
   mu_list_t action_list;     /* Actions */
   mu_list_t comp_list;       /* Comparators */
@@ -54,6 +72,7 @@ struct mu_sieve_machine
   sieve_op_t *prog;          /* Compiled program */
 
   /* Runtime data */
+  enum mu_sieve_state state;
   size_t pc;                 /* Current program counter */
   long reg;                  /* Numeric register */
   mu_list_t stack;           /* Runtime stack */
@@ -76,43 +95,68 @@ struct mu_sieve_machine
   char *daemon_email;
   void *data;
 };
+
+enum mu_sieve_node_type
+  {
+    mu_sieve_node_noop,
+    mu_sieve_node_false,
+    mu_sieve_node_true,
+    mu_sieve_node_test,
+    mu_sieve_node_action,
+    mu_sieve_node_cond,
+    mu_sieve_node_anyof,
+    mu_sieve_node_allof,
+    mu_sieve_node_not,
+  };
+
+struct mu_sieve_node
+{
+  struct mu_sieve_node *prev, *next;
+  enum mu_sieve_node_type type;
+  struct mu_locus_range locus;
+  union
+  {
+    mu_sieve_value_t *value;
+    mu_list_t list;
+    struct mu_sieve_node *node;
+    struct
+    {
+      struct mu_sieve_node *expr;
+      struct mu_sieve_node *iftrue;
+      struct mu_sieve_node *iffalse;
+    } cond;
+    struct
+    {
+      mu_sieve_register_t *reg;
+      mu_list_t arg;
+    } command;
+  } v;
+};
+
+int mu_sieve_yyerror (const char *s);
+int mu_sieve_yylex (void);
 
-extern struct mu_locus mu_sieve_locus;
+int mu_i_sv_lex_begin (const char *name);
+int mu_i_sv_lex_begin_string (const char *buf, int bufsize,
+			      const char *fname, int line);
+void mu_i_sv_lex_finish (struct mu_sieve_machine *mach);
+
 extern mu_sieve_machine_t mu_sieve_machine;
-extern int mu_sieve_error_count; 
 
 #define TAG_COMPFUN "__compfun__"
-#define TAG_RELFUN  "__relfun__"
-
-void mu_sv_compile_error (struct mu_locus *locus,
-			  const char *fmt, ...) MU_PRINTFLIKE(2,3);
-void mu_sv_print_value_list (mu_list_t list, mu_stream_t str);
-void mu_sv_print_tag_list (mu_list_t list, mu_stream_t str);
 
-int mu_sv_lex_begin (const char *name);
-int mu_sv_lex_begin_string (const char *buf, int bufsize,
-			    const char *fname, int line);
-void mu_sv_lex_finish (void);
-int mu_sieve_yyerror (const char *s);
-int mu_sieve_yylex (); 
+int mu_i_sv_code (struct mu_sieve_machine *mach, sieve_op_t op);
 
-void mu_sv_register_standard_actions (mu_sieve_machine_t mach);
-void mu_sv_register_standard_tests (mu_sieve_machine_t mach);
-void mu_sv_register_standard_comparators (mu_sieve_machine_t mach);
+void mu_i_sv_compile_error (struct mu_sieve_machine *mach,
+			    const char *fmt, ...) MU_PRINTFLIKE(2,3);
 
-int mu_sv_code (sieve_op_t *op);
-int mu_sv_code_instr (sieve_instr_t instr);
-int mu_sv_code_handler (mu_sieve_handler_t handler);
-int mu_sv_code_list (mu_list_t list);
-int mu_sv_code_number (long num);
-int mu_sv_code_test (mu_sieve_register_t *reg, mu_list_t arglist);
-int mu_sv_code_action (mu_sieve_register_t *reg, mu_list_t arglist);
-void mu_sv_code_anyof (size_t start);
-void mu_sv_code_allof (size_t start);
-int mu_sv_code_source (const char *name);
-int mu_sv_code_line (size_t line);
-void mu_sv_change_source (void);
+int mu_i_sv_locus (struct mu_sieve_machine *mach, struct mu_locus_range *lr);
+int mu_i_sv_code_action (struct mu_sieve_machine *mach,
+			 mu_sieve_register_t *reg, mu_list_t arglist);
+int mu_i_sv_code_test (struct mu_sieve_machine *mach,
+		       mu_sieve_register_t *reg, mu_list_t arglist);
 
+/* Opcodes */
 void _mu_sv_instr_action (mu_sieve_machine_t mach);
 void _mu_sv_instr_test (mu_sieve_machine_t mach);
 void _mu_sv_instr_push (mu_sieve_machine_t mach);
@@ -126,4 +170,13 @@ void _mu_sv_instr_source (mu_sieve_machine_t mach);
 void _mu_sv_instr_line (mu_sieve_machine_t mach);
 
 int mu_sv_load_add_dir (mu_sieve_machine_t mach, const char *name);
-  
+
+void mu_sv_register_standard_actions (mu_sieve_machine_t mach);
+void mu_sv_register_standard_tests (mu_sieve_machine_t mach);
+void mu_sv_register_standard_comparators (mu_sieve_machine_t mach);
+
+void mu_sv_print_value_list (mu_list_t list, mu_stream_t str);
+void mu_sv_print_tag_list (mu_list_t list, mu_stream_t str);
+
+void mu_i_sv_error (mu_sieve_machine_t mach);
+

@@ -35,10 +35,6 @@
 #include <sieve-priv.h>
 #include <sieve-gram.h>
   
-struct mu_locus mu_sieve_locus;
-ino_t sieve_source_inode;
-
-static mu_list_t string_list;
 static char *multiline_delimiter;
 static int strip_tabs; 
 
@@ -57,6 +53,11 @@ static void sieve_searchpath (void);
 static char *str_unescape (char *text, size_t len);
 static int isemptystr (char *text);
 
+static ino_t sieve_source_inode;
+struct mu_locus mu_sieve_locus;
+static int newline;
+static mu_list_t file_names;
+
 static mu_stream_t input_stream;
  
 static int
@@ -89,6 +90,61 @@ fillbuf (char *buf, size_t max_size)
         yy_switch_to_buffer (s); \
 } while (0)
 
+static int
+file_name_cmp (const void *a, const void *b)  
+{
+  return strcmp (a, b);
+}
+
+static void
+init_locus (char const *name, ino_t ino)
+{
+  mu_sieve_locus.mu_file = NULL;
+  if (name)
+    {
+      if (!file_names)
+	{
+	  mu_list_create (&file_names);
+	  mu_list_set_comparator (file_names, file_name_cmp);
+	  mu_list_set_destroy_item (file_names, mu_list_free_item);
+	}
+      else
+	{
+	  mu_list_locate (file_names, (void*) name,
+			  (void**) &mu_sieve_locus.mu_file);
+	}
+      if (!mu_sieve_locus.mu_file)
+	{
+	  mu_sieve_locus.mu_file = strdup (name);//FIXME: Error checking
+	  mu_list_append (file_names, mu_sieve_locus.mu_file);
+	}
+    }
+  mu_sieve_locus.mu_line = 1;
+  mu_sieve_locus.mu_col = 0;
+  newline = 0;
+  sieve_source_inode = ino;
+}
+ 
+static void
+advance_locus (void)
+{
+  if (newline)
+    {
+      mu_sieve_locus.mu_line++;
+      mu_sieve_locus.mu_col = 0;
+      yylloc.beg = yylloc.end = mu_sieve_locus;                        
+    }
+  else
+    {
+      mu_sieve_locus.mu_col += yyleng;
+      yylloc.beg = yylloc.end = mu_sieve_locus;                        
+      yylloc.beg.mu_col -= yyleng;                             
+    }
+  newline = yytext[yyleng-1] == '\n';
+}
+ 
+#define YY_USER_ACTION advance_locus ();
+   
 struct buffer_ctx
 {
   struct buffer_ctx *prev;
@@ -125,8 +181,9 @@ push_source (const char *name)
         
   if (stat (name, &st))
     {
-      mu_sv_compile_error (&mu_sieve_locus,
-                           _("cannot stat `%s': %s"), name, strerror (errno));
+      mu_diag_at_locus (MU_LOG_ERROR, &mu_sieve_locus,
+			_("cannot stat `%s': %s"), name, strerror (errno));
+      mu_i_sv_error (mu_sieve_machine);
       return 1;
     }
 
@@ -139,28 +196,36 @@ push_source (const char *name)
     {
       yyerror (_("recursive inclusion"));
       if (ctx->prev)
-	mu_sv_compile_error (&ctx->prev->locus,
-                             _("`%s' already included here"),
-                             name);
+	{
+	  mu_diag_at_locus (MU_LOG_ERROR, &ctx->prev->locus,
+			    _("`%s' already included here"),
+			    name);
+	  mu_i_sv_error (mu_sieve_machine);
+	}
       else
-	mu_sv_compile_error (&mu_sieve_locus, 
-                             _("`%s' already included at top level"),
-		             name);
+	{
+	  mu_diag_at_locus (MU_LOG_ERROR, &mu_sieve_locus,
+			    _("`%s' already included at top level"),
+			    name);
+	  mu_i_sv_error (mu_sieve_machine);
+	}	  
       return 1;
     }
 
   rc = mu_file_stream_create (&stream, name, MU_STREAM_READ);
   if (rc)
     {
-      mu_sv_compile_error (&mu_sieve_locus, 
-                           _("cannot open file `%s': %s"),
-			   name, mu_strerror (rc));
+      mu_diag_at_locus (MU_LOG_ERROR, &mu_sieve_locus,
+			_("cannot open file `%s': %s"),
+			name, mu_strerror (rc));
+      mu_i_sv_error (mu_sieve_machine);
       return 1;
     }
 
   /* Push current context */
   if (mu_sieve_locus.mu_file)
     {
+      advance_locus ();
       ctx = mu_sieve_alloc (sizeof (*ctx));
       ctx->locus = mu_sieve_locus;
       ctx->i_node = sieve_source_inode;
@@ -172,14 +237,9 @@ push_source (const char *name)
       SET_BUFFER_STATE (ctx->state);
     }
   input_stream = stream;
-  
-  mu_sieve_locus.mu_file = strdup (name);
-  mu_sieve_locus.mu_line = 1;
-  mu_sieve_locus.mu_col = 0; /* FIXME: not used */
-  sieve_source_inode = st.st_ino;
 
-  mu_sv_change_source ();
-  
+  init_locus (name, st.st_ino);
+
   return 0;
 }
 
@@ -190,26 +250,21 @@ pop_source ()
 
   mu_stream_destroy (&input_stream);
 
-  if (mu_sieve_locus.mu_file)
-    free (mu_sieve_locus.mu_file);
   if (!context_stack)
     {
       input_stream = NULL;
-      mu_sieve_locus.mu_file = NULL;
+      init_locus (NULL, 0);
       return 1;
     }
   /* Restore previous context */
   input_stream = context_stack->input;
   mu_sieve_locus = context_stack->locus;
-  mu_sieve_locus.mu_line++; /* #include rule did not increment it */
   sieve_source_inode = context_stack->i_node;
   RESTORE_BUFFER_STATE (context_stack->state);
   ctx = context_stack->prev;
   free (context_stack);
   context_stack = ctx;
 
-  mu_sv_change_source ();
-
   return 0;
 }
 %}
@@ -225,16 +280,16 @@ SIZESUF [kKmMgG]
 
 %%
          /* C-style comments */
-"/*"         BEGIN(COMMENT);
+"/*"                    { BEGIN(COMMENT); }
 <COMMENT>[^*\n]*        /* eat anything that's not a '*' */
 <COMMENT>"*"+[^*/\n]*   /* eat up '*'s not followed by '/'s */
-<COMMENT>\n             ++mu_sieve_locus.mu_line;
-<COMMENT>"*"+"/"        BEGIN(INITIAL);
+<COMMENT>\n             ;
+<COMMENT>"*"+"/"        { BEGIN (INITIAL); }
          /* Preprocessor directives (an extension) */
 #[ \t]*include.*\n      { sieve_include (); }
-#[ \t]*searchpath.*\n          { sieve_searchpath (); } 
+#[ \t]*searchpath.*\n   { sieve_searchpath (); } 
          /* End-of-line comments */
-#.*\n   { mu_sieve_locus.mu_line++; }
+#.*\n   ;
 #.*     /* end-of-file comment */;
          /* Reserved words */
 require return REQUIRE; 
@@ -244,6 +299,8 @@ else	return ELSE;
 anyof	return ANYOF;   
 allof	return ALLOF;   
 not     return NOT;
+false   return FALSE;
+true    return TRUE;
          /* Identifiers */
 {IDENT}  { ident (yytext); return IDENT; } 
 :{IDENT} { ident (yytext + 1); return TAG; }
@@ -254,34 +311,28 @@ not     return NOT;
          /* Quoted strings */
 \"[^\\"\n]*\"                 { return string (); }
 \"[^\\"\n]*\\.    { BEGIN(STR);
-                   line_begin ();
-		   line_add (str_unescape (yytext + 1, yyleng - 1), 0); }
+                    line_begin ();
+  		    line_add (str_unescape (yytext + 1, yyleng - 1), 0); }
 <STR>[^\\"\n]*\\. { line_add (str_unescape (yytext, yyleng), 0); }
 <STR>[^\\"\n]*\" { BEGIN(INITIAL);
-                   if (yyleng > 1) 
+                  if (yyleng > 1) 
                      line_add (yytext, yyleng - 1); 
                    line_finish ();
 		   return STRING; }
          /* Multiline strings */
 text:-?[ \t]*#.*\n       { BEGIN(ML);
-                           multiline_begin ();
-			   mu_sieve_locus.mu_line++; }
+                           multiline_begin (); }
 text:-?[ \t]*\n          { BEGIN(ML);
-                           multiline_begin ();
-			   mu_sieve_locus.mu_line++; }
-text:-?\\?{IDENT}[ \t]*#.*\n { BEGIN(ML); multiline_begin ();
-                               mu_sieve_locus.mu_line++; }
-text:-?\\?{IDENT}[ \t]*\n    { BEGIN(ML); multiline_begin ();
-                               mu_sieve_locus.mu_line++; }
+                           multiline_begin (); }
+text:-?\\?{IDENT}[ \t]*#.*\n { BEGIN(ML);
+                               multiline_begin (); }
+text:-?\\?{IDENT}[ \t]*\n    { BEGIN(ML);
+                               multiline_begin (); }
 <ML>#[ \t]*include.*\n    { if (multiline_delimiter[0] == '\\')
-                              {
-				mu_sieve_locus.mu_line++;
-                                multiline_add (NULL);
-			      }
+                              multiline_add (NULL);
                             else
                               sieve_include (); }
 <ML>.*\n { char *p = multiline_strip_tabs (yytext);
-           mu_sieve_locus.mu_line++;
 	   
            if (strncmp (p, multiline_delimiter, strlen (multiline_delimiter))
 	        == 0
@@ -296,7 +347,7 @@ text:-?\\?{IDENT}[ \t]*\n    { BEGIN(ML); multiline_begin ();
 	    multiline_add (NULL); } 
 {WS}     ;
          /* Other tokens */
-\n { mu_sieve_locus.mu_line++; } 
+\n       ;
 . return yytext[0];
 
 %%
@@ -400,7 +451,7 @@ sieve_include ()
 }
 
 static void
-sieve_searchpath ()
+sieve_searchpath (void)
 {
   char *p, *endp = yytext + yyleng, *name;
   
@@ -416,14 +467,14 @@ sieve_searchpath ()
 }
 
 int
-mu_sv_lex_begin (const char *name)
+mu_i_sv_lex_begin (const char *name)
 {
   return push_source (name);
 }
 
 int
-mu_sv_lex_begin_string (const char *buf, int bufsize,
-			const char *fname, int line)
+mu_i_sv_lex_begin_string (const char *buf, int bufsize,
+			  const char *fname, int line)
 {
   int rc;
   
@@ -439,19 +490,18 @@ mu_sv_lex_begin_string (const char *buf, int bufsize,
       return 1;
     }
 
-  mu_sieve_locus.mu_file = strdup (fname);
-  mu_sieve_locus.mu_line = line;
-  sieve_source_inode = 0;
+  init_locus (fname, 0);
 
-  mu_sv_change_source ();
   return 0;
 }
 
 void
-mu_sv_lex_finish ()
+mu_i_sv_lex_finish (struct mu_sieve_machine *mach)
 {
   while (pop_source () == 0)
     ;
+  mach->source_list = file_names;
+  file_names = NULL;
 }
   
 static int
@@ -507,19 +557,7 @@ multiline_strip_tabs (char *text)
 static void
 line_add (char *text, size_t len)
 {
-  char *s;
-  
-  if (len == 0)
-    len = strlen (text);
-  s = malloc (len + 1);
-  if (!s)
-    {
-      yyerror (_("not enough memory"));
-      exit (1);
-    }
-  memcpy (s, text, len);
-  s[len] = 0;
-  mu_list_append (string_list, s);
+  mu_opool_append (mu_sieve_machine->string_pool, text, len);
 }
 
 static void
@@ -527,27 +565,17 @@ multiline_add (char *s)
 {
   if (!s)
     s = multiline_strip_tabs (yytext);
-  line_add (s, 0);
+  mu_opool_appendz (mu_sieve_machine->string_pool, s);
 }
 
 static void
-line_begin ()
+line_begin (void)
 {
-  int status;
-
-  if (string_list)
-    mu_sieve_slist_destroy (&string_list);
-  status = mu_list_create (&string_list);
-  if (status)
-    {
-      mu_sv_compile_error (&mu_sieve_locus,
-                           "mu_list_create: %s", mu_strerror (status));
-      exit (1);
-    }
+  /* nothing */
 }
 
 static void
-multiline_begin ()
+multiline_begin (void)
 {
   char *p = yytext + 5; /* past the text: keyword */
 
@@ -587,42 +615,14 @@ multiline_begin ()
 }
 
 static void
-line_finish ()
+line_finish (void)
 {
-  mu_iterator_t itr;
-  int length = 0;
-  char *p;
-  
-  if (!string_list || mu_list_get_iterator (string_list, &itr))
-    return;
-
-  /* Count number of characters in the multiline */
-  for (mu_iterator_first (itr); !mu_iterator_is_done (itr); mu_iterator_next (itr))
-    {
-      char *s;
-      mu_iterator_current (itr, (void **)&s);
-      length += strlen (s);
-    }
-
-  /* Copy the contents */
-  yylval.string = mu_sieve_malloc (mu_sieve_machine, length + 1);
-  p = yylval.string;
-  for (mu_iterator_first (itr); !mu_iterator_is_done (itr);
-       mu_iterator_next (itr))
-    {
-      char *s;
-      mu_iterator_current (itr, (void **)&s);
-      strcpy (p, s);
-      p += strlen (s);
-      free (s);
-    }
-  *p = 0;
-  mu_iterator_destroy (&itr);
-  mu_list_destroy (&string_list);
+  mu_opool_append_char (mu_sieve_machine->string_pool, 0);
+  yylval.string = mu_opool_finish (mu_sieve_machine->string_pool, NULL);
 }
 
 static void
-multiline_finish ()
+multiline_finish (void)
 {
   line_finish ();
 }

@@ -320,22 +320,6 @@ sieve_test_size (mu_sieve_machine_t mach, mu_list_t args, mu_list_t tags)
 }
 
 int
-sieve_test_true (mu_sieve_machine_t mach, mu_list_t args, mu_list_t tags)
-{
-  if (mach->debug_level & MU_SIEVE_DEBUG_TRACE)
-    mu_sieve_debug (mach, "TRUE");
-  return 1;
-}
-
-int
-sieve_test_false (mu_sieve_machine_t mach, mu_list_t args, mu_list_t tags)
-{
-  if (mach->debug_level & MU_SIEVE_DEBUG_TRACE)
-    mu_sieve_debug (mach, "FALSE");
-  return 0;
-}
-
-int
 _test_exists (void *item, void *data)
 {
   mu_header_t hdr = data;
@@ -446,8 +430,7 @@ mu_sieve_tag_group_t header_tag_groups[] = {
 void
 mu_sv_register_standard_tests (mu_sieve_machine_t mach)
 {
-  mu_sieve_register_test (mach, "false", sieve_test_false, NULL, NULL, 1);
-  mu_sieve_register_test (mach, "true", sieve_test_true, NULL, NULL, 1);
+  /* true and false are built-ins */
   mu_sieve_register_test (mach, "address", sieve_test_address,
 		       address_req_args, address_tag_groups, 1);
   mu_sieve_register_test (mach, "size", sieve_test_size,

@@ -171,7 +171,7 @@ mu_sieve_value_create (mu_sieve_data_type type, void *data)
       break;
 	
     default:
-      mu_sv_compile_error (&mu_sieve_locus, _("invalid data type"));
+      mu_error ("%s", _("invalid data type"));
       abort ();
     }
   return val;
@@ -186,24 +186,6 @@ mu_sieve_value_get (mu_list_t vlist, size_t index)
 }
 
 void
-mu_sv_compile_error (struct mu_locus *ploc, const char *fmt, ...)
-{
-  va_list ap;
-
-  va_start (ap, fmt);
-  mu_sieve_error_count++;
-  mu_stream_ioctl (mu_sieve_machine->errstream,
-                   MU_IOCTL_LOGSTREAM, MU_IOCTL_LOGSTREAM_SET_LOCUS,
-		   ploc);
-  mu_stream_printf (mu_sieve_machine->errstream,
-		    "\033s<%d>\033O<%d>",
-		    MU_LOG_ERROR, MU_LOGMODE_LOCUS);
-  mu_stream_vprintf (mu_sieve_machine->errstream, fmt, ap);
-  mu_stream_write (mu_sieve_machine->errstream, "\n", 1, NULL);
-  va_end (ap);
-}
-
-void
 mu_sieve_error (mu_sieve_machine_t mach, const char *fmt, ...)
 {
   va_list ap;

@@ -83,7 +83,9 @@ set_debug_level (const char *arg)
 	  break;
 
 	case 'g':
-	  mu_sieve_yydebug = 1;
+	  mu_debug_get_category_level (mu_sieve_debug_handle, &lev);
+	  mu_debug_set_category_level (mu_sieve_debug_handle,
+				    lev | MU_DEBUG_LEVEL_MASK(MU_DEBUG_TRACE1));
 	  break;
 
 	case 't':

@@ -38,7 +38,7 @@ if header :comparator "i;ascii-numeric" :contains "X-Number" "15"
     discard;
   }
 ],[78],[],
-[sieve: prog:5: comparator `i;ascii-numeric' is incompatible with match type `contains' in call to `header'
+[sieve: prog:4:1: comparator `i;ascii-numeric' is incompatible with match type `contains' in call to `header'
 ])
 AT_CLEANUP