Commit cef5f35d cef5f35da5f0e9b326d4d9a56f6b62ad565ad643 by Sergey Poznyakoff

Redo message set parser in imap4d

The new parser complies to RFC3501. Its output message sets are
formatted as MU lists of non-overlapping contiguous message ranges,
which reduces memory consumption and imposes less strain on CPU.
The parser automatically translates message UIDs to sequence numbers
and provides better error handling.

* imap4d/imap4d.h (util_msgset): Remove.
(util_parse_msgset): New proto.
(imap4d_message_action_t): New typedef.
(util_foreach_message): New proto.
* imap4d/util.c  (util_msgset): Remove.
(util_parse_msgset): New function.

* imap4d/copy.c: Use util_parse_msgset to parse message set specs and
util_foreach_message to iterate over the returned list.
* imap4d/fetch.c: Likewise.
* imap4d/search.c: Likewise.
* imap4d/store.c: Likewise.
* imap4d/tests/IDEF0955.at: Update the test.
* include/mailutils/list.h (mu_list_action_t): Fix typedef.
* libmailutils/list/foreach.c (mu_list_foreach)
(mu_list_do): Update signatures.
1 parent f9a034c7
......@@ -55,88 +55,111 @@ imap4d_copy (struct imap4d_command *command, imap4d_tokbuf_t tok)
return io_completion_response (command, rc, "%s", text);
}
struct copy_env
{
mu_mailbox_t dst;
mu_mailbox_t src;
mu_off_t total;
int ret;
char **err_text;
};
static int
copy_check_size (mu_mailbox_t mbox, size_t n, size_t *set, mu_off_t *size)
size_sum (size_t msgno, void *data)
{
int status;
size_t i;
mu_off_t total = 0;
struct copy_env *env = data;
mu_message_t msg = NULL;
int rc;
for (i = 0; i < n; i++)
rc = mu_mailbox_get_message (env->src, msgno, &msg);
if (rc)
{
mu_diag_funcall (MU_DIAG_ERROR, "mu_mailbox_get_message", NULL, rc);
env->ret = RESP_NO;
return MU_ERR_FAILURE;
}
else
{
mu_message_t msg = NULL;
size_t msgno = set[i];
if (msgno)
size_t size;
rc = mu_message_size (msg, &size);
if (rc)
{
status = mu_mailbox_get_message (mbox, msgno, &msg);
if (status)
{
mu_diag_funcall (MU_DIAG_ERROR, "mu_mailbox_get_message", NULL,
status);
return RESP_BAD;
}
else
{
size_t size;
status = mu_message_size (msg, &size);
if (status)
{
mu_diag_funcall (MU_DIAG_ERROR, "mu_message_size", NULL,
status);
return RESP_BAD;
}
total += size;
}
mu_diag_funcall (MU_DIAG_ERROR, "mu_message_size", NULL, rc);
env->ret = RESP_BAD;
return MU_ERR_FAILURE;
}
env->total += size;
}
*size = total;
return quota_check (total);
return 0;
}
static int
try_copy (mu_mailbox_t dst, mu_mailbox_t src, size_t n, size_t *set)
do_copy (size_t msgno, void *data)
{
int result;
size_t i;
mu_off_t total;
result = copy_check_size (src, n, set, &total);
if (result)
return result;
for (i = 0; i < n; i++)
{
mu_message_t msg = NULL;
size_t msgno = set[i];
struct copy_env *env = data;
mu_message_t msg = NULL;
int status;
if (msgno)
{
int status = mu_mailbox_get_message (src, msgno, &msg);
if (status)
{
mu_diag_funcall (MU_DIAG_ERROR, "mu_mailbox_get_message", NULL,
status);
return RESP_BAD;
}
status = mu_mailbox_get_message (env->src, msgno, &msg);
if (status)
{
mu_diag_funcall (MU_DIAG_ERROR, "mu_mailbox_get_message", NULL,
status);
env->ret = RESP_BAD;
return MU_ERR_FAILURE;
}
imap4d_enter_critical ();
status = mu_mailbox_append_message (dst, msg);
imap4d_leave_critical ();
if (status)
{
mu_diag_funcall (MU_DIAG_ERROR, "mu_mailbox_append_message",
NULL,
status);
return RESP_BAD;
}
}
imap4d_enter_critical ();
status = mu_mailbox_append_message (env->dst, msg);
imap4d_leave_critical ();
if (status)
{
mu_diag_funcall (MU_DIAG_ERROR, "mu_mailbox_append_message", NULL,
status);
env->ret = RESP_BAD;
return MU_ERR_FAILURE;
}
quota_update (total);
return RESP_OK;
return 0;
}
static int
safe_copy (mu_mailbox_t dst, mu_mailbox_t src, size_t n, size_t *set,
try_copy (mu_mailbox_t dst, mu_mailbox_t src, mu_list_t msglist,
char **err_text)
{
int rc;
struct copy_env env;
env.dst = dst;
env.src = src;
env.total = 0;
env.ret = RESP_OK;
env.err_text = err_text;
*env.err_text = "Operation failed";
/* Check size */
rc = util_foreach_message (msglist, size_sum, &env);
if (rc)
return RESP_NO;
if (env.ret != RESP_OK)
return env.ret;
rc = quota_check (env.total);
if (rc)
{
*env.err_text = "Mailbox quota exceeded";
return RESP_NO;
}
env.total = 0;
rc = util_foreach_message (msglist, do_copy, &env);
quota_update (env.total);
if (rc)
return RESP_NO;
return env.ret;
}
static int
safe_copy (mu_mailbox_t dst, mu_mailbox_t src, mu_list_t msglist,
char **err_text)
{
size_t nmesg;
......@@ -151,16 +174,11 @@ safe_copy (mu_mailbox_t dst, mu_mailbox_t src, size_t n, size_t *set,
return RESP_NO;
}
status = try_copy (dst, src, n, set);
if (status)
status = try_copy (dst, src, msglist, err_text);
if (status != RESP_OK)
{
size_t maxmesg;
if (status == RESP_NO)
*err_text = "Mailbox quota exceeded";
else
*err_text = "Operation failed";
/* If the COPY command is unsuccessful for any reason, server
implementations MUST restore the destination mailbox to its state
before the COPY attempt. */
......@@ -211,11 +229,11 @@ imap4d_copy0 (imap4d_tokbuf_t tok, int isuid, char **err_text)
{
int status;
char *msgset;
mu_list_t msglist;
char *name;
char *mailbox_name;
const char *delim = "/";
size_t *set = NULL;
int n = 0;
char *end;
mu_mailbox_t cmbox = NULL;
int arg = IMAP4_ARG_1 + !!isuid;
int ns;
......@@ -230,23 +248,14 @@ imap4d_copy0 (imap4d_tokbuf_t tok, int isuid, char **err_text)
msgset = imap4d_tokbuf_getarg (tok, arg);
name = imap4d_tokbuf_getarg (tok, arg + 1);
/* Get the message numbers in set[]. */
status = util_msgset (msgset, &set, &n, isuid);
if (status != 0)
status = util_parse_msgset (msgset, isuid, mbox, &msglist, &end);
if (status)
{
/* See RFC 3501, section 6.4.8, and a comment to the equivalent code
in fetch.c */
*err_text = "Completed";
return RESP_OK;
*err_text = "Error parsing message set";
/* FIXME: print error location */
return RESP_BAD;
}
if (isuid)
{
int i;
/* Fixup the message set. Perhaps util_msgset should do it itself? */
for (i = 0; i < n; i++)
set[i] = uid_to_msgno (set[i]);
}
mailbox_name = namespace_getfullpath (name, delim, &ns);
if (!mailbox_name)
......@@ -256,7 +265,7 @@ imap4d_copy0 (imap4d_tokbuf_t tok, int isuid, char **err_text)
}
/* If the destination mailbox does not exist, a server should return
an error. */
an error. */
status = mu_mailbox_create_default (&cmbox, mailbox_name);
if (status == 0)
{
......@@ -264,13 +273,13 @@ imap4d_copy0 (imap4d_tokbuf_t tok, int isuid, char **err_text)
status = mu_mailbox_open (cmbox, MU_STREAM_RDWR | mailbox_mode[ns]);
if (status == 0)
{
status = safe_copy (cmbox, mbox, n, set, err_text);
if (!mu_list_is_empty (msglist))
status = safe_copy (cmbox, mbox, msglist, err_text);
mu_mailbox_close (cmbox);
}
mu_mailbox_destroy (&cmbox);
}
free (set);
free (mailbox_name);
mu_list_destroy (&msglist);
if (status == 0)
{
......
......@@ -37,6 +37,8 @@ struct fetch_runtime_closure
mu_message_t msg; /* The message itself */
mu_list_t msglist; /* A list of referenced messages. See KLUDGE below. */
char *err_text; /* On return: error description if failed. */
mu_list_t fnlist;
};
struct fetch_function_closure;
......@@ -62,8 +64,7 @@ struct fetch_parse_closure
{
int isuid;
mu_list_t fnlist;
size_t *set;
int count;
mu_list_t msgnumlist;
};
......@@ -1751,38 +1752,16 @@ fetch_thunk (imap4d_parsebuf_t pb)
{
int status;
char *msgset;
char *end;
struct fetch_parse_closure *pclos = imap4d_parsebuf_data (pb);
msgset = imap4d_parsebuf_next (pb, 1);
/* Get the message numbers in set[]. */
status = util_msgset (msgset, &pclos->set, &pclos->count, pclos->isuid);
switch (status)
{
case 0:
/* Very good! */
break;
case EINVAL:
/* RFC3501, section 6.4.8.
A non-existent unique identifier is ignored without any error
message generated. Thus, it is possible for a UID FETCH command
to return an OK without any data or a UID COPY or UID STORE to
return an OK without performing any operations.
Obviously the same holds true for non-existing message numbers
as well, although I did not find any explicit mention thereof
in the RFC.
FIXME: This code also causes imap4d to silently ignore erroneous
msgset specifications (e.g. FETCH foobar (FLAGS)), which should
be fixed. */
return RESP_OK;
default:
imap4d_parsebuf_exit (pb, "Failed to parse message set");
}
/* Parse sequence numbers. */
status = util_parse_msgset (msgset, pclos->isuid, mbox,
&pclos->msgnumlist, &end);
if (status)
imap4d_parsebuf_exit (pb, "Failed to parse message set");
/* Compile the expression */
......@@ -1797,6 +1776,23 @@ fetch_thunk (imap4d_parsebuf_t pb)
return RESP_OK;
}
int
_fetch_from_message (size_t msgno, void *data)
{
int rc = 0;
struct fetch_runtime_closure *frc = data;
frc->msgno = msgno;
if (mu_mailbox_get_message (mbox, msgno, &frc->msg) == 0)
{
io_sendf ("* %lu FETCH (", (unsigned long) msgno);
frc->eltno = 0;
rc = mu_list_foreach (frc->fnlist, _do_fetch, frc);
io_sendf (")\n");
}
return rc;
}
/* Where the real implementation is. It is here since UID command also
calls FETCH. */
int
......@@ -1823,32 +1819,20 @@ imap4d_fetch0 (imap4d_tokbuf_t tok, int isuid, char **err_text)
if (rc == RESP_OK)
{
size_t i;
struct fetch_runtime_closure frc;
memset (&frc, 0, sizeof (frc));
frc.fnlist = pclos.fnlist;
/* Prepare status code. It will be replaced if an error occurs in the
loop below */
frc.err_text = "Completed";
for (i = 0; i < pclos.count && rc == RESP_OK; i++)
{
frc.msgno = (isuid) ? uid_to_msgno (pclos.set[i]) : pclos.set[i];
if (frc.msgno &&
mu_mailbox_get_message (mbox, frc.msgno, &frc.msg) == 0)
{
io_sendf ("* %lu FETCH (", (unsigned long) frc.msgno);
frc.eltno = 0;
rc = mu_list_foreach (pclos.fnlist, _do_fetch, &frc);
io_sendf (")\n");
}
}
util_foreach_message (pclos.msgnumlist, _fetch_from_message, &frc);
mu_list_destroy (&frc.msglist);
}
mu_list_destroy (&pclos.fnlist);
free (pclos.set);
mu_list_destroy (&pclos.msgnumlist);
return rc;
}
......
......@@ -554,7 +554,6 @@ imap4d_master_signal (int signo)
}
int
main (int argc, char **argv)
{
......
......@@ -368,10 +368,17 @@ extern int util_getstate (void);
extern int util_do_command (imap4d_tokbuf_t);
extern char *util_tilde_expansion (const char *, const char *);
extern char *util_getfullpath (const char *, const char *);
extern int util_msgset (char *, size_t **, int *, int);
extern struct imap4d_command *util_getcommand (char *,
struct imap4d_command []);
extern int util_parse_msgset (char *s, int isuid, mu_mailbox_t mbx,
mu_list_t *plist, char **end);
typedef int (*imap4d_message_action_t) (size_t, void *);
int util_foreach_message (mu_list_t list, imap4d_message_action_t action,
void *data);
enum datetime_parse_mode /* how to parse date/time strings */
{
datetime_default, /* default mode */
......
......@@ -62,11 +62,7 @@ struct value
char *string;
mu_off_t number;
time_t date;
struct
{
int n;
size_t *set;
} msgset;
mu_list_t msgset;
} v;
};
......@@ -168,7 +164,7 @@ struct cond condlist[] =
{ "SUBJECT", "s", cond_subject },
{ "TEXT", "s", cond_text },
{ "TO", "s", cond_to },
{ "UID", "m", cond_uid },
{ "UID", "u", cond_uid },
{ NULL }
};
......@@ -584,8 +580,6 @@ parse_simple_key (struct parsebuf *pb)
struct search_node *node;
struct cond *condp;
time_t time;
size_t *set = NULL;
int n = 0;
for (condp = condlist; condp->name && mu_c_strcasecmp (condp->name, pb->token);
condp++)
......@@ -593,13 +587,14 @@ parse_simple_key (struct parsebuf *pb)
if (!condp->name)
{
if (util_msgset (pb->token, &set, &n, 0) == 0)
mu_list_t msglist;
if (util_parse_msgset (pb->token, pb->isuid, mbox, &msglist, NULL) == 0)
{
struct search_node *np = parse_alloc (pb, sizeof *np);
np->type = node_value;
np->v.value.type = value_msgset;
np->v.value.v.msgset.n = n;
np->v.value.v.msgset.set = parse_regmem (pb, set);
np->v.value.v.msgset = msglist;
node = parse_alloc (pb, sizeof *node);
node->type = node_call;
......@@ -630,9 +625,7 @@ parse_simple_key (struct parsebuf *pb)
{
char *t = condp->argtypes;
char *s;
int n;
mu_off_t number;
size_t *set;
struct search_node *arg;
for (; *t; t++, parse_gettoken (pb, 0))
......@@ -680,20 +673,20 @@ parse_simple_key (struct parsebuf *pb)
arg->v.value.v.date = time;
break;
case 'm': /* message set */
if (util_msgset (pb->token, &set, &n, 1)) /*FIXME: isuid?*/
case 'u': /* UID message set */
if (util_parse_msgset (pb->token, 0, NULL,
&arg->v.value.v.msgset, NULL))
{
pb->err_mesg = "Bogus number set";
return NULL;
}
arg->v.value.type = value_msgset;
arg->v.value.v.msgset.n = n;
arg->v.value.v.msgset.set = parse_regmem (pb, set);
break;
default:
mu_diag_output (MU_DIAG_CRIT, _("%s:%d: INTERNAL ERROR (please report)"),
__FILE__, __LINE__);
mu_diag_output (MU_DIAG_CRIT,
_("%s:%d: INTERNAL ERROR (please report)"),
__FILE__, __LINE__);
abort (); /* should never happen */
}
node->v.key.arg[node->v.key.narg++] = arg;
......@@ -860,15 +853,9 @@ static void
cond_msgset (struct parsebuf *pb, struct search_node *node, struct value *arg,
struct value *retval)
{
int n = arg[0].v.msgset.n;
size_t *set = arg[0].v.msgset.set;
int i, rc;
for (i = rc = 0; rc == 0 && i < n; i++)
rc = set[i] == pb->msgno;
int rc = mu_list_locate (arg[0].v.msgset, &pb->msgno, NULL);
retval->type = value_number;
retval->v.number = rc;
retval->v.number = rc == 0;
}
static void
......@@ -1084,16 +1071,12 @@ static void
cond_uid (struct parsebuf *pb, struct search_node *node, struct value *arg,
struct value *retval)
{
int n = arg[0].v.msgset.n;
size_t *set = arg[0].v.msgset.set;
int rc;
size_t uid = 0;
int i, rc;
mu_message_get_uid (pb->msg, &uid);
for (i = rc = 0; rc == 0 && i < n; i++)
rc = set[i] == uid;
rc = mu_list_locate (arg[0].v.msgset, &pb->msgno, NULL);
retval->type = value_number;
retval->v.number = rc;
retval->v.number = rc == 0;
}
......
......@@ -25,8 +25,7 @@ struct store_parse_closure
int ack;
int type;
int isuid;
size_t *set;
int count;
mu_list_t msgnumlist;
};
static int
......@@ -36,6 +35,7 @@ store_thunk (imap4d_parsebuf_t p)
char *msgset;
char *data;
int status;
char *end;
msgset = imap4d_parsebuf_next (p, 1);
data = imap4d_parsebuf_next (p, 1);
......@@ -70,20 +70,10 @@ store_thunk (imap4d_parsebuf_t p)
}
/* Get the message numbers in set[]. */
status = util_msgset (msgset, &pclos->set, &pclos->count, pclos->isuid);
switch (status)
{
case 0:
break;
case EINVAL:
/* See RFC 3501, section 6.4.8, and a comment to the equivalent code
in fetch.c */
return RESP_OK;
default:
imap4d_parsebuf_exit (p, "Failed to parse message set");
}
status = util_parse_msgset (msgset, pclos->isuid, mbox,
&pclos->msgnumlist, &end);
if (status)
imap4d_parsebuf_exit (p, "Failed to parse message set");
if (p->token[0] != '(')
imap4d_parsebuf_exit (p, "Syntax error");
......@@ -101,12 +91,59 @@ store_thunk (imap4d_parsebuf_t p)
return RESP_OK;
}
static int
_do_store (size_t msgno, void *data)
{
struct store_parse_closure *pclos = data;
mu_message_t msg = NULL;
mu_attribute_t attr = NULL;
mu_mailbox_get_message (mbox, msgno, &msg);
mu_message_get_attribute (msg, &attr);
switch (pclos->how)
{
case STORE_ADD:
mu_attribute_set_flags (attr, pclos->type);
break;
case STORE_UNSET:
mu_attribute_unset_flags (attr, pclos->type);
break;
case STORE_SET:
mu_attribute_unset_flags (attr, 0xffffffff); /* FIXME */
mu_attribute_set_flags (attr, pclos->type);
}
if (pclos->ack)
{
io_sendf ("* %lu FETCH (", (unsigned long) msgno);
if (pclos->isuid)
{
size_t uid;
int rc = mu_mailbox_translate (mbox, MU_MAILBOX_UID_TO_MSGNO,
msgno, &uid);
if (rc == 0)
io_sendf ("UID %lu ", (unsigned long) uid);
}
io_sendf ("FLAGS (");
util_print_flags (attr);
io_sendf ("))\n");
}
/* Update the flags of uid table. */
imap4d_sync_flags (msgno);
return 0;
}
int
imap4d_store0 (imap4d_tokbuf_t tok, int isuid, char **ptext)
{
int rc;
struct store_parse_closure pclos;
memset (&pclos, 0, sizeof pclos);
pclos.ack = 1;
pclos.isuid = isuid;
......@@ -118,53 +155,12 @@ imap4d_store0 (imap4d_tokbuf_t tok, int isuid, char **ptext)
ptext);
if (rc == RESP_OK)
{
size_t i;
for (i = 0; i < pclos.count; i++)
{
mu_message_t msg = NULL;
mu_attribute_t attr = NULL;
size_t msgno = isuid ? uid_to_msgno (pclos.set[i]) : pclos.set[i];
if (msgno)
{
mu_mailbox_get_message (mbox, msgno, &msg);
mu_message_get_attribute (msg, &attr);
switch (pclos.how)
{
case STORE_ADD:
mu_attribute_set_flags (attr, pclos.type);
break;
case STORE_UNSET:
mu_attribute_unset_flags (attr, pclos.type);
break;
case STORE_SET:
mu_attribute_unset_flags (attr, 0xffffffff); /* FIXME */
mu_attribute_set_flags (attr, pclos.type);
}
}
if (pclos.ack)
{
io_sendf ("* %lu FETCH (", (unsigned long) msgno);
if (isuid)
io_sendf ("UID %lu ", (unsigned long) msgno);
io_sendf ("FLAGS (");
util_print_flags (attr);
io_sendf ("))\n");
}
/* Update the flags of uid table. */
imap4d_sync_flags (pclos.set[i]);
}
util_foreach_message (pclos.msgnumlist, _do_store, &pclos);
*ptext = "Completed";
}
free (pclos.set);
mu_list_destroy (&pclos.msgnumlist);
return rc;
}
......
......@@ -39,6 +39,11 @@ imap4d IMAP4D_OPTIONS < input | remove_uidvalidity | tr -d '\r'
* FLAGS (\Answered \Flagged \Deleted \Seen \Draft)
* OK [[PERMANENTFLAGS (\Answered \Flagged \Deleted \Seen \Draft)]] Permanent flags
1 OK [[READ-WRITE]] SELECT Completed
* 1 FETCH (FLAGS (\Recent))
* 2 FETCH (FLAGS (\Recent))
* 3 FETCH (FLAGS (\Recent))
* 4 FETCH (FLAGS (\Recent))
* 5 FETCH (FLAGS (\Recent))
2 OK FETCH Completed
* 1 FETCH (UID 1 FLAGS (\Recent))
* 2 FETCH (UID 2 FLAGS (\Recent))
......
......@@ -141,13 +141,13 @@ int mu_list_get_iterator (mu_list_t _list, mu_iterator_t *_pitr);
/* A general-purpose iteration function. When called, _item points to
the item currently visited and _data points to call-specific data. */
typedef int mu_list_action_t (void *_item, void *_data);
typedef int (*mu_list_action_t) (void *_item, void *_data);
/* Execute _action for each element in _list. Use _data as the call-specific
data. */
int mu_list_foreach (mu_list_t _list, mu_list_action_t *_action, void *_data);
int mu_list_foreach (mu_list_t _list, mu_list_action_t _action, void *_data);
/* A historical alias to the above. */
int mu_list_do (mu_list_t, mu_list_action_t *, void *) MU_DEPRECATED;
int mu_list_do (mu_list_t, mu_list_action_t, void *) MU_DEPRECATED;
/* ************************************************* */
/* Functions for combining two lists. */
......
......@@ -24,7 +24,7 @@
#include <mailutils/errno.h>
int
mu_list_foreach (mu_list_t list, mu_list_action_t *action, void *cbdata)
mu_list_foreach (mu_list_t list, mu_list_action_t action, void *cbdata)
{
mu_iterator_t itr;
int status = 0;
......@@ -49,7 +49,7 @@ mu_list_foreach (mu_list_t list, mu_list_action_t *action, void *cbdata)
/* Retained for compatibility with previous versions.
In the future it will be removed, or changed to a define or weak alias. */
int
mu_list_do (mu_list_t list, mu_list_action_t *action, void *cbdata)
mu_list_do (mu_list_t list, mu_list_action_t action, void *cbdata)
{
return mu_list_foreach (list, action, cbdata);
}
......