Various fixes in imap4d.

Imap4d successfully passes all tests.

* imap4d/close.c (imap4d_close0): Fix improper use of MU_STREAM_READ
(flags changed their semantics since then).
* imap4d/fetch.c: Send \n terminated lines, rely on filters to recode
line terminators to \r\n.
(fetch_io): Rewrite using CRLF encoder and util_copy_out.
(_frt_header_fields): Rewind memory stream before reading from it.

* imap4d/auth_gsasl.c: Send \n terminated lines, rely on filters
to recode line terminators to \r\n.
* imap4d/auth_gss.c: Likewise.
* imap4d/capability.c: Likewise.
* imap4d/copy.c: Likewise.
* imap4d/id.c: Likewise.
* imap4d/idle.c: Likewise.
* imap4d/list.c: Likewise.
* imap4d/namespace.c: Likewise.
* imap4d/preauth.c: Likewise.
* imap4d/search.c: Likewise.
* imap4d/status.c: Likewise.
* imap4d/store.c: Likewise.
* imap4d/select.c: Likewise.
(imap4d_select_status): Fix improper use of MU_STREAM_READ.

* imap4d/util.c: Send \n terminated lines, rely on filters
to recode line terminators to \r\n.
(util_setio): Apply CRLF filters to both input and output streams
(in opposite directions).
(util_copy_out): New function.
(remove_cr): Remove.
* imap4d/imap4d.h (util_copy_out): New prototype.

@@ -95,7 +95,7 @@ auth_gsasl (struct imap4d_command *command, char *auth_type, char **username)
   while ((rc = gsasl_step64 (sess_ctx, input_str, &output))
 	   == GSASL_NEEDS_MORE)
     {
-      util_send ("+ %s\r\n", output);
+      util_send ("+ %s\n", output);
       imap4d_getline (&input_str, &input_size, &input_len);
     }
   
@@ -112,7 +112,7 @@ auth_gsasl (struct imap4d_command *command, char *auth_type, char **username)
      returned, and clients must respond with an empty response. */
   if (output[0])
     {
-      util_send ("+ %s\r\n", output);
+      util_send ("+ %s\n", output);
       imap4d_getline (&input_str, &input_size, &input_len);
       if (input_len != 0)
 	{

@@ -163,7 +163,7 @@ auth_gssapi (struct imap4d_command *command,
 
   /* Start the dialogue */
 
-  util_send ("+ \r\n");
+  util_send ("+ \n");
   util_flush_output ();
   
   context = GSS_C_NO_CONTEXT;
@@ -192,7 +192,7 @@ auth_gssapi (struct imap4d_command *command,
 	  if (outbuf.length)
 	    {
 	      mu_base64_encode (outbuf.value, outbuf.length, &tmp, &size);
-	      util_send ("+ %s\r\n", tmp);
+	      util_send ("+ %s\n", tmp);
 	      free (tmp);
 	      gss_release_buffer (&min_stat, &outbuf);
 	    }
@@ -212,7 +212,7 @@ auth_gssapi (struct imap4d_command *command,
   if (outbuf.length)
     {
       mu_base64_encode (outbuf.value, outbuf.length, &tmp, &size);
-      util_send ("+ %s\r\n", tmp);
+      util_send ("+ %s\n", tmp);
       free (tmp);
       gss_release_buffer (&min_stat, &outbuf);
       imap4d_getline (&token_str, &token_size, &token_len);
@@ -232,7 +232,7 @@ auth_gssapi (struct imap4d_command *command,
     }
   
   mu_base64_encode (outbuf.value, outbuf.length, &tmp, &size);
-  util_send ("+ %s\r\n", tmp);
+  util_send ("+ %s\n", tmp);
   free (tmp);
 
   imap4d_getline (&token_str, &token_size, &token_len);

@@ -80,7 +80,7 @@ imap4d_capability (struct imap4d_command *command, imap4d_tokbuf_t tok)
   mu_list_do (capa_list, print_capa, NULL);
   
   imap4d_auth_capability ();
-  util_send ("\r\n");
+  util_send ("\n");
 
   return util_finish (command, RESP_OK, "Completed");
 }

@@ -30,7 +30,7 @@ imap4d_close0 (struct imap4d_command *command, imap4d_tokbuf_t tok,
     return util_finish (command, RESP_BAD, "Invalid arguments");
   
   mu_mailbox_get_flags (mbox, &flags);
-  if ((flags & MU_STREAM_READ) == 0)
+  if (flags & MU_STREAM_WRITE)
     {
       status = mu_mailbox_flush (mbox, expunge);
       if (status)

@@ -52,7 +52,7 @@ imap4d_copy (struct imap4d_command *command, imap4d_tokbuf_t tok)
       int new_state = (rc == RESP_OK) ? command->success : command->failure;
       if (new_state != STATE_NONE)
 	state = new_state;
-      return util_send ("%s %s\r\n", command->tag, text);
+      return util_send ("%s %s\n", command->tag, text);
     }
   return util_finish (command, rc, "%s", text);
 }

@@ -690,39 +690,36 @@ static int
 fetch_io (mu_stream_t stream, size_t start, size_t size, size_t max)
 {
   mu_stream_t rfc = NULL;
+  
   size_t n = 0;
 
   mu_filter_create (&rfc, stream, "rfc822", MU_FILTER_ENCODE,
-		    MU_STREAM_READ|MU_STREAM_NO_CHECK|MU_STREAM_NO_CLOSE);
-
+		    MU_STREAM_READ|MU_STREAM_SEEK|MU_STREAM_NO_CLOSE);
+  
   if (start == 0 && size == (size_t) -1)
     {
-      char *buffer;
-      size_t bufsize;
       int rc;
       
-      for (bufsize = max; (buffer = malloc (bufsize)) == NULL; bufsize /= 2)
-	if (bufsize < 512)
-  	  imap4d_bye (ERR_NO_MEM);
-
       rc = mu_stream_seek (rfc, 0, MU_SEEK_SET, NULL);
       if (rc)
 	{
-	  mu_error ("seek error: %s", mu_stream_strerror (rfc, rc));
+	  mu_error ("seek error: %s", mu_stream_strerror (stream, rc));
 	  return RESP_BAD;
 	}
       if (max)
 	{
-	  util_send (" {%lu}\r\n", (unsigned long) max);
-	  while (mu_stream_read (rfc, buffer, bufsize, &n) == 0 && n > 0)
-	    util_send_bytes (buffer, n);
-
+	  util_send (" {%lu}\n", (unsigned long) max);
+	  util_copy_out (rfc, max);
 	  /* FIXME: Make sure exactly max bytes were sent */
-          free (buffer); 
 	}
       else
 	util_send (" \"\"");
     }
+  else if (start > max)
+    {
+      util_send ("<%lu>", (unsigned long) start);
+      util_send (" \"\"");
+    }
   else if (size + 2 < size) /* Check for integer overflow */
     {
       mu_stream_destroy (&rfc);
@@ -738,16 +735,17 @@ fetch_io (mu_stream_t stream, size_t start, size_t size, size_t max)
       if (!p)
 	imap4d_bye (ERR_NO_MEM);
 
-      rc = mu_stream_seek (stream, start, MU_SEEK_SET, NULL);
+      rc = mu_stream_seek (rfc, start, MU_SEEK_SET, NULL);
       if (rc)
 	{
 	  mu_error ("seek error: %s", mu_stream_strerror (rfc, rc));
 	  free (buffer);
+	  mu_stream_destroy (&rfc);
 	  return RESP_BAD;
 	}
 
       while (total < size
-	     && mu_stream_read (rfc, p, size - total + 1, &n) == 0
+	     && mu_stream_read (rfc, p, size - total, &n) == 0
 	     && n > 0)
 	{
 	  total += n;
@@ -757,7 +755,7 @@ fetch_io (mu_stream_t stream, size_t start, size_t size, size_t max)
       util_send ("<%lu>", (unsigned long) start);
       if (total)
 	{
-	  util_send (" {%lu}\r\n", (unsigned long) total);
+	  util_send (" {%lu}\n", (unsigned long) total);
 	  util_send_bytes (buffer, total);
 	}
       else
@@ -1071,6 +1069,7 @@ _frt_header_fields (struct fetch_function_closure *ffc,
   
   /* Output collected data */
   mu_stream_size (stream, &size);
+  mu_stream_seek (stream, 0, MU_SEEK_SET, NULL);
   status = fetch_io (stream, ffc->start, ffc->size, size + lines);
   mu_stream_destroy (&stream);
   
@@ -1648,7 +1647,7 @@ imap4d_fetch0 (imap4d_tokbuf_t tok, int isuid, char **err_text)
 	      util_send ("* %lu FETCH (", (unsigned long) frc.msgno);
 	      frc.eltno = 0;
 	      rc = mu_list_do (pclos.fnlist, _do_fetch, &frc);
-	      util_send (")\r\n");
+	      util_send (")\n");
 	    }
 	}
       }
@@ -1684,4 +1683,3 @@ imap4d_fetch (struct imap4d_command *command, imap4d_tokbuf_t tok)
   rc = imap4d_fetch0 (tok, 0, &err_text);
   return util_finish (command, rc, "%s", err_text);
 }
-

@@ -202,7 +202,7 @@ imap4d_id (struct imap4d_command *command, imap4d_tokbuf_t tok)
 	}
       mu_iterator_destroy (&itr);
       if (outcnt)
-	util_send (")\r\n");
+	util_send (")\n");
     }
   return util_finish (command, RESP_OK, "Completed");
 }

@@ -32,7 +32,7 @@ imap4d_idle (struct imap4d_command *command, imap4d_tokbuf_t tok)
   if (util_wait_input (0) == -1)
     return util_finish (command, RESP_NO, "Cannot idle");
 
-  util_send ("+ idling\r\n");
+  util_send ("+ idling\n");
   util_flush_output ();
 
   start = time (NULL);

@@ -202,7 +202,7 @@ extern int imap4d_transcript;
 extern mu_list_t imap4d_id_list;
 extern int imap4d_argc;                 
 extern char **imap4d_argv;
-  
+
 #ifndef HAVE_STRTOK_R
 extern char *strtok_r (char *s, const char *delim, char **save_ptr);
 #endif
@@ -334,6 +334,8 @@ extern int  util_send (const char *, ...) MU_PRINTFLIKE(1,2);
 extern int  util_send_bytes (const char *buf, size_t size);
 extern int  util_send_qstring (const char *);
 extern int  util_send_literal (const char *);
+extern int  util_copy_out (mu_stream_t str, size_t size);
+
 extern int  util_start (char *);
 extern int  util_finish (struct imap4d_command *, int, const char *, ...) 
                          MU_PRINTFLIKE(3,4);

@@ -97,11 +97,11 @@ list_fun (mu_folder_t folder, struct mu_list_response *resp, void *data)
   name = refinfo->buf;
   
   if (strpbrk (name, "\"{}"))
-    util_send ("{%lu}\r\n%s\r\n", (unsigned long) strlen (name), name);
+    util_send ("{%lu}\n%s\n", (unsigned long) strlen (name), name);
   else if (is_atom (name))
-    util_send ("%s\r\n", name);
+    util_send ("%s\n", name);
   else
-    util_send ("\"%s\"\r\n", name);
+    util_send ("\"%s\"\n", name);
   return 0;
 }
 

@@ -126,7 +126,7 @@ imap4d_namespace (struct imap4d_command *command, imap4d_tokbuf_t tok)
   print_namespace (NS_OTHER);
   util_send (" ");
   print_namespace (NS_SHARED);
-  util_send ("\r\n");
+  util_send ("\n");
 
   return util_finish (command, RESP_OK, "Completed");
 }

@@ -366,7 +366,7 @@ do_preauth_ident (struct sockaddr *clt_sa, struct sockaddr *srv_sa)
       return NULL;
     }
 
-  mu_stream_printf (stream, "%u , %u\r\n",
+  mu_stream_printf (stream, "%u , %u\n",
 		    ntohs (clt_addr->sin_port),
 		    ntohs (srv_addr->sin_port));
   mu_stream_shutdown (stream, MU_STREAM_WRITE);

@@ -363,7 +363,7 @@ do_search (struct parsebuf *pb)
 	    util_send (" %s", mu_umaxtostr (0, pb->msgno));
 	}
     }
-  util_send ("\r\n");
+  util_send ("\n");
 }
 
 /* Parse buffer functions */

@@ -74,9 +74,9 @@ imap4d_select0 (struct imap4d_command *command, const char *mboxname,
 	{
 	  free (mailbox_name);
 	  /* Need to set the state explicitely for select.  */
-	  return util_send ("%s OK [%s] %s Completed\r\n", command->tag,
-			    (flags & MU_STREAM_READ)  ?
-			    "READ-ONLY" : "READ-WRITE", command->name);
+	  return util_send ("%s OK [%s] %s Completed\n", command->tag,
+			    ((flags & MU_STREAM_RDWR) == MU_STREAM_RDWR) ?
+			    "READ-WRITE" : "READ-ONLY", command->name);
 	}
     }
   
@@ -119,7 +119,7 @@ imap4d_select_status ()
   /* FIXME:
      - '\*' can be supported if we use the attribute_set userflag()
      - Answered is still not set in the mailbox code.  */
-  if (select_flags & MU_STREAM_READ)
+  if (!(select_flags & MU_STREAM_WRITE))
     util_out (RESP_OK, "[PERMANENTFLAGS ()] No Permanent flags");
   else
     util_out (RESP_OK, "[PERMANENTFLAGS (%s)] Permanent flags", pflags);

@@ -136,7 +136,7 @@ imap4d_status (struct imap4d_command *command, imap4d_tokbuf_t tok)
 
 	  
 	  if (count > 0)
-	    util_send (")\r\n");
+	    util_send (")\n");
 	  mu_mailbox_close (smbox);
 	}
       mu_mailbox_destroy (&smbox);

@@ -157,7 +157,7 @@ imap4d_store0 (imap4d_tokbuf_t tok, int isuid, char **ptext)
 		util_send ("UID %lu ", (unsigned long) msgno);
 	      util_send ("FLAGS (");
 	      util_print_flags (attr);
-	      util_send ("))\r\n");
+	      util_send ("))\n");
 	    }
 	  /* Update the flags of uid table.  */
 	  imap4d_sync_flags (pclos.set[i]);

@@ -270,6 +270,12 @@ util_msgset (char *s, size_t ** set, int *n, int isuid)
 }
 
 int
+util_copy_out (mu_stream_t str, size_t size)
+{
+  return mu_stream_copy (ostream, str, size);
+}
+
+int
 util_send_bytes (const char *buf, size_t size)
 {
   return mu_stream_write (ostream, buf, size, NULL);
@@ -325,7 +331,7 @@ util_send_qstring (const char *buffer)
 int
 util_send_literal (const char *buffer)
 {
-  return util_send ("{%lu}\r\n%s", (unsigned long) strlen (buffer), buffer);
+  return util_send ("{%lu}\n%s", (unsigned long) strlen (buffer), buffer);
 }
 
 /* Send an unsolicited response.  */
@@ -337,7 +343,7 @@ util_out (int rc, const char *format, ...)
   int status = 0;
   va_list ap;
 
-  asprintf (&tempbuf, "* %s%s\r\n", sc2string (rc), format);
+  asprintf (&tempbuf, "* %s%s\n", sc2string (rc), format);
   va_start (ap, format);
   vasprintf (&buf, tempbuf, ap);
   va_end (ap);
@@ -346,7 +352,7 @@ util_out (int rc, const char *format, ...)
 
   if (imap4d_transcript)
     {
-      int len = strcspn (buf, "\r\n");
+      int len = strcspn (buf, "\n");
       mu_diag_output (MU_DIAG_DEBUG, "sent: %*.*s", len, len, buf);
     }
 
@@ -393,7 +399,7 @@ util_finish (struct imap4d_command *command, int rc, const char *format, ...)
 
   mu_stream_write (ostream, buf, strlen (buf), NULL);
   free (buf);
-  mu_stream_write (ostream, "\r\n", 2, NULL);
+  mu_stream_write (ostream, "\n", 2, NULL);
 
   /* Reset the state.  */
   if (rc == RESP_OK)
@@ -800,16 +806,24 @@ util_uidvalidity (mu_mailbox_t smbox, unsigned long *uidvp)
 void
 util_setio (FILE *in, FILE *out)
 {
+  mu_stream_t tmp;
+  
   if (!in)
     imap4d_bye (ERR_NO_IFILE);
   if (!out)
     imap4d_bye (ERR_NO_OFILE);
 
-  if (mu_stdio_stream_create (&istream, fileno (in), MU_STREAM_NO_CLOSE))
+  if (mu_stdio_stream_create (&tmp, fileno (in), MU_STREAM_NO_CLOSE))
     imap4d_bye (ERR_NO_IFILE);
-  if (mu_stdio_stream_create (&ostream, fileno (out), MU_STREAM_NO_CLOSE))
-    imap4d_bye (ERR_NO_OFILE);
+  mu_stream_set_buffer (tmp, mu_buffer_line, 1024);
+  mu_filter_create (&istream, tmp, "rfc822", MU_FILTER_DECODE, MU_STREAM_READ);
   mu_stream_set_buffer (istream, mu_buffer_line, 1024);
+  
+  if (mu_stdio_stream_create (&tmp, fileno (out), MU_STREAM_NO_CLOSE))
+    imap4d_bye (ERR_NO_OFILE);
+  mu_stream_set_buffer (tmp, mu_buffer_line, 1024);
+  mu_filter_create (&ostream, tmp, "rfc822", MU_FILTER_ENCODE,
+		    MU_STREAM_WRITE);
   mu_stream_set_buffer (ostream, mu_buffer_line, 1024);
 }
 
@@ -917,7 +931,7 @@ void
 util_bye ()
 {
   int rc = istream != ostream;
-  
+
   mu_stream_close (istream);
   mu_stream_destroy (&istream);
 
@@ -926,7 +940,6 @@ util_bye ()
       mu_stream_close (ostream);
       mu_stream_destroy (&ostream);
     }
-      
   mu_list_do (atexit_list, atexit_run, 0);
 }
 
@@ -1021,30 +1034,6 @@ is_atom (const char *s)
      
 
 static size_t
-remove_cr (char *line, size_t len)
-{
-  char *prev = NULL;
-  size_t rlen = len;
-  char *p;
-  while ((p = memchr (line, '\r', len)))
-    {
-      if (prev)
-	{
-	  memmove (prev, line, p - line);
-	  prev += p - line;
-	}
-      else
-	prev = p;
-      rlen--;
-      len -= p - line + 1;
-      line = p + 1;
-    }
-  if (prev)
-    memmove (prev, line, len);
-  return rlen;
-}
-
-static size_t
 unquote (char *line, size_t len)
 {
   char *prev = NULL;
@@ -1318,7 +1307,7 @@ imap4d_readline (struct imap4d_tokbuf *tok)
 	  /* Client can ask for non-synchronised literal,
 	     if a '+' is appended to the octet count. */
 	  if (*sp == '}')
-	    util_send ("+ GO AHEAD\r\n");
+	    util_send ("+ GO AHEAD\n");
 	  else if (*sp != '+')
 	    break;
 	  imap4d_tokbuf_expand (tok, number + 1);
@@ -1334,7 +1323,6 @@ imap4d_readline (struct imap4d_tokbuf *tok)
                len += sz;
             }
 	  check_input_err (rc, len);
-	  len = remove_cr (buf, len);
 	  imap4d_tokbuf_unquote (tok, &off, &len);
 	  tok->level += len;
 	  tok->buffer[tok->level++] = 0;