diff --git a/bbs/socket.c b/bbs/socket.c index 8cff7bb..8a5a76f 100644 --- a/bbs/socket.c +++ b/bbs/socket.c @@ -34,7 +34,6 @@ #include #ifdef __linux__ -#define SOL_TCP 6 /* TCP level */ /* includes some of the same stuff as , so don't include both, just one or the other */ #include #else @@ -315,6 +314,24 @@ int bbs_block_fd(int fd) return 0; } +int bbs_node_cork(struct bbs_node *node, int enabled) +{ + int i = enabled; + /* This is TCP-specific, so it has to operate on the actual socket file descriptor, + * not any of the other node file descriptors. */ +#ifdef __FreeBSD__ + if (setsockopt(node->sfd, IPPROTO_TCP, TCP_NOPUSH, &i, sizeof(i))) { /* FreeBSD */ +#else + if (setsockopt(node->sfd, IPPROTO_TCP, TCP_CORK, &i, sizeof(i))) { /* Linux */ +#endif + bbs_error("setsockopt failed: %s\n", strerror(errno)); + return -1; + } else { + bbs_debug(3, "%s corking on node %d\n", enabled ? "Enabled" : "Disabled", node->id); + } + return 0; +} + int bbs_set_fd_tcp_nodelay(int fd, int enabled) { int i = enabled; @@ -380,7 +397,7 @@ int bbs_node_wait_until_output_sent(struct bbs_node *node) /* If it's the network link that's actually slow, * then the acks will take some time to come back. */ for (;;) { - if (getsockopt(node->sfd, SOL_TCP, TCP_INFO, &info, &len)) { + if (getsockopt(node->sfd, IPPROTO_TCP, TCP_INFO, &info, &len)) { bbs_error("getsockopt failed: %s\n", strerror(errno)); return -1; } diff --git a/include/mod_mimeparse.h b/include/mod_mimeparse.h index 5101b93..9fabae9 100644 --- a/include/mod_mimeparse.h +++ b/include/mod_mimeparse.h @@ -1,22 +1,52 @@ -/* - * LBBS -- The Lightweight Bulletin Board System - * - * Copyright (C) 2023, Naveen Albert - * - * Naveen Albert - * - */ - -/*! \file - * - * \brief MIME Parser Supplements for IMAP Server - * - */ - -/*! - * \brief Generate the BODY/BODYSTRUCTURE data item for FETCH responses - * \param itemname BODY or BODYSTRUCTURE - * \param file File containing email message - * \returns NULL on failure, BODYSTRUCTURE text on success, which must be be freed using free() - */ -char *mime_make_bodystructure(const char *itemname, const char *file); +/* + * LBBS -- The Lightweight Bulletin Board System + * + * Copyright (C) 2023, Naveen Albert + * + * Naveen Albert + * + */ + +/*! \file + * + * \brief MIME Parser Supplements for IMAP Server + * + */ + +struct bbs_mime_message; + +/*! + * \brief Create a MIME message structure by parsing a message file + * \param filename Path to email message + * \return NULL on failure, opaque MIME structure on success which must be destroyed using bbs_mime_message_parse + */ +struct bbs_mime_message *bbs_mime_message_parse(const char *filename) __attribute__((nonnull (1))); + +/*! + * \brief Destroy a MIME message structure + */ +void bbs_mime_message_destroy(struct bbs_mime_message *mime) __attribute__((nonnull (1))); + +/*! + * \brief Generate the BODY/BODYSTRUCTURE data item for FETCH responses + * \param mime + * \returns NULL on failure, BODYSTRUCTURE text on success, which must be be freed using free() + */ +char *bbs_mime_make_bodystructure(struct bbs_mime_message *mime) __attribute__((nonnull (1))); + +enum mime_part_filter { + MIME_PART_FILTER_ALL = 0, /* Retrieve the entire part */ + MIME_PART_FILTER_MIME, /* Retrieve just the MIME of the part */ + MIME_PART_FILTER_HEADERS, /* Retrieve the headers, but only if the part's Content-Type is message/rfc822 */ + MIME_PART_FILTER_TEXT, /* Retrieve the text, but only if the part's Content-Type is message/rfc822 */ +}; + +/*! + * \brief Retrieve a particular part of the body, by part specification + * \param mime + * \param spec Part specification, e.g. 2.3. This should be ONLY the part number portion of the spec (e.g. not including .MIME, .TEXT, etc.) + * \param[out] outlen Length of returned part, if return value is non-NULL + * \param[in] filter What to retrieve and return + * \returns NULL on failure, requested section on success, which must be freed using free() + */ +char *bbs_mime_get_part(struct bbs_mime_message *mime, const char *spec, size_t *restrict outlen, enum mime_part_filter filter) __attribute__((nonnull (1, 2, 3))); diff --git a/include/socket.h b/include/socket.h index 070b084..df96bf1 100644 --- a/include/socket.h +++ b/include/socket.h @@ -55,6 +55,16 @@ int bbs_unblock_fd(int fd); /*! \brief Put a socket in blocking mode */ int bbs_block_fd(int fd); +/*! + * \brief Cork or uncork a node's TCP session + * \param node + * \param enabled 1 to buffer data in the kernel until full packets are available to send, 0 to disable + * \note If enabled, this MUST be disabled at some point to ensure pending data is fully written! + * \note You should not use this function unless no better alternative is available, use with caution! + * \retval 0 on success, -1 on failure + */ +int bbs_node_cork(struct bbs_node *node, int enabled); + /*! * \brief Enable or disable Nagle's algorithm * \param fd diff --git a/modules/mod_mimeparse.c b/modules/mod_mimeparse.c index 35d3947..950ddc2 100644 --- a/modules/mod_mimeparse.c +++ b/modules/mod_mimeparse.c @@ -17,7 +17,7 @@ * \author Naveen Albert */ -/* XXX Not sure which this is necessary, but it is... */ +/* XXX Not sure why this is necessary, but it is... */ #define BBS_LOCK_WRAPPERS_NOWARN #include "include/bbs.h" @@ -343,20 +343,13 @@ static void write_part_bodystructure(GMimeObject *part, GString *gs, int level) g_string_append_c(gs, ')'); } -char *mime_make_bodystructure(const char *itemname, const char *file) +static GMimeMessage *mk_mime(const char *file) { GMimeFormat format = GMIME_FORMAT_MESSAGE; GMimeMessage *message; GMimeParser *parser; GMimeStream *stream; - GString *str; - gchar *result; int fd; -#ifdef CHECK_VALIDITY - int p = 0; - int in_quoted = 0; - char *s; -#endif fd = open(file, O_RDONLY, 0); if (fd < 0) { @@ -376,12 +369,45 @@ char *mime_make_bodystructure(const char *itemname, const char *file) close(fd); if (!message) { - bbs_error("Failed to parse message as MIME\n"); + bbs_error("Failed to parse message %s as MIME\n", file); + } + + return message; +} + +/* Opaque structure for MIME message, so other modules can reuse this for multiple operations */ +struct bbs_mime_message { + GMimeMessage *message; +}; + +struct bbs_mime_message *bbs_mime_message_parse(const char *filename) +{ + struct bbs_mime_message *mime = malloc(sizeof(*mime)); + if (ALLOC_FAILURE(mime)) { return NULL; } + mime->message = mk_mime(filename); + return mime; +} + +void bbs_mime_message_destroy(struct bbs_mime_message *mime) +{ + g_object_unref(mime->message); + free(mime); +} + +char *bbs_mime_make_bodystructure(struct bbs_mime_message *mime) +{ + GMimeMessage *message = mime->message; + GString *str; + gchar *result; +#ifdef CHECK_VALIDITY + int p = 0; + int in_quoted = 0; + char *s; +#endif str = g_string_new(""); - g_string_append_printf(str, "%s ", itemname); write_part_bodystructure(message->mime_part, str, 0); #ifdef CHECK_VALIDITY @@ -420,10 +446,154 @@ char *mime_make_bodystructure(const char *itemname, const char *file) #endif result = g_string_free(str, FALSE); /* Free the g_string but not the buffer */ - g_object_unref(message); return result; /* gchar is just a typedef for char, so this returns a char */ } +#if SEMVER_VERSION(GMIME_MAJOR_VERSION, GMIME_MINOR_VERSION, GMIME_MICRO_VERSION) < SEMVER_VERSION(3, 2, 8) +/* This function was only added in gmime commit 3efc24a6cdc88198e11e43f23e03e32f12b13bd8, + * so older packages of the library don't have it. Define it manually for those cases. */ +static ssize_t g_mime_object_write_content_to_stream(GMimeObject *object, GMimeFormatOptions *options, GMimeStream *stream) +{ + g_return_val_if_fail (GMIME_IS_OBJECT (object), -1); + g_return_val_if_fail (GMIME_IS_STREAM (stream), -1); + + return GMIME_OBJECT_GET_CLASS(object)->write_to_stream(object, options, TRUE, stream); +} +#endif + +static void write_part(GMimeObject *part, GMimeStream *stream, enum mime_part_filter filter) +{ + char *buf; + GMimeMessage *message; + GMimeContentType *content_type; + const char *subtype; + GMimeFormatOptions *format = g_mime_format_options_get_default(); + +#define IS_RFC822(part) (!strcasecmp(g_mime_content_type_get_media_type(content_type), "message") && subtype && !strcasecmp(subtype, "rfc822")) + + content_type = g_mime_object_get_content_type(part); + subtype = g_mime_content_type_get_media_subtype(content_type); + + switch (filter) { + case MIME_PART_FILTER_TEXT: + /* If the type is message/rfc822, proceed. If not, illegal (just return empty). */ + if (!IS_RFC822(part)) { + bbs_debug(1, "Ignoring request for TEXT, since its type is %s/%s\n", g_mime_content_type_get_media_type(content_type), S_IF(subtype)); + return; + } + /* Might be a more elegant way to do this with gmime, but not sure what it is at the moment. + * For now, return the whole thing, and later strip out the headers. */ + /* Fall through */ + case MIME_PART_FILTER_ALL: + if (g_mime_object_write_content_to_stream((GMimeObject *) part, format, stream) == -1) { + bbs_warning("Failed to write part to stream\n"); + } + break; + case MIME_PART_FILTER_MIME: + /* XXX In theory, since we get a string, we could return this directly, + * rather than adding to a stream and then allocating another string. + * However, the stream allows us to be a little more abstract here. */ + buf = g_mime_object_get_headers(part, format); + if (!strlen_zero(buf)) { + g_mime_stream_printf(stream, "%s\r\n", buf); /* Need to include end of headers */ + } + g_free(buf); + break; + case MIME_PART_FILTER_HEADERS: + /* If the type is message/rfc822, proceed. If not, illegal (just return empty). */ + if (!IS_RFC822(part)) { + bbs_debug(1, "Ignoring request for HEADERS, since its type is %s/%s\n", g_mime_content_type_get_media_type(content_type), S_IF(subtype)); + return; + } + /* XXX Same thing here about returning a string directly */ + message = g_mime_message_part_get_message((GMimeMessagePart *) part); + buf = g_mime_object_get_headers((GMimeObject *) message, format); + if (!strlen_zero(buf)) { + g_mime_stream_printf(stream, "%s\r\n", buf); /* Need to include end of headers */ + } + g_free(buf); + break; + } +} + +/*! * \brief Get the contents of a MIME part by part number */ +static int get_part(GMimeMessage *message, GMimeStream *mem, const char *spec, enum mime_part_filter filter) +{ + GMimePartIter *iter; + GMimeObject *part; + + iter = g_mime_part_iter_new((GMimeObject *) message); + if (!g_mime_part_iter_is_valid(iter)) { + bbs_warning("Part iteration is invalid\n"); + g_mime_part_iter_free(iter); + return -1; + } + if (!g_mime_part_iter_jump_to(iter, spec)) { + bbs_warning("Failed to fetch part number %s\n", spec); + g_mime_part_iter_free(iter); + return -1; + } + + part = g_mime_part_iter_get_current(iter); + write_part(part, mem, filter); + g_mime_part_iter_free(iter); + return 0; +} + +char *bbs_mime_get_part(struct bbs_mime_message *mime, const char *spec, size_t *restrict outlen, enum mime_part_filter filter) +{ + GMimeMessage *message = mime->message; + GMimeStream *mem; + GByteArray *buffer; + char *buf; + unsigned char *bufdata; + size_t buflen; + + mem = g_mime_stream_mem_new(); + if (!mem) { + bbs_error("Failed to allocate stream buffer\n"); + return NULL; + } + + if (get_part(message, mem, spec, filter)) { + g_object_unref(mem); + return NULL; + } + + buffer = g_mime_stream_mem_get_byte_array((GMimeStreamMem *) mem); + + if (filter == MIME_PART_FILTER_TEXT) { + char *eoh; + size_t diff; + /* Now we have to pay the piper... skip past the headers. */ + eoh = memmem(buffer->data, buffer->len, "\r\n\r\n", STRLEN("\r\n\r\n")); + if (!eoh) { + bbs_debug(3, "Message has no body, just headers, if that...\n"); + g_object_unref(mem); + return NULL; + } + diff = (size_t) (eoh - (char*) buffer->data); + diff += STRLEN("\r\n\r\n"); + bufdata = buffer->data + diff; + buflen = buffer->len - diff; + } else { + bufdata = buffer->data; + buflen = buffer->len; + } + + buf = malloc(buflen + 1); + if (ALLOC_FAILURE(buf)) { + g_object_unref(mem); + return NULL; + } + memcpy(buf, bufdata, buflen); + buf[buflen] = '\0'; + *outlen = buflen; + g_object_unref(mem); + + return buf; /* gchar is just a typedef for char, so this returns a char */ +} + static int load_module(void) { g_mime_init(); @@ -434,7 +604,7 @@ static int load_module(void) static int unload_module(void) { - /* This doesn't free everything (possibly lost leaks in valgrind . See: + /* This doesn't free everything (possibly lost leaks in valgrind). See: * Q: https://mail.gnome.org/archives/gmime-devel-list/2012-November/msg00000.html * A: https://mail.gnome.org/archives/gmime-devel-list/2012-November/msg00001.html */ diff --git a/nets/net_imap/imap.h b/nets/net_imap/imap.h index 7f386dc..1cb2fa0 100644 --- a/nets/net_imap/imap.h +++ b/nets/net_imap/imap.h @@ -168,8 +168,10 @@ extern int imap_debug_level; #define __imap_parallel_reply(imap, fmt, ...) imap_debug(4, "%p <= " fmt, imap, ## __VA_ARGS__); bbs_node_any_fd_writef(imap->node, imap->node->wfd, fmt, ## __VA_ARGS__); #define _imap_reply_nolock_fd(imap, fd, fmt, ...) imap_debug(4, "%p <= " fmt, imap, ## __VA_ARGS__); bbs_node_fd_writef(imap->node, fd, fmt, ## __VA_ARGS__); +#define _imap_reply_nolock_fd_lognewline(imap, fd, fmt, ...) imap_debug(4, "%p <= " fmt "\r\n", imap, ## __VA_ARGS__); bbs_node_fd_writef(imap->node, fd, fmt, ## __VA_ARGS__); #define _imap_reply_nolock(imap, fmt, ...) _imap_reply_nolock_fd(imap, imap->node->wfd, fmt, ## __VA_ARGS__); #define _imap_reply(imap, fmt, ...) _imap_reply_nolock(imap, fmt, ## __VA_ARGS__) +#define imap_send_nocrlf(imap, fmt, ...) _imap_reply_nolock_fd_lognewline(imap, imap->node->wfd, fmt, ## __VA_ARGS__); #define imap_send(imap, fmt, ...) _imap_reply(imap, "%s " fmt "\r\n", "*", ## __VA_ARGS__) #define imap_reply(imap, fmt, ...) _imap_reply(imap, "%s " fmt "\r\n", imap->tag, ## __VA_ARGS__) diff --git a/nets/net_imap/imap_server_fetch.c b/nets/net_imap/imap_server_fetch.c index eb1bcd4..b321e10 100644 --- a/nets/net_imap/imap_server_fetch.c +++ b/nets/net_imap/imap_server_fetch.c @@ -239,171 +239,569 @@ static int process_fetch_envelope(const char *fullname, char *response, size_t r return 0; } -static int process_fetch_rfc822header(const char *fullname, char *response, size_t responselen, char **buf, int *len, - char *headers, size_t headerslen, int unoriginal, size_t *bodylen) +/*! \brief Adjust send offset and size based on partial data request */ +static void adjust_send_offset(struct fetch_body_request *fbr, off_t *restrict offset, size_t *restrict sendsize) { - FILE *fp; + int realskip = 0; + if (fbr->substart != -1) { + /* sendsize is currently how many bytes are left. + * So for this offset to work, + * fbr->substart must be at most sendsize, and we + * reduce sendsize by fbr->substart bytes. */ + realskip = MIN(fbr->substart, (int) *sendsize); + *offset += realskip; + *sendsize -= (size_t) realskip; + } + if (fbr->sublength != -1) { + /* fbr->sublength is an upperbound on the number of bytes to send */ + *sendsize = MIN((size_t) fbr->sublength, *sendsize); /* Can be at most sendsize. */ + } +} + +#define send_headers(imap, fbr, itemname, fp, fullname) send_filtered_headers(imap, fbr, itemname, fp, fullname, NULL, 0) + +static int send_filtered_headers(struct imap_session *imap, struct fetch_body_request *fbr, const char *itemname, FILE **restrict fp, const char *fullname, const char *headerlist, int filter) +{ + char headersbuf[8192]; + size_t headersbuflen = sizeof(headersbuf); + char *buf = headersbuf; + size_t len = headersbuflen; char linebuf[1001]; - char *headpos = headers; - size_t headlen = headerslen; + char listbuf[4096]; + int inside_match = filter ? 0 : 1; + size_t headerslen; + char *sendstart; + size_t sendsize; + + if (filter) { + size_t headerlistlen; + if (filter == 1) { + headerlist += STRLEN("HEADER.FIELDS ("); + } else { /* filter == -1 */ + headerlist += STRLEN("HEADER.FIELDS.NOT ("); + } + /* stpcpy would be useful here, to copy and compute length in one pass... */ + headerlistlen = strlen(headerlist); + if (headerlistlen >= sizeof(listbuf) - 2) { + bbs_error("List of desired headers is too long (%lu)\n", headerlistlen); + } + /* Well, now that we checked the length explicitly, strcpy is safe */ + strcpy(listbuf + 1, headerlist); /* Safe */ + /* Replace all the spaces with colons, so they match the format in the message */ + listbuf[0] = ':'; /* Also start with : so we can search for :$HEADERNAME: in the buffer */ + bbs_strreplace(listbuf + 1, ' ', ':'); + /* Also need to replace the trailing ) with : to match the last header. + * Rather than a manual loop on both ' ' and ')', it's more efficient to just replace the last one. */ + if (headerlistlen > 0) { + /* Normally, we'd subtract 1, but we copied into the buffer starting at position 1, so it cancels out */ + listbuf[headerlistlen] = ':'; + } + } /* Read the file until the first CR LF CR LF (end of headers) */ - fp = fopen(fullname, "r"); - if (!fp) { - bbs_error("Failed to open %s: %s\n", fullname, strerror(errno)); - return -1; + if (!*fp) { + *fp = fopen(fullname, "r"); + if (!*fp) { + bbs_error("Failed to open %s: %s\n", fullname, strerror(errno)); + return -1; + } + } else { + rewind(*fp); } + /* The RFC says no line should be more than 1,000 octets (bytes). * Most clients will wrap at 72 characters, but we shouldn't rely on this. */ - while ((fgets(linebuf, sizeof(linebuf), fp))) { + while ((fgets(linebuf, sizeof(linebuf), *fp))) { /* fgets does store the newline, so line should end in CR LF */ - if (!strcmp(linebuf, "\r\n")) { + if (!strcmp(linebuf, "\r\n") || !strcmp(linebuf, "\n")) { /* Some messages include only a LF at end of headers? */ break; /* End of headers */ } - /* I hope gcc optimizes this to not use snprintf under the hood */ - SAFE_FAST_COND_APPEND_NOSPACE(headers, headerslen, headpos, headlen, 1, "%s", linebuf); - } - fclose(fp); - *bodylen = (size_t) (headpos - headers); /* XXX cheaper than strlen, although if truncation happened, this may be wrong (too high). */ - if (!unoriginal) { - SAFE_FAST_COND_APPEND(response, responselen, *buf, *len, 1, "RFC822.HEADER"); - } - return 0; -} - -static int process_fetch_finalize(struct imap_session *imap, struct fetch_request *fetchreq, int seqno, const char *fullname, char *response, size_t responselen, char **buf, int *len) -{ - char headers[10000] = ""; /* XXX Large enough for all headers, etc.? Better might be sendfile, no buffering */ - char rangebuf[32] = ""; - int multiline = 0; - size_t bodylen = 0; - int sendbody = 0; - int skipheaders = 0; - int unoriginal = 0; /* BODY[HEADER] or BODY.PEEK[HEADER], rather than RFC822.HEADER, and the type has already been appended to the buffer. */ - FILE *fp = NULL; - char *dyn = NULL; - /* For BODY and BODY.PEEK: */ - int peek = fetchreq->bodypeek ? 1 : 0; /* NOT whether we are peeking the message, this is purely if it's BODY.PEEK vs. BODY */ - int body = fetchreq->bodyargs || fetchreq->bodypeek; /* One of BODY or BODY.PEEK ? */ - const char *bodyargs = peek ? fetchreq->bodypeek : fetchreq->bodyargs; - - /* BODY or BODY.PEEK (RFC 3501 6.4.5) - * format is BODY[section] - * section = 0+ part specifiers: - * - HEADER: all headers - * - HEADER.FIELDS: only the specified headers - * - HEADER.FIELDS.NOT: only those headers that don't match the provided list - * - MIME: MIME header for this part - * - TEXT: body (not including headers) - * - If empty, it's the entire message (including headers) - * partial = substring in a.b format (a = position of first octet, b = max # of octets to fetch) - * - Note: BODY[]<0.2048> of a 1500-octet message will be returned as BODY[]<0>, not BODY[] - * - Substrings should be supported for HEADER.FIELDS and HEADER.FIELDS.NOT too! - */ - - /* HEADER.FIELDS involves a multiline response, so this should be processed at the end of this loop since it appends to response. - * Otherwise, something else might concatenate itself on at the end and break the response. */ - if (body) { - /* Can be HEADER, HEADER.FIELDS, HEADER.FIELDS.NOT, MIME, TEXT */ - char linebuf[1001]; - char *headpos = headers; - int headlen = sizeof(headers); - - if (!strcmp(bodyargs, "HEADER")) { /* e.g. BODY.PEEK[HEADER] */ - /* Just treat it as if we got a HEADER request directly, to send all the headers. */ - unoriginal = 1; - SAFE_FAST_COND_APPEND(response, responselen, *buf, *len, 1, "%s", peek ? "BODY.PEEK[HEADER]" : "BODY[HEADER]"); - fetchreq->rfc822header = 1; - fetchreq->bodyargs = fetchreq->bodypeek = NULL; /* Don't execute the if statement below, so that we can execute the else if */ - } else if (STARTS_WITH(bodyargs, "HEADER.FIELDS") || STARTS_WITH(bodyargs, "HEADER.FIELDS.NOT")) { - /* e.g. BODY[HEADER.FIELDS (From To Cc Bcc Subject Date Message-ID Priority X-Priority References Newsgroups In-Reply-To Content-Type Reply-To Received)] */ - char *headerlist, *tmp; - int inverted = 0; - int in_match = 0; - multiline = 1; - if (STARTS_WITH(bodyargs, "HEADER.FIELDS.NOT")) { - inverted = 1; - bodyargs += STRLEN("HEADER.FIELDS.NOT ("); - } else { - bodyargs += STRLEN("HEADER.FIELDS ("); - } - /* Read the file until the first CR LF CR LF (end of headers) */ - fp = fopen(fullname, "r"); - if (!fp) { - bbs_error("Failed to open %s: %s\n", fullname, strerror(errno)); - return -1; - } - headerlist = malloc(strlen(bodyargs) + 2); /* Add 2, 1 for NUL and 1 for : at the beginning */ - if (ALLOC_FAILURE(headerlist)) { - fclose(fp); - return -1; - } - headerlist[0] = ':'; - strcpy(headerlist + 1, bodyargs); /* Safe */ - tmp = headerlist + 1; /* No need to check the first byte as it's already a :, so save a CPU cycle by skipping it */ - while (*tmp) { - if (*tmp == ' ' || *tmp == ')') { - *tmp = ':'; - } - tmp++; - } - /* The RFC says no line should be more than 1,000 octets (bytes). - * Most clients will wrap at 72 characters, but we shouldn't rely on this. */ + if (filter) { + /* I have seen headers as crazy long as this: + * X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp */ #define MAX_HEADER_NAME_LENGTH 72 - while ((fgets(linebuf, sizeof(linebuf), fp))) { - /* I have seen headers as crazy long as this: - * X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp */ - char headername[MAX_HEADER_NAME_LENGTH]; - /* fgets does store the newline, so line should end in CR LF */ - if (!strcmp(linebuf, "\r\n") || !strcmp(linebuf, "\n")) { /* Some messages include only a LF at end of headers? */ - break; /* End of headers */ - } - if (isspace(linebuf[0])) { /* It's part of a previous header (mutliline header) */ - SAFE_FAST_COND_APPEND_NOSPACE(headers, sizeof(headers), headpos, headlen, in_match, "%s", linebuf); /* Append if in match */ - continue; - } + char headername[MAX_HEADER_NAME_LENGTH]; + if (!isspace(linebuf[0])) { /* Continuation of multiline header */ + char *tmp; headername[0] = ':'; - safe_strncpy(headername + 1, linebuf, sizeof(headername) - 1); /* Don't copy the whole line. XXX This assumes that no header name is longer than MAX_HEADER_NAME_LENGTH chars. */ + safe_strncpy(headername + 1, linebuf, sizeof(headername) - 1); tmp = strchr(headername + 1, ':'); if (!tmp) { bbs_warning("Unexpected end of headers: %s\n", linebuf); break; } - /* Since safe_strncpy will always null terminate, it is always safe to null terminate the character after this */ + /* Since safe_strncpy will always null terminate, + * it is always safe to null terminate the character after this + * (if there wasn't room in the buffer, the character wouldn't be here) */ *(tmp + 1) = '\0'; /* Only include headers that were asked for. */ /* Note that some header names can be substrings of others, e.g. the "To" header should not match for "In-Reply-To" - * bodyargs contains a list of (space delimited) header names that we can match on, so we can't just use strncmp. - * Above, we transform the list into a : delimited list (every header has a : after it, including the last one), - * so NOW we can just use strstr for :NAME: + * headerlist contains a list of (space delimited) header names that we can match on, so we can't just use strncmp. + * Above, we transform the list into a : delimited list (every header has a : before/after it, including the last one), + * so NOW we can just use strstr for :$HEADERNAME: + */ + inside_match = (filter == 1 && strcasestr(listbuf, headername)) || (filter == -1 && !strcasestr(listbuf, headername)); + } /* else, append if in match (fall through... will only append if inside_match in macro) */ + } + /* I hope gcc optimizes this to not use snprintf under the hood */ + SAFE_FAST_COND_APPEND_NOSPACE(headersbuf, headersbuflen, buf, len, inside_match, "%s", linebuf); + } + SAFE_FAST_COND_APPEND_NOSPACE(headersbuf, headersbuflen, buf, len, 1, "\r\n"); /* Include CR LF after */ + headerslen = (size_t) (buf - headersbuf); + + /* Leading space, since this is not the first item in the response */ + sendstart = headersbuf; + sendsize = headerslen; + if (fbr && fbr->substart != -1) { + off_t offset = 0; + bbs_debug(3, "Orig input(%ld): %s\n", sendsize, sendstart); + adjust_send_offset(fbr, &offset, &sendsize); + sendstart += offset; + _imap_reply(imap, " %s<%ld> {%lu}\r\n%s", itemname, offset, sendsize, sendstart); + } else { + _imap_reply(imap, " %s {%lu}\r\n%s", itemname, sendsize, sendstart); + } + return 0; +} + +/* See RFC 3501 6.4.5. */ +enum partspec_suffix { + PARTSPEC_ENTIRE = 0, + PARTSPEC_MIME, + PARTSPEC_TEXT, + PARTSPEC_HEADER, + PARTSPEC_HEADER_FIELDS, + PARTSPEC_HEADER_FIELDS_NOT, +}; + +/*! \brief Similar to send_filtered_headers, but operating on a string rather than a file */ +static char *filter_headers_string(char *headers, const char *filter, size_t *restrict partlen, enum partspec_suffix suffix) +{ + char headersbuf[8192]; /* Hopefully, all headers fit in here */ + size_t headerslen = sizeof(headersbuf); + char *pos = headersbuf; + size_t left = headerslen; + char *header; + int in_match = 0; + + /* Skip to first header name */ + while (*filter && *filter != '(') { + filter++; + } + if (*filter == '(') { + filter++; + } + + /* Thankfully, this string is no longer needed afterwards, so we can modify it using strsep for easy parsing */ + while ((header = strsep(&headers, "\r\n"))) { + if (strlen_zero(header)) { /* Yes, this is necessary... for some reason, every other strsep returns an empty string */ + continue; + } + if (suffix == PARTSPEC_HEADER || suffix == PARTSPEC_MIME) { + in_match = 1; + } else if (isspace(header[0])) { + if (!in_match) { + continue; + } + } else { + char *colon = strchr(header, ':'); + if (!colon) { + /* Huh? Header name not followed by colon??? */ + continue; + } + /* Case-insensitively look for the entire word from header, up to but not including colon. + * So that we can use strstr, temporarily augment the colon. */ + *colon = '\0'; + + if (strcasestr(filter, header)) { + /* This doesn't necessarily mean there's match. There are several possible match types: + * '(FIRSTHEADER ' + * '(ONLYHEADER)' + * ' MIDDLEHEADER ' + * ' LASTHEADER)' + * + * So, we need to do some further checks to be sure. + * There is a more efficient way to do this here, but given if we've hit this path, + * it's almost always a match anyways, it's not really increasing the runtime (ignoring constants). */ - if ((!inverted && strcasestr(headerlist, headername)) || (inverted && !strcasestr(headerlist, headername))) { - /* I hope gcc optimizes this to not use snprintf under the hood */ - SAFE_FAST_COND_APPEND_NOSPACE(headers, sizeof(headers), headpos, headlen, 1, "%s", linebuf); - in_match = 1; + char filterdup[8192]; + char *fheader, *fheaders; + + bbs_strncpy_until(filterdup, S_IF(filter), sizeof(filterdup), ')'); /* Ignoring opening ( and closing ) */ + fheaders = filterdup; + /* Now it's space-delimited */ + while ((fheader = strsep(&fheaders, " "))) { + if (!strcasecmp(header, fheader)) { + break; + } + } + if (fheader) { + /* Yes, an exact match for the header was found */ + in_match = suffix == PARTSPEC_HEADER_FIELDS; } else { - in_match = 0; + in_match = suffix == PARTSPEC_HEADER_FIELDS_NOT; } + } else { + /* No, header was not found. */ + in_match = suffix == PARTSPEC_HEADER_FIELDS_NOT; + } + + *colon = ':'; /* Restore */ + } + /* If it's a match, include it */ + /* Readd CR LF since strsep removed it. But if it's a multiline continuation of a header, don't add CR LF */ + SAFE_FAST_COND_APPEND_NOSPACE(headersbuf, headerslen, pos, left, in_match, "%s%s", pos > headersbuf && !isspace(header[0]) ? "\r\n" : "", header); + } + SAFE_FAST_COND_APPEND_NOSPACE(headersbuf, headerslen, pos, left, pos > headersbuf, "\r\n"); /* Line ending for last header, if there was one */ + /* RFC 3501 6.4.5 + * Subsetting does not exclude the [RFC-2822] delimiting blank line between the header and the body; + * the blank line is included in all header fetches, except in the case of a message which has no body and no blank line. + * + * XXX We assume a body is always present, but technically this should be contingent upon having a body (if we knew that here) */ + SAFE_FAST_COND_APPEND_NOSPACE(headersbuf, headerslen, pos, left, 1, "\r\n"); + /* Now, should have the length */ + *partlen = (size_t) (pos - headersbuf); + return strndup(headersbuf, *partlen); +} + +/*! + * \brief Parse a part specifier, as defined in RFC 3501 6.4.5 + * \param[out] buf Just the part number prefix of the part specifier + * \param[in] len Size of buf + * \param[in] Entire part specifier to parse + */ +static enum partspec_suffix parse_part_spec(char *restrict buf, size_t len, const char *input) +{ + enum partspec_suffix suffix = PARTSPEC_ENTIRE; + char *tmp; + + safe_strncpy(buf, input, len); + + /* It's either going to be just a part number, + * or it's going to have .MIME, .HEADER, or .TEXT (or .HEADER.FIELDS or .HEADER.FIELDS.NOT) + * all of which (except .MIME) are only valid for message/rfc822 subparts. + * We want to pass just the part number for spec, if there is a modifier. + * + * Can't use strrchr, since HEADER.FIELDS and HEADER.FIELDS.NOT include commas. + * Can't do "ends with str" either, since HEADER.FIELDS and HEADER.FIELDS.NOT take arguments. + * Most reliable way is keep parsing the string until we hit something not numeric or a period. */ + + tmp = buf; + while (*tmp && (isdigit(*tmp) || *tmp == '.')) { + tmp++; + } + + if (*tmp && tmp > buf) { + /* We ran out of the numeric portion, terminate here to split the string */ + *(tmp - 1) = '\0'; + } + + if (!strlen_zero(tmp)) { + if (!strcasecmp(tmp, "MIME")) { + *tmp = '\0'; + suffix = PARTSPEC_MIME; + } else if (!strcasecmp(tmp, "TEXT")) { + *tmp = '\0'; + suffix = PARTSPEC_TEXT; + } else if (!strcasecmp(tmp, "HEADER")) { + *tmp = '\0'; + suffix = PARTSPEC_HEADER; + /* This must be first, since HEADER.FIELDS.NOT also starts with HEADER.FIELDS */ + } else if (STARTS_WITH(tmp, "HEADER.FIELDS.NOT")) { + *tmp = '\0'; + suffix = PARTSPEC_HEADER_FIELDS_NOT; + } else if (STARTS_WITH(tmp, "HEADER.FIELDS")) { + *tmp = '\0'; + suffix = PARTSPEC_HEADER_FIELDS; + } else if (!strcmp(tmp, ".")) { + bbs_warning("Malformed part specifier: %s\n", input); + } else { + if (!isdigit(*tmp)) { + bbs_warning("Unknown part specifier: %s\n", input); + } + } + } + + return suffix; +} + +/*! + * \brief Get offset into file at which body starts (after skipping headers) + * \note This function assumes fp is positioned at beginning of file + * \return offset at which the body starts + */ +static long int compute_body_offset(FILE *fp) +{ + char linebuf[1001]; + + while ((fgets(linebuf, sizeof(linebuf), fp))) { + /* fgets does store the newline, so line should end in CR LF */ + if (!strcmp(linebuf, "\r\n")) { + /* End of headers... we are now positioned at start of body */ + return ftell(fp); + } + } + return ftell(fp); /* Message has no body? */ +} + +#define BOTH 0 +#define HEADERS_ONLY 1 +#define BODY_ONLY 2 + +/*! \brief Send only the body (no headers) */ +static ssize_t send_message(struct imap_session *imap, struct fetch_body_request *fbr, const char *item, FILE **restrict fp, size_t *restrict size, const char *fullname, int sendmask) +{ + size_t sendsize; + off_t offset = 0; + + if (!*fp) { + *fp = fopen(fullname, "r"); + if (!*fp) { + bbs_error("Failed to open %s: %s\n", fullname, strerror(errno)); + return -1; + } + } + if (!*size) { + /* First time, compute size of message */ + fseek(*fp, 0L, SEEK_END); /* EOF */ + *size = (size_t) ftell(*fp); + rewind(*fp); /* Be kind, rewind */ + if (!*size) { + bbs_warning("File size of %s is %ld bytes?\n", fullname, *size); + } + bbs_debug(3, "File size of %s is %ld\n", fullname, *size); + } + + if (sendmask == BODY_ONLY) { + offset = compute_body_offset(*fp); + sendsize = *size - (size_t) offset; + } else if (sendmask == HEADERS_ONLY) { + offset = compute_body_offset(*fp); + sendsize = (size_t) offset; + } else { + sendsize = *size; + } + + if (fbr && fbr->substart != -1) { + adjust_send_offset(fbr, &offset, &sendsize); + /* Format is described in RFC 3501 7.4.2: BODY[
]<> + * We only include the starting octet, not the length. + * Client must assume truncation may have occured. */ + _imap_reply(imap, " %s<%ld> {%ld}\r\n", item, offset, sendsize); + } else { + _imap_reply(imap, " %s {%ld}\r\n", item, sendsize); + } + + /* We must manually tell it the offset or it will be at the EOF, even with rewind() */ + return bbs_sendfile(imap->node->wfd, fileno(*fp), &offset, sendsize); +} + +static int send_part(struct imap_session *imap, struct fetch_body_request *fbr, struct bbs_mime_message **restrict mime, const char *fullname) +{ + char *part, *partstart; + size_t partlen = 0; + char partnumber[128]; + enum partspec_suffix suffix; + enum mime_part_filter filter; + char *tmp; + + /* Part specification, which could be just a part number, but not necessarily, e.g. + * 1, 2, 2.1, 2.1.MIME are all valid. + * + * RFC 3501 6.4.5 sums it up concisely: + * + * The HEADER, HEADER.FIELDS, HEADER.FIELDS.NOT, and TEXT part + * specifiers can be the sole part specifier or can be prefixed by + * one or more numeric part specifiers, provided that the numeric + * part specifier refers to a part of type MESSAGE/RFC822. The + * MIME part specifier MUST be prefixed by one or more numeric + * part specifiers. + * + * In other words, keep the following 4 things in mind: + * - MIME only needs to be handled in this code block, not as a bodyarg by itself. + * - Any part spec can end in .MIME + * - Any message/rfc822 subpart can end in .HEADER, .TEXT (as shown in the RFC 3501 6.4.5 example), + * as well as .HEADER.FIELDS (header1 header2 ...), .HEADER.FIELDS.NOT (header 1 header2 ...). + * - A request for just HEADER, TEXT, HEADER.FIELDS, or HEADER.FIELDS.NOT is already handled + * as a bodyarg by itself, here we just need to handle these as a suffix of a part number. + */ + suffix = parse_part_spec(partnumber, sizeof(partnumber), fbr->bodyarg); + if (!*mime) { + *mime = bbs_mime_message_parse(fullname); + if (!*mime) { + return -1; + } + } + + /* We'll handle HEADER, HEADER.FIELDS, HEADER.FIELDS.NOT, and TEXT (all the message/rfc822 specific stuff) + * here in net_imap, since that's not really in the scope of what mod_mimeparse is for. + * However, MIME is something that mod_mimeparse is well-equipped to do for us already, + * and since it's so small compared to an entire part, it'd be kind of wasteful to allocate the whole thing + * when we don't need all of that. + * As such, if the part specifier is only for .MIME, then filter appropriately to get just that. */ + switch (suffix) { + case PARTSPEC_ENTIRE: + filter = MIME_PART_FILTER_ALL; + break; + case PARTSPEC_MIME: + filter = MIME_PART_FILTER_MIME; + break; + case PARTSPEC_HEADER: + case PARTSPEC_HEADER_FIELDS: + case PARTSPEC_HEADER_FIELDS_NOT: + /* These part specifiers are only legal if the subpart is of type message/rfc822 */ + filter = MIME_PART_FILTER_HEADERS; + break; + case PARTSPEC_TEXT: + /* This part specifier is only legal if the subpart is of type message/rfc822 */ + filter = MIME_PART_FILTER_TEXT; + break; + } + + part = bbs_mime_get_part(*mime, partnumber, &partlen, filter); + if (!part) { + _imap_reply(imap, " %s[%s] {0}\r\n", "BODY", fbr->bodyarg); + return 0; + } + + switch (suffix) { + case PARTSPEC_HEADER: + /* Remove the trailing line endings from HEADER response */ + tmp = part + partlen - 1; + while (tmp > part && (*tmp == '\r' || *tmp == '\n')) { + *tmp-- = '\0'; + partlen--; } - fclose(fp); - free(headerlist); - bodylen = strlen(headers); /* Can't just subtract end of headers, we'd have to keep track of bytes added on each round (which we probably should anyways) */ - - /* bodyargs ends in a ')', so don't tack an additional one on afterwards */ - /* Here, the condition will only be true for one or the other: */ - SAFE_FAST_COND_APPEND(response, responselen, *buf, *len, !inverted, "BODY[HEADER.FIELDS (%s]", bodyargs); - SAFE_FAST_COND_APPEND(response, responselen, *buf, *len, inverted, "BODY[HEADER.FIELDS.NOT (%s]", bodyargs); - } else if (!strcmp(bodyargs, "TEXT")) { /* Empty (e.g. BODY.PEEK[] or BODY[], or TEXT */ - multiline = 1; - sendbody = 1; - skipheaders = 1; - } else if (!strcmp(bodyargs, "MIME")) { - bbs_error("MIME is currently unsupported!\n"); /*! \todo Support it */ - } else if (!strcmp(bodyargs, "")) { /* Empty (e.g. BODY.PEEK[] or BODY[], or TEXT */ - multiline = 1; - sendbody = 1; - } else if (isdigit(*bodyargs)) { - int part_number = atoi(bodyargs); + /* Use filter_headers_string to properly combine multiline headers onto a single line for response. */ + /* Fall through. */ + case PARTSPEC_MIME: /* Force multiline header continuations onto the same line */ + case PARTSPEC_HEADER_FIELDS: + case PARTSPEC_HEADER_FIELDS_NOT: + /* Some post-processing is now required, since we need + * to eliminate any headers that don't match the filter in fbr->bodyarg + * While we do already have code to look for HEADER.FIELDS and HEADER.FIELDS.NOT + * in the entire message, that operates by scanning over a file, + * while here we need to scan over a string. So similar logic to send_filtered_headers, but separate function. */ + tmp = filter_headers_string(part, fbr->bodyarg, &partlen, suffix); + /* Swap it out */ + free(part); + part = tmp; + break; + case PARTSPEC_TEXT: + case PARTSPEC_ENTIRE: + /* No post-processing required */ + break; + } + + partstart = part; + if (fbr->substart != -1) { + off_t offset = 0; + adjust_send_offset(fbr, &offset, &partlen); + partstart += offset; + _imap_reply(imap, " %s[%s]<%ld> {%ld}\r\n", "BODY", fbr->bodyarg, offset, partlen); + } else { + _imap_reply(imap, " %s[%s] {%ld}\r\n", "BODY", fbr->bodyarg, partlen); + } + + if (!strlen_zero(partstart)) { + bbs_node_fd_write(imap->node, imap->node->wfd, partstart, partlen); + } + + free(part); + return 0; +} + +/*! + * \brief Finalize and send the FETCH response + * \note Most simple items are already prepared prior to calling this function. + * Anything to do with fetch_body_request or the body itself is handled here. + * \param imap + * \param fetchreq + * \param seqno Sequence number of the message + * \param fullname Full path to message file + * \param[in] response Argument 1 to SAFE_FAST_COND_APPEND. Actual entire buffer. + * \param[in] responselen Argument 2 to SAFE_FAST_COND_APPEND. Total size of buffer. + * \param[in] buf Argument 3 to SAFE_FAST_COND_APPEND. Current buffer head for next write. + * \param[in] len Argument 4 to SAFE_FAST_COND_APPEND. Amount of buffer remaining. + */ +static int process_fetch_finalize(struct imap_session *imap, struct fetch_request *fetchreq, int seqno, const char *fullname, char *response, size_t responselen, char **buf, int *len) +{ + struct fetch_body_request *fbr; + FILE *fp = NULL; + size_t size = 0; + int res = 0; + struct bbs_mime_message *mime = NULL; /* For BODYSTRUCTURE/BODY[]/BODY.PEEK[] operations that need the MIME message */ + + /* There are a few important things to keep in mind in this function: + * + * 1. response is a large (but not infinite) temporary buffer that can be used to build up most (but not all) + * of the FETCH response. + * It's explicitly NOT suitable for large, variable responses, like the body or any part of the body. + * + * 2. It is desirable, to the extent that is easily possible, to minimize the number of system calls + * that write to the connection, both to reduce system call overhead and also to reduce the total + * number of packets sent. + * Because it is often easier to write directly to the node (especially when impractical to use the fixed-size buffer), + * we employ TCP_CORK to temporarily "hold" the data in the kernel and unbuffer it all at once. + * TCP_CORK will keep data in the kernel until a full packet of data is present or we manually remove it. + * + * 3. The IMAP session must remain locked atomically throughout the entire function. + * We thus must take care to avoid functions that lock the session for us. + * + * 4. imap_send must be avoided, since it adds CR LF and logs the entire payload. + * We do sometimes use imap_send_nocrlf or _imap_reply, if we want to log it, but anytime we send a potentially long payload, + * this MUST NOT be logged (and thus, we use bbs_node_fd_write instead). + * + * 5. It isn't obvious from reading the RFC, but BODY.PEEK is only used in client commands. It is not used in server responses. + * We always reply with BODY[... even when the client sent BODY.PEEK[... + */ + +#define CLEANUP_IF_NULL(x) if (!x) { res = -1; goto cleanup; } + + bbs_mutex_lock(&imap->lock); + bbs_node_cork(imap->node, 1); /* Cork the TCP session */ + + /* Do as much as possible before we even start the reply, to minimize number of writes */ + if (fetchreq->rfc822header) { + send_headers(imap, NULL, "RFC822.HEADER", &fp, fullname); + } + if (fetchreq->body || fetchreq->bodystructure) { + char *bodystructure; + /* BODY is BODYSTRUCTURE without extensions (which we don't send anyways, in either case) */ + /* Excellent reference for BODYSTRUCTURE: http://sgerwk.altervista.org/imapbodystructure.html */ + if (!mime) { + mime = bbs_mime_message_parse(fullname); + CLEANUP_IF_NULL(mime); + } + bodystructure = bbs_mime_make_bodystructure(mime); + if (!strlen_zero(bodystructure)) { + SAFE_FAST_COND_APPEND(response, responselen, *buf, *len, 1, "%s %s", fetchreq->bodystructure ? "BODYSTRUCTURE" : "BODY", bodystructure); + free(bodystructure); + } + } + + imap_send_nocrlf(imap, /* Start the IMAP response, and ensure no CR LF is automatically added. */ + "* %d " /* Number after FETCH is always a message sequence number, not UID, even if usinguid */ + "FETCH (%s", seqno, response); /* No closing paren, we do that at the very end */ + + /* Now, send anything that could be a large, variable amount of data. + * All of these will use a format literal / multiline response */ + if (fetchreq->rfc822) { /* Same as BODY[] */ + send_message(imap, NULL, "RFC822", &fp, &size, fullname, BOTH); + } + if (fetchreq->rfc822header) { + send_message(imap, NULL, "RFC822.HEADER", &fp, &size, fullname, HEADERS_ONLY); + } + if (fetchreq->rfc822text) { /* Same as BODY[TEXT] */ + send_message(imap, NULL, "RFC822.TEXT", &fp, &size, fullname, BODY_ONLY); + } + RWLIST_TRAVERSE(&fetchreq->bodyfetches, fbr, entry) { + if (strlen_zero(fbr->bodyarg)) { /* Empty (e.g. BODY.PEEK[] or BODY[] */ + send_message(imap, fbr, "BODY[]", &fp, &size, fullname, BOTH); + } else if (!strcasecmp(fbr->bodyarg, "TEXT")) { + send_message(imap, fbr, "BODY[TEXT]", &fp, &size, fullname, BODY_ONLY); + } else if (isdigit(*fbr->bodyarg)) { + int part_number = atoi(fbr->bodyarg); if (part_number == 0) { /* BODY[0] (or BODY.PEEK[0]) = ([RFC-822] header of the message) MULTIPART/MIXED * Thunderbird-based clients may fall back to this if they are unsatisfied with @@ -417,137 +815,51 @@ static int process_fetch_finalize(struct imap_session *imap, struct fetch_reques * and since we just do the same thing here, it's unlikely to like that either. * This phenomenon is thus probably symptomatic of a bug somewhere... * for that reason, log a warning here for now. + * + * Furthermore, most IMAP servers don't seem to support this with IMAP4rev1 + * (after all, it isn't in the IMAPrev1 spec). Most of them treat it as a bad command. */ - - /* RFC822 header */ bbs_warning("Requesting headers using part number 0 was obsoleted in IMAP4rev1\n"); - unoriginal = 1; - SAFE_FAST_COND_APPEND(response, responselen, *buf, *len, 1, "%s", peek ? "BODY.PEEK[0]" : "BODY[0]"); - fetchreq->rfc822header = 1; - fetchreq->bodyargs = fetchreq->bodypeek = NULL; /* Don't execute the if statement below, so that we can execute the else if */ + send_message(imap, fbr, "BODY[0]", &fp, &size, fullname, HEADERS_ONLY); } else { - /*! \todo BUGBUG Needs to be supported!!! */ - /* Also note that part numbers are not necessarily integers, - * they could be X.Y.Z (e.g. 1.2.3) */ - bbs_warning("Part numbers not currently supported\n"); + if (send_part(imap, fbr, &mime, fullname)) { + res = -1; + goto cleanup; + } } + /* If sending all headers, we don't need to combine multiline headers onto a single line, + * pure passthrough works correctly. + * Surprisingly, this seems to be true for HEADER.FIELDS and HEADER.FIELDS.NOT too (at this level). + * For subparts (within send_part), it's different. */ + } else if (!strcasecmp(fbr->bodyarg, "HEADER")) { /* e.g. BODY.PEEK[HEADER] */ + send_headers(imap, fbr, "BODY[HEADER]", &fp, fullname); + } else if (STARTS_WITH(fbr->bodyarg, "HEADER.FIELDS")) { + char itemname[1024]; + snprintf(itemname, sizeof(itemname), "%s[%s]", "BODY", fbr->bodyarg); + send_filtered_headers(imap, fbr, itemname, &fp, fullname, fbr->bodyarg, 1); + } else if (STARTS_WITH(fbr->bodyarg, "HEADER.FIELDS.NOT")) { + char itemname[1024]; + snprintf(itemname, sizeof(itemname), "%s[%s]", "BODY", fbr->bodyarg); + send_filtered_headers(imap, fbr, itemname, &fp, fullname, fbr->bodyarg, -1); } else { - bbs_warning("Invalid BODY argument: %s\n", bodyargs); - } - } - if (fetchreq->rfc822header) { /* not a else if, because it could have just been set true. */ - multiline = 1; - if (process_fetch_rfc822header(fullname, response, responselen, buf, len, headers, sizeof(headers), unoriginal, &bodylen)) { - return -1; + bbs_warning("Unknown FETCH BODY item: %s\n", fbr->bodyarg); } } - /* Actual body, if being sent, should be last */ - if (fetchreq->rfc822 || fetchreq->rfc822text) { - multiline = 1; - sendbody = 1; + if (bbs_node_fd_writef(imap->node, imap->node->wfd, ")\r\n") < 0) { /* And the finale (don't use imap_send for this either) */ + res = -1; } - if (fetchreq->body || fetchreq->bodystructure) { - /* BODY is BODYSTRUCTURE without extensions (which we don't send anyways, in either case) */ - /* Excellent reference for BODYSTRUCTURE: http://sgerwk.altervista.org/imapbodystructure.html */ - /* But we just use the top of the line gmime library for this task (see https://stackoverflow.com/a/18813164) */ - dyn = mime_make_bodystructure(fetchreq->bodystructure ? "BODYSTRUCTURE" : "BODY", fullname); - } - - if (multiline) { - /* {D} tells client this is a multiline response, with D more bytes remaining */ - long size, fullsize; - skipheaders |= (fetchreq->rfc822text && !fetchreq->rfc822); /* Other conditions in which we skip sending headers */ - if (sendbody) { - ssize_t res; - char resptype[48]; - off_t offset; - fp = fopen(fullname, "r"); - if (!fp) { - bbs_error("Failed to open %s: %s\n", fullname, strerror(errno)); - return -1; - } - fseek(fp, 0L, SEEK_END); /* Go to EOF */ - fullsize = size = ftell(fp); - rewind(fp); /* Be kind, rewind */ - if (!size) { - bbs_warning("File size of %s is %ld bytes?\n", fullname, size); - } - - /* XXX Assumes not sending headers and bodylen at same time. - * In reality, I think that *might* be fine because the body contains everything, - * and you wouldn't request just the headers and then the whole body in the same FETCH. */ - if (bodylen) { - bbs_error("Can't send body and headers simultaneously!\n"); - } - offset = 0; - if (skipheaders) { /* Only body. No headers. */ - char linebuf[1001]; - /* XXX Refactor so we can just get the offset to body start via function call */ - while ((fgets(linebuf, sizeof(linebuf), fp))) { - /* fgets does store the newline, so line should end in CR LF */ - offset += (off_t) strlen(linebuf); /* strlen includes CR LF already */ - if (!strcmp(linebuf, "\r\n")) { - break; /* End of headers */ - } - } - size -= offset; - } - - if (fetchreq->sublength) { - int realskip = 0; - if (fetchreq->substart) { - /* size is currently how many bytes are left. - * So for this offset to work, - * fetchreq->substart must be at most size, and we - * reduce size by fetchreq->substart bytes. */ - realskip = MIN(fetchreq->substart, (int) size); - offset += realskip; - size -= realskip; - } - size = MIN(fetchreq->sublength, size); /* Can be at most size. */ - /* Format is described in RFC 3501 7.4.2: BODY[
]<> - * We only include the starting octet, not the length. - * Client must assume truncation may have occured. */ - snprintf(rangebuf, sizeof(rangebuf), "<%d>", realskip); - } - - /* If request used RFC822, use that. If it used BODY, use BODY */ - snprintf(resptype, sizeof(resptype), "%s%s", fetchreq->rfc822 ? "RFC822" : fetchreq->rfc822text ? "RFC822.TEXT" : skipheaders ? "BODY[TEXT]" : "BODY[]", rangebuf); - - imap_send(imap, "%d FETCH (%s%s%s %s {%ld}", seqno, S_IF(dyn), dyn ? " " : "", response, resptype, size); /* No close paren here, last write will do that */ - - bbs_mutex_lock(&imap->lock); - res = bbs_sendfile(imap->node->wfd, fileno(fp), &offset, (size_t) size); /* We must manually tell it the offset or it will be at the EOF, even with rewind() */ - bbs_node_fd_writef(imap->node, imap->node->wfd, ")\r\n"); /* And the finale (don't use imap_send for this) */ - bbs_mutex_unlock(&imap->lock); - - fclose(fp); - if (res == (ssize_t) size) { - imap_debug(5, "Sent %ld/%ld-byte body for %s\n", res, fullsize, fullname); /* either partial or entire body */ - } - } else { - const char *headersptr = headers; - if (fetchreq->sublength) { - int realskip = 0; - if (fetchreq->substart) { - realskip = MIN(fetchreq->substart, (int) bodylen); - headersptr += realskip; - bodylen -= (size_t) realskip; - } - bodylen = MIN((size_t) fetchreq->sublength, bodylen); - snprintf(rangebuf, sizeof(rangebuf), "<%d>", realskip); - } - imap_send(imap, "%d FETCH (%s%s%s%s {%lu}\r\n%s)", seqno, S_IF(dyn), dyn ? " " : "", response, rangebuf, bodylen, headersptr); - } - } else { - /* Number after FETCH is always a message sequence number, not UID, even if usinguid */ - imap_send(imap, "%d FETCH (%s%s%s)", seqno, S_IF(dyn), dyn ? " " : "", response); /* Single line response */ +cleanup: + bbs_node_cork(imap->node, 0); /* Uncork the node, to ensure data is fully written */ + if (mime) { + bbs_mime_message_destroy(mime); } - - free_if(dyn); - return 0; + if (fp) { + fclose(fp); + } + bbs_mutex_unlock(&imap->lock); + return res; } /*! \brief Get beginning of keyword letters in a filename, if present */ @@ -628,7 +940,7 @@ static int process_fetch(struct imap_session *imap, int usinguid, struct fetch_r } while (fno < files && (entry = entries[fno++])) { - char response[1024]; + char response[8192]; char *buf = response; int len = sizeof(response); unsigned int msguid; @@ -665,7 +977,7 @@ static int process_fetch(struct imap_session *imap, int usinguid, struct fetch_r * The maildir_msg_setflags API doesn't currently provide us back with the new renamed filename. * So what we do is check if we need to mark as seen, but not actually mark as seen until the END of the loop. * Consequently, we have to append the seen flag to the flags response manually if needed. */ - markseen = (fetchreq->bodyargs && !fetchreq->bodypeek) || fetchreq->rfc822text; + markseen = fetchreq->nopeek || fetchreq->rfc822text; /* We don't store the \Recent flag anywhere, it's a computed flag. * \Recent corresponds to messages that were in the new directory (as opposed to cur) @@ -725,7 +1037,7 @@ static int process_fetch(struct imap_session *imap, int usinguid, struct fetch_r return 0; } -static int parse_body_tail(struct fetch_request *fetchreq, char *s) +static int parse_body_tail(struct fetch_body_request *fbr, char *s) { char *tmp; @@ -744,16 +1056,21 @@ static int parse_body_tail(struct fetch_request *fetchreq, char *s) } b = tmp; a = strsep(&b, "."); - if (strlen_zero(a) || strlen_zero(b)) { + if (strlen_zero(a)) { return -1; } - fetchreq->substart = atoi(a); /* Can be 0 (to start from the beginning) */ - if (fetchreq->substart < 0) { + fbr->substart = atoi(a); /* Can be 0 (to start from the beginning) */ + if (fbr->substart < 0) { return -1; } - fetchreq->sublength = atol(b); /* cannot be 0 (or negative) */ - if (fetchreq->sublength <= 0) { - return -1; + if (!strlen_zero(b)) { + /* This is the maximum number of octets desired */ + fbr->sublength = atol(b); /* cannot be 0 (or negative) */ + if (fbr->sublength <= 0) { + return -1; + } + } else { + fbr->sublength = -1; } } return 0; @@ -807,6 +1124,8 @@ int handle_fetch_full(struct imap_session *imap, char *s, int usinguid, int tagg char *sequences; char *items, *item; struct fetch_request fetchreq; + struct fetch_body_request *fbr; + int res = 0; REQUIRE_ARGS(s); sequences = strsep(&s, " "); /* Messages, specified by sequence number or by UID (if usinguid) */ @@ -849,20 +1168,20 @@ int handle_fetch_full(struct imap_session *imap, char *s, int usinguid, int tagg } else { bbs_warning("Unexpected FETCH modifier: %s\n", s); imap_reply(imap, "BAD FETCH failed. Illegal arguments."); - return 0; + goto cleanup; } } /* RFC 7162 3.2.6 */ if (fetchreq.vanished) { if (!usinguid) { imap_reply(imap, "BAD Must use UID FETCH, not FETCH"); - return 0; + goto cleanup; } else if (!imap->qresync) { imap_reply(imap, "BAD Must enabled QRESYNC first"); - return 0; + goto cleanup; } else if (!fetchreq.changedsince) { imap_reply(imap, "BAD Must use in conjunction with CHANGEDSINCE"); - return 0; + goto cleanup; } } @@ -876,19 +1195,28 @@ int handle_fetch_full(struct imap_session *imap, char *s, int usinguid, int tagg fetchreq.body = 1; } else if (!strcmp(item, "BODYSTRUCTURE")) { fetchreq.bodystructure = 1; - } else if (STARTS_WITH(item, "BODY[")) { + } else if (STARTS_WITH(item, "BODY[") || STARTS_WITH(item, "BODY.PEEK[")) { /* Leave just the contents inside the [] */ - tmp = item + STRLEN("BODY["); - if (parse_body_tail(&fetchreq, tmp)) { - return -1; + fbr = calloc(1, sizeof(*fbr)); + if (ALLOC_FAILURE(fbr)) { + res = -1; + goto cleanup; } - fetchreq.bodyargs = tmp; /* Make assignment after, since this is a const char */ - } else if (STARTS_WITH(item, "BODY.PEEK[")) { - tmp = item + STRLEN("BODY.PEEK["); - if (parse_body_tail(&fetchreq, tmp)) { - return -1; + if (STARTS_WITH(item, "BODY.PEEK[")) { + fbr->peek = 1; + } else { + fetchreq.nopeek = 1; } - fetchreq.bodypeek = tmp; + tmp = item + (fbr->peek ? STRLEN("BODY.PEEK[") : STRLEN("BODY[")); + fbr->substart = -1; /* Initialize to -1 */ + if (parse_body_tail(fbr, tmp)) { + bbs_warning("Failed to parse partial fetch directive: %s\n", tmp); + res = -1; + free(fbr); + goto cleanup; + } + fbr->bodyarg = tmp; /* Make assignment after, since this is a const char */ + RWLIST_INSERT_TAIL(&fetchreq.bodyfetches, fbr, entry); } else if (!strcmp(item, "ENVELOPE")) { fetchreq.envelope = 1; } else if (!strcmp(item, "FLAGS")) { @@ -932,10 +1260,16 @@ int handle_fetch_full(struct imap_session *imap, char *s, int usinguid, int tagg } else { bbs_warning("Unsupported FETCH item: %s\n", item); imap_reply(imap, "BAD FETCH failed. Illegal arguments."); - return 0; + goto cleanup; } } /* Process the request, for each message that matches sequence number. */ - return process_fetch(imap, usinguid, &fetchreq, sequences, tagged); + res = process_fetch(imap, usinguid, &fetchreq, sequences, tagged); + +cleanup: + while ((fbr = RWLIST_REMOVE_HEAD(&fetchreq.bodyfetches, entry))) { + free(fbr); + } + return res; } diff --git a/nets/net_imap/imap_server_fetch.h b/nets/net_imap/imap_server_fetch.h index 4193dfc..872c0f0 100644 --- a/nets/net_imap/imap_server_fetch.h +++ b/nets/net_imap/imap_server_fetch.h @@ -13,11 +13,18 @@ * */ +/* There can be multiple of these in a request */ +struct fetch_body_request { + const char *bodyarg; /*!< BODY arguments */ + int substart; /*!< For BODY and BODY.PEEK partial fetch, the beginning octet */ + long sublength; /*!< For BODY and BODY.PEEK partial fetch, number of bytes to fetch */ + unsigned int peek:1; /*!< Whether this is a BODY.PEEK */ + RWLIST_ENTRY(fetch_body_request) entry; +}; + +RWLIST_HEAD(body_fetches, fetch_body_request); + struct fetch_request { - const char *bodyargs; /*!< BODY arguments */ - const char *bodypeek; /*!< BODY.PEEK arguments */ - int substart; /*!< For BODY and BODY.PEEK partial fetch, the beginning octet */ - long sublength; /*!< For BODY and BODY.PEEK partial fetch, number of bytes to fetch */ const char *flags; unsigned long changedsince; unsigned int envelope:1; @@ -31,6 +38,8 @@ struct fetch_request { unsigned int uid:1; unsigned int modseq:1; unsigned int vanished:1; + unsigned int nopeek:1; /*!< An operation was requested that will implicitly mark seen */ + struct body_fetches bodyfetches; }; /*! \brief strsep-like FETCH items tokenizer */ diff --git a/tests/alternative.eml b/tests/alternative.eml new file mode 100755 index 0000000..3fa687d --- /dev/null +++ b/tests/alternative.eml @@ -0,0 +1,55 @@ +Return-Path: user@example.com +Received: from [HIDDEN] (Authenticated sender: user@example.com) + by example.com with ESMTP + for ; Sun, Nov 10 2024 19:58:16 -0500 +To: user@example.com +From: User +Subject: Alternative test +Message-ID: <5e539fce-2d97-a7fc-ec3d-c13b63325dc7@example.com> +Date: Sun, 10 Nov 2024 19:58:17 -0500 +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="------------B56ACAA1D42503EF82F52166" +Content-Language: en-US + +This is a multi-part message in MIME format. +--------------B56ACAA1D42503EF82F52166 +Content-Type: multipart/alternative; + boundary="------------4E58F4DE7C0534A99765CD4D" + + +--------------4E58F4DE7C0534A99765CD4D +Content-Type: text/plain; charset=utf-8; format=flowed +Content-Transfer-Encoding: 7bit + +This is a short message, whose /purpose/ is to have both a plain text, +and an *HTML* component. + + +--------------4E58F4DE7C0534A99765CD4D +Content-Type: text/html; charset=utf-8 +Content-Transfer-Encoding: 7bit + + + + + + + +

This is a short message, whose purpose + is to have both a plain text, and an HTML component.
+

+ + + +--------------4E58F4DE7C0534A99765CD4D-- + +--------------B56ACAA1D42503EF82F52166 +Content-Type: text/plain; charset=UTF-8; + name="attachment.txt" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="attachment.txt" + +VGVzdCB0ZXh0IGZpbGUgYXR0YWNobWVudA== +--------------B56ACAA1D42503EF82F52166-- diff --git a/tests/multipart.eml b/tests/multipart.eml new file mode 100755 index 0000000..a541837 --- /dev/null +++ b/tests/multipart.eml @@ -0,0 +1,54 @@ +Return-Path: <> +Date: Sat, 9 Nov 2024 19:47:01 +0000 (UTC) +From: postmaster@example.com +To: postmaster@exmaple.com +Message-ID: <227366741.17305.1731181621979@example.com> +Subject: Test +MIME-Version: 1.0 +Content-Type: multipart/report; + boundary="----=_Part_17304_1769721302.1731181621976"; + report-type=delivery-status +Auto-Submitted: auto-replied +X-Auto-Response-Suppress: All + +------=_Part_17304_1769721302.1731181621976 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +We could not deliver the attached mail for the following recipients. (550) + +------=_Part_17304_1769721302.1731181621976 +Content-Type: message/delivery-status; name=status.dat +Content-Transfer-Encoding: 7bit +Content-Description: Delivery Status Notification +Content-Disposition: attachment; filename=status.dat + +Reporting-MTA: dns;example.com +Arrival-Date: Sat, 09 Nov 2024 19:47:01 +0000 + +Final-Recipient: postmaster@example.com +Action: failed +Status: 550 +Diagnostic-Code: error; 550 No such person at this address. + +Remote-MTA: dns; example.com. (192.0.2.25) +Last-Attempt-Date: Sat, 09 Nov 2024 19:47:01 +0000 + +------=_Part_17304_1769721302.1731181621976 +Content-Type: text/rfc822-headers +Content-Transfer-Encoding: 7bit + +Received: by example.com (SMTP) with ESMTPSA id 1234 + for + (version=TLSv1.3 cipher=TLS_AES_256_GCM_SHA384); + Sat, 09 Nov 2024 19:46:57 +0000 (UTC) +To: postmaster@example.com +From: Postmaster +Subject: Test +Message-ID: <854bff2f-0f60-fead-c9c5-5d22cd7b6dd6@example.com> +Date: Sat, 9 Nov 2024 14:46:55 -0500 +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8; format=flowed +Content-Transfer-Encoding: 7bit +Content-Language: en-US +------=_Part_17304_1769721302.1731181621976-- diff --git a/tests/multipart2.eml b/tests/multipart2.eml new file mode 100755 index 0000000..87417d9 --- /dev/null +++ b/tests/multipart2.eml @@ -0,0 +1,67 @@ +Return-Path: <> +Received: from localhost (localhost [127.0.0.1]) +Date: Fri, 08 Nov 2024 14:34:06 +0000 +From: "Mail Delivery Subsystem" +Subject: Delivery Status Notification (Failure) +To: +Auto-Submitted: auto-replied +Message-ID: +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="----attachment_774801610416369" + +This is a multi-part message in MIME format. + +------attachment_774801610416369 +Content-Description: Notification +Content-Type: text/plain; charset=utf-8 + +This is the mail system at host example.com. + +I'm sorry to have to inform you that your message could not +be delivered to one or more recipients. It's attached below. + +For further assistance, please send mail to postmaster. + +If you do so, please include this problem report. You can delete your own text from the attached returned message. + +Please, do not reply to this message. + + +: + host example.com[192.0.2.25] said: + 550 No such user + +------attachment_774801610416369 +Content-Description: Delivery report +Content-Type: message/delivery-status + +Reporting-MTA: dns; example.com +Arrival-Date: Fri, 08 Nov 2024 14:34:06 +0000 + +Final-Recipient: rfc822; +Action: failed +Remote-MTA: dns; example.net +Diagnostic-Code: x-unknown; 550 [S10] Blocked + +------attachment_774801610416369 +Content-Description: Undelivered message +Content-Type: message/rfc822 + +Received: from [10.1.1.1] + by example.com with ESMTPSA + for ; Fri, Nov 8 2024 14:34:06 +0000 +To: user@example.net +From: +Subject: Test +Message-ID: +Date: Fri, 8 Nov 2024 09:34:05 -0500 +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8; format=flowed +Content-Transfer-Encoding: 7bit +Content-Language: en-US + +Test + + +------attachment_774801610416369-- diff --git a/tests/test_imap.c b/tests/test_imap.c index a03d866..56f3882 100644 --- a/tests/test_imap.c +++ b/tests/test_imap.c @@ -52,7 +52,7 @@ static int send_message(int client1, size_t extrabytes) char subject[32]; if (!send_count++) { - CLIENT_EXPECT(client1, "220"); + CLIENT_EXPECT_EVENTUALLY(client1, "220 "); SWRITE(client1, "EHLO " TEST_EXTERNAL_DOMAIN ENDL); CLIENT_EXPECT_EVENTUALLY(client1, "250 "); /* "250 " since there may be multiple "250-" responses preceding it */ } else { @@ -605,7 +605,7 @@ static int run(void) if (smtpfd < 0) { goto cleanup; } - CLIENT_EXPECT(smtpfd, "220"); + CLIENT_EXPECT_EVENTUALLY(smtpfd, "220 "); SWRITE(smtpfd, "EHLO myclient" ENDL); CLIENT_EXPECT_EVENTUALLY(smtpfd, "250 "); /* "250 " since there may be multiple "250-" responses preceding it */ SWRITE(smtpfd, "AUTH PLAIN\r\n"); diff --git a/tests/test_imap_fetch.c b/tests/test_imap_fetch.c new file mode 100755 index 0000000..7ecb6a4 --- /dev/null +++ b/tests/test_imap_fetch.c @@ -0,0 +1,375 @@ +/* + * LBBS -- The Lightweight Bulletin Board System + * + * Copyright (C) 2024, Naveen Albert + * + * Naveen Albert + * + * This program is free software, distributed under the terms of + * the GNU General Public License Version 2. See the LICENSE file + * at the top of the source tree. + */ + +/*! \file + * + * \brief IMAP FETCH Tests + * + * \author Naveen Albert + * + * \note There are a handful of basic FETCH tests in test_imap, + * this test module specifically focuses on more extensively + * testing the various functionality in the FETCH command + */ + +#include "test.h" + +#include +#include +#include +#include +#include +#include + +#ifdef __linux__ +#include +#elif defined(__FreeBSD__) +#include +#include +#include +#else +#error "sendfile API unavailable" +#endif + +static int pre(void) +{ + test_preload_module("mod_mail.so"); + test_preload_module("mod_mimeparse.so"); + test_preload_module("net_smtp.so"); + test_load_module("mod_smtp_delivery_local.so"); + test_load_module("net_imap.so"); + test_load_module("mod_mail_events.so"); + + TEST_ADD_CONFIG("mod_mail.conf"); + TEST_ADD_CONFIG("net_smtp.conf"); + TEST_ADD_CONFIG("net_imap.conf"); + TEST_ADD_CONFIG("mod_mail_events.conf"); + + system("rm -rf " TEST_MAIL_DIR); /* Purge the contents of the directory, if it existed. */ + mkdir(TEST_MAIL_DIR, 0700); /* Make directory if it doesn't exist already (of course it won't due to the previous step) */ + return 0; +} + +static int write_file_to_socket(int client, const char *filename) +{ + int fd; + off_t offset, fsize; + + fd = open(filename, O_RDONLY); + if (fd < 0) { + bbs_error("Failed to open %s: %s\n", filename, strerror(errno)); + return -1; + } + fsize = lseek(fd, 0, SEEK_END); + offset = 0; + bbs_debug(3, "Sending %s (%ld bytes)\n", filename, fsize); + /* Assuming the file is small enough, it should succeed in one go, + * and bbs_sendfile wrapper isn't needed */ +#ifdef __linux__ + if (sendfile(client, fd, &offset, (size_t) fsize) != fsize) { +#elif defined(__FreeBSD__) + if (sendfile(client, fd, &offset, (size_t) fsize, NULL, NULL, 0) != fsize) { +#else +#error "Missing sendfile" +#endif + bbs_error("Failed to write file: %s\n", strerror(errno)); + close(fd); + return -1; + } + close(fd); + return 0; +} + +static int send_count = 0; + +static int send_message(int client1, const char *filename) +{ + if (!send_count++) { + CLIENT_EXPECT_EVENTUALLY(client1, "220 "); + SWRITE(client1, "EHLO " TEST_EXTERNAL_DOMAIN ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "250 "); /* "250 " since there may be multiple "250-" responses preceding it */ + } else { + SWRITE(client1, "RSET" ENDL); + CLIENT_EXPECT(client1, "250"); + } + + SWRITE(client1, "MAIL FROM:<" TEST_EMAIL_EXTERNAL ">\r\n"); + CLIENT_EXPECT(client1, "250"); + SWRITE(client1, "RCPT TO:<" TEST_EMAIL ">\r\n"); + CLIENT_EXPECT(client1, "250"); + SWRITE(client1, "DATA\r\n"); + CLIENT_EXPECT(client1, "354"); + + /* Note: Make sure these files end in CR LF. + * Otherwise it might get added when uploading to another IMAP server as a "fix-up" + * and that will throw some of the sizes off by 2. */ + if (write_file_to_socket(client1, filename)) { + goto cleanup; + } + +/* +0700: 2d2d 3d5f 5061 7274 5f31 3733 3034 5f31 3736 3937 3231 3330 322e 3137 3331 3138 --=_Part_17304_1769721302.173118 +0720: 3136 3231 3937 362d 2d 1621976-- + +0000: 2e0d 0a ... + +0000: 5253 4554 0d0a RSET.. + +0000: 5253 4554 0d0a RSET.. + +0700: 2d2d 3d5f 5061 7274 5f31 3733 3034 5f31 3736 3937 3231 3330 322e 3137 3331 3138 --=_Part_17304_1769721302.173118 +0720: 3136 3231 3937 362d 2d 1621976-- + +0000: 0d0a 2e0d 0a ..... + +0000: 3235 3020 322e 362e 3020 4d65 7373 6167 6520 6163 6365 7074 6564 2066 6f72 2064 250 2.6.0 Message accepted for d +0020: 656c 6976 6572 790d 0a elivery.. +*/ + + /* Messages end in CR LF, so only send . CR LF here */ + SWRITE(client1, "." ENDL); /* EOM */ + CLIENT_EXPECT(client1, "250"); + return 0; + +cleanup: + return -1; +} + +static int make_messages(void) +{ + int clientfd; + int res = 0; + + clientfd = test_make_socket(25); + if (clientfd < 0) { + return -1; + } + + res |= send_message(clientfd, "multipart.eml"); + res |= send_message(clientfd, "multipart2.eml"); + res |= send_message(clientfd, "alternative.eml"); + + close(clientfd); /* Close SMTP connection */ + return res; +} + +static int run(void) +{ + int client1 = -1; + int res = -1; + + if (make_messages()) { + return -1; + } + /* Verify that the email messages were all sent properly. */ + DIRECTORY_EXPECT_FILE_COUNT(TEST_MAIL_DIR "/1/new", send_count); + + client1 = test_make_socket(143); + if (client1 < 0) { + return -1; + } + + /* Connect and log in */ + CLIENT_EXPECT(client1, "OK"); + SWRITE(client1, "a1 LOGIN \"" TEST_USER "\" \"" TEST_PASS "\"" ENDL); + CLIENT_EXPECT(client1, "a1 OK"); + + SWRITE(client1, "a2 SELECT INBOX" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "a2 OK"); + + SWRITE(client1, "b1 FETCH 1 (BODYSTRUCTURE)" ENDL); + /* Actual BODYSTRUCTURE format may vary be server since they all do that differently... but certain things should ALWAYS appear verbatim */ + CLIENT_EXPECT_EVENTUALLY(client1, "_Part_17304_1769721302"); /* Hard to test with all the quotes, but just make sure it works at all */ + + /* Length is an easy way to validate that the output is as expected. + * The lengths are taken from running the same tests against another IMAP server that should be compliant. + * Since net_imap prefixes all FETCH responses with the UID, we make sure to include that as well. + * We also test that when using BODY.PEEK, the message is never marked as soon, + * and that BODY.PEEK is not used in the response (should be BODY). + * Since the CR LF sequence might be written later, particularly for larger payloads, + * we don't include that in our EXPECT (though it should be there). */ + + SWRITE(client1, "b2 FETCH 1 (BODY.PEEK[])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 BODY[] {1835}"); + + SWRITE(client1, "b3 FETCH 1 (BODY.PEEK[1])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 BODY[1] {76}"); + + SWRITE(client1, "b4 FETCH 1 (BODY.PEEK[2])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 BODY[2] {310}"); + + SWRITE(client1, "b5 FETCH 1 (BODY.PEEK[3])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 BODY[3] {522}"); + + SWRITE(client1, "b6 FETCH 1 (BODY.PEEK[1.MIME])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 BODY[1.MIME] {79}"); + + SWRITE(client1, "b7 FETCH 1 (BODY.PEEK[2.MIME])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 BODY[2.MIME] {196}" ENDL); + + SWRITE(client1, "b8 FETCH 1 (BODY.PEEK[3.MIME])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 BODY[3.MIME] {70}"); + + /* Request multiple things at once */ + SWRITE(client1, "c1 FETCH 1 (UID FLAGS INTERNALDATE RFC822.SIZE BODYSTRUCTURE BODY.PEEK[HEADER.FIELDS (Date Subject From To X-Priority Importance X-MSMail-Priority Priority)])" ENDL); /* actual FETCH command used by mod_webmail */ + + /* Since we were only peeking, message should not have \Seen flag... it happens that the \Recent flag is set, so we're okay with that */ + SWRITE(client1, "c2 FETCH 1 (FLAGS)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 FLAGS (\\Recent))"); + + SWRITE(client1, "c3 FETCH 1 (UID FLAGS INTERNALDATE RFC822.SIZE BODYSTRUCTURE BODY.PEEK[HEADER.FIELDS (Date Subject From To X-Priority Importance X-MSMail-Priority Priority)])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "Priority)] {119}"); + + /* Go ahead and mark it seen now */ + SWRITE(client1, "d1 FETCH 1 (BODY[])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "d1 OK"); + + SWRITE(client1, "d2 FETCH 1 (FLAGS)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* 1 FETCH (UID 1 FLAGS (\\Seen \\Recent))" ENDL); + + /* Request multiple BODY things in the same command */ + SWRITE(client1, "d3 FETCH 1 (BODY[2.MIME] BODY[3.MIME])" ENDL); + /* Since we don't have reliable readline, we can really only check for a single thing in response to a command, so need to check twice */ + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[2.MIME] {196}"); + SWRITE(client1, "d4 FETCH 1 (BODY[2.MIME] BODY[3.MIME])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[3.MIME] {70}"); + + SWRITE(client1, "d5 FETCH 1 (BODY[HEADER.FIELDS (TO)] BODY[HEADER.FIELDS (SUBJECT CC)])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "Subject: Test"); + + /* Message 2 contains a message/rfc822, so we can ask for the additional part specifiers that are specific to that: */ + SWRITE(client1, "e1 FETCH 1:2 (BODY.PEEK[1.MIME])" ENDL); /* Do 2 messages at once */ + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1.MIME] {78}"); + + SWRITE(client1, "e2 FETCH 2 (BODY.PEEK[3])" ENDL); /* Part spec 3 is the subpart corresponding to a message/rfc822 */ + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[3] {429}"); + + SWRITE(client1, "e3 FETCH 2 (BODY.PEEK[3.HEADER])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[3.HEADER] {417}"); + + SWRITE(client1, "e4 FETCH 2 (BODY.PEEK[3.TEXT])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[3.TEXT] {8}"); + + SWRITE(client1, "e5 FETCH 2 (BODY.PEEK[3.HEADER.FIELDS (TO SUBJECT)])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[3.HEADER.FIELDS (TO SUBJECT)] {39}"); + + SWRITE(client1, "e6 FETCH 2 (FLAGS BODY.PEEK[3.HEADER.FIELDS (TO SUBJECT)] BODY.PEEK[3.HEADER.FIELDS (CC SUBJECT)])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[3.HEADER.FIELDS (CC SUBJECT)] {17}"); + SWRITE(client1, "e7 FETCH 2 (FLAGS BODY.PEEK[3.HEADER.FIELDS (TO SUBJECT)] BODY.PEEK[3.HEADER.FIELDS (CC SUBJECT)])" ENDL); /* Repeat to capture other one */ + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[3.HEADER.FIELDS (TO SUBJECT)] {39}"); + + SWRITE(client1, "e8 FETCH 2 (BODY.PEEK[3.HEADER.FIELDS.NOT (TO CC SUBJECT)])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[3.HEADER.FIELDS.NOT (TO CC SUBJECT)] {380}"); + + /* The Received header in this message is multiline, and should all appear on a single line without CR LF inbetween + * This is true for HEADER.FIELDS/HEADER.FIELDS.NOT as well as just HEADER */ + SWRITE(client1, "e9 FETCH 2 (BODY.PEEK[3.HEADER.FIELDS.NOT (TO CC SUBJECT)])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "Received: from [10.1.1.1]\tby example.com"); + + SWRITE(client1, "e10 FETCH 2 (BODY.PEEK[3.HEADER.FIELDS (Received To)])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "Received: from [10.1.1.1]\tby example.com"); + + SWRITE(client1, "e11 FETCH 2 (BODY.PEEK[])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[] {1999}"); + + SWRITE(client1, "f1 FETCH 3 (BODYSTRUCTURE)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "\"------------B56ACAA1D42503EF82F52166\""); + + SWRITE(client1, "f2 FETCH 3 (BODY.PEEK[])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[] {1671}"); + + SWRITE(client1, "f3 FETCH 3 (RFC822.HEADER)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "RFC822.HEADER {494}"); + + /* Obsoleted BODY[0] syntax. Defined in RFC 1730 but obsoleted by RFC 3501. Should be equivalent to getting the header. */ + SWRITE(client1, "f4 FETCH 3 (BODY.PEEK[0])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[0] {494}"); + + SWRITE(client1, "f5 FETCH 3 (BODY.PEEK[1])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1] {714}"); + + SWRITE(client1, "f6 FETCH 3 (BODY.PEEK[1.1])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1.1] {101}"); + + SWRITE(client1, "f7 FETCH 3 (BODY.PEEK[1.2])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1.2] {319}"); + + SWRITE(client1, "f8 FETCH 3 (BODY.PEEK[2])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[2] {36}"); + + SWRITE(client1, "f9 FETCH 3 (BODY.PEEK[1.MIME])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1.MIME] {88}"); + + SWRITE(client1, "f10 FETCH 3 (BODY.PEEK[1.1.MIME])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1.1.MIME] {91}"); + + SWRITE(client1, "f11 FETCH 3 (BODY.PEEK[1.2.MIME])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1.2.MIME] {75}"); + + SWRITE(client1, "f12 FETCH 3 (BODY.PEEK[2.MIME])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[2.MIME] {161}"); + + /* Nonexistent subpart */ + SWRITE(client1, "f13 FETCH 3 (BODY.PEEK[2.1])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[2.1] {0}"); + + /* .TEXT isn't legal on non message/rfc822 subparts */ + SWRITE(client1, "g1 FETCH 2 (BODY.PEEK[1.TEXT])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1.TEXT] {0}"); + SWRITE(client1, "g2 FETCH 3 (BODY.PEEK[1.TEXT])" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[1.TEXT] {0}"); + + /* Now, test partial fetches: */ + SWRITE(client1, "g3 FETCH 3 (BODY.PEEK[]<0>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[]<0> {1671}"); + + SWRITE(client1, "g4 FETCH 3 (BODY.PEEK[]<2>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[]<2> {1669}"); /* start is 0-indexed, so this should be 2 smaller than entire body */ + + SWRITE(client1, "g5 FETCH 3 (BODY.PEEK[]<2.3>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[]<2> {3}"); + + SWRITE(client1, "g6 FETCH 3 (BODY.PEEK[]<0.9999>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[]<0> {1671}"); /* Should be truncated to available length */ + + SWRITE(client1, "g7 FETCH 3 (BODY.PEEK[]<8.9999>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[]<8> {1663}"); + + SWRITE(client1, "g8 FETCH 3 (BODY.PEEK[2]<1.4>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[2]<1> {4}"); + + SWRITE(client1, "g9 FETCH 3 (BODY.PEEK[2]<1.4>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "GVzd"); + + SWRITE(client1, "g10 FETCH 3 (BODY.PEEK[2]<25.15>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[2]<25> {11}"); + + SWRITE(client1, "g11 FETCH 3 (BODY.PEEK[2]<25.15>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "WNobWVudA=="); + + SWRITE(client1, "g12 FETCH 3 (BODY.PEEK[HEADER]<1.25>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "BODY[HEADER]<1> {25}"); + + /* Use message 2, since it has a message/rfc822 attachment */ + SWRITE(client1, "g13 FETCH 2 (BODY.PEEK[3.HEADER.FIELDS (Content-Type)]<1.12>)" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "ontent-Type:"); + + SWRITE(client1, "z999 LOGOUT" ENDL); + CLIENT_EXPECT_EVENTUALLY(client1, "* BYE"); + res = 0; + +cleanup: + close_if(client1); + return res; +} + +TEST_MODULE_INFO_STANDARD("IMAP FETCH Tests"); diff --git a/tests/test_imap_notify.c b/tests/test_imap_notify.c index efc66f3..63cfddd 100644 --- a/tests/test_imap_notify.c +++ b/tests/test_imap_notify.c @@ -52,7 +52,7 @@ static int send_message(int client1, size_t extrabytes) char subject[32]; if (!send_count++) { - CLIENT_EXPECT(client1, "220"); + CLIENT_EXPECT_EVENTUALLY(client1, "220 "); SWRITE(client1, "EHLO " TEST_EXTERNAL_DOMAIN ENDL); CLIENT_EXPECT_EVENTUALLY(client1, "250 "); /* "250 " since there may be multiple "250-" responses preceding it */ } else { @@ -96,7 +96,7 @@ static int send_message2(int client1, size_t extrabytes) char subject[32]; if (!send_count++) { - CLIENT_EXPECT(client1, "220"); + CLIENT_EXPECT_EVENTUALLY(client1, "220 "); SWRITE(client1, "EHLO " TEST_EXTERNAL_DOMAIN ENDL); CLIENT_EXPECT_EVENTUALLY(client1, "250 "); /* "250 " since there may be multiple "250-" responses preceding it */ } else { diff --git a/tests/test_mailscript.c b/tests/test_mailscript.c index dfa83bc..9df3c29 100644 --- a/tests/test_mailscript.c +++ b/tests/test_mailscript.c @@ -72,7 +72,7 @@ static int run(void) return -1; } - CLIENT_EXPECT(clientfd, "220"); + CLIENT_EXPECT_EVENTUALLY(clientfd, "220 "); /* Test each of the rules in test/.rules, one by one */ diff --git a/tests/test_pop3.c b/tests/test_pop3.c index 75373c1..86db31b 100644 --- a/tests/test_pop3.c +++ b/tests/test_pop3.c @@ -49,7 +49,7 @@ static int send_message(int client1) char subject[32]; if (!send_count++) { - CLIENT_EXPECT(client1, "220"); + CLIENT_EXPECT_EVENTUALLY(client1, "220 "); SWRITE(client1, "EHLO " TEST_EXTERNAL_DOMAIN ENDL); CLIENT_EXPECT_EVENTUALLY(client1, "250 "); /* "250 " since there may be multiple "250-" responses preceding it */ } else { diff --git a/tests/test_sieve.c b/tests/test_sieve.c index acb5fbe..c9664f2 100644 --- a/tests/test_sieve.c +++ b/tests/test_sieve.c @@ -75,7 +75,7 @@ static int run(void) return -1; } - CLIENT_EXPECT(clientfd, "220"); + CLIENT_EXPECT_EVENTUALLY(clientfd, "220 "); /* Test each of the rules in test/.sieve, one by one */ diff --git a/tests/test_smtp_mailing_lists.c b/tests/test_smtp_mailing_lists.c index 6deb9c6..d3ad887 100644 --- a/tests/test_smtp_mailing_lists.c +++ b/tests/test_smtp_mailing_lists.c @@ -75,7 +75,7 @@ static int handshake(int clientfd, int reset) SWRITE(clientfd, "RSET" ENDL); CLIENT_EXPECT(clientfd, "250"); } else { - CLIENT_EXPECT(clientfd, "220"); + CLIENT_EXPECT_EVENTUALLY(clientfd, "220 "); SWRITE(clientfd, "EHLO " TEST_EXTERNAL_DOMAIN ENDL); CLIENT_EXPECT_EVENTUALLY(clientfd, "250 "); /* "250 " since there may be multiple "250-" responses preceding it */ } diff --git a/tests/test_smtp_msa.c b/tests/test_smtp_msa.c index 33dac91..a2fac97 100644 --- a/tests/test_smtp_msa.c +++ b/tests/test_smtp_msa.c @@ -71,7 +71,7 @@ static int handshake(int clientfd, int reset) SWRITE(clientfd, "RSET" ENDL); CLIENT_EXPECT(clientfd, "250"); } else { - CLIENT_EXPECT(clientfd, "220"); + CLIENT_EXPECT_EVENTUALLY(clientfd, "220 "); SWRITE(clientfd, "EHLO " TEST_EXTERNAL_DOMAIN ENDL); CLIENT_EXPECT_EVENTUALLY(clientfd, "250 "); /* "250 " since there may be multiple "250-" responses preceding it */ }