Commit 25522f349db7b4e127fcb39bbebfaf89a890c9e8 - fdm

+4

-0

CHANGES less more

	0	24 August 2007
	1
	2	* remove-header now accepts a list of headers in {}.
	3
0	4	23 August 2007
1	5
2	6	* Yet another fetch reorganisation to try to make it clearer. Move tracking

+2

-2

TODO less more

21	21	- better website
22	22	- generic flags on account so that no-verify/no-apop don't have to be in the
23	23	right order
24		- document mbox fetch
25		- remove-headers { ... }
26	24	- pop3 over pipe
27	25	- pop3/imap tests via pipe and coprocess
28	26	- aio for fetch maildir
29	27	- fetch from gzipped mbox
	28	- document mbox fetch
	29	- document remove-headers { ... }

+6

-6

child-fetch.c less more

66	66	/*
67	67	* Complete flag. Set when fetch code is finished. All fetching finished after
68	68	* queues drain.
69		*/
	69	*/
70	70	int fetch_complete;
71	71
72	72	int

325	325	if (n == last \|\| n % conf.purge_after != 0)
326	326	return (0);
327	327	last = n;
328
	328
329	329	log_debug("%s: purging after %u mails", a->name, n);
330	330	return (1);
331	331	}

427	427	ARRAY_ADD(&iol, pio);
428	428	if (a->fetch->fill != NULL)
429	429	a->fetch->fill(a, &iol);
430
	430
431	431	/* Poll for fetch data or privsep messages. */
432	432	log_debug3("%s: queued %u; blocked %u; flags 0x%02x", a->name,
433	433	fetch_queued, fetch_blocked, fctx.flags);

442	442	log_warnx("%s: polling error. aborted", a->name);
443	443	else
444	444	log_warnx("%s: fetching error. aborted", a->name);
445
	445
446	446	aborted = 1;
447	447
448	448	finished:

453	453
454	454	xfree(fctx.lbuf);
455	455	fetch_free();
456		ARRAY_FREE(&iol);
	456	ARRAY_FREE(&iol);
457	457
458	458	/* Close caches. */
459	459	TAILQ_FOREACH(cache, &conf.caches, entry) {

469	469	n = fetch_dropped + fetch_kept;
470	470	if (n > 0) {
471	471	log_info("%s: %u messages processed (%u kept) in %.3f "
472		"seconds (average %.3f)", a->name, n, fetch_kept,
	472	"seconds (average %.3f)", a->name, n, fetch_kept,
473	473	tim, tim / n);
474	474	} else {
475	475	log_info("%s: 0 messages processed in %.3f seconds",

+42

-34

deliver-remove-header.c less more

40	40	struct deliver_remove_header_data *data = ti->data;
41	41	char ptr, hdr;
42	42	size_t len, off, wrap;
43		u_int i;
	43	u_int i, j;
44	44
45		hdr = replacestr(&data->hdr, m->tags, m, &m->rml);
46		if (hdr == NULL \|\| *hdr == '\0') {
47		if (hdr != NULL)
48		xfree(hdr);
49		log_warnx("%s: empty header", a->name);
50		return (DELIVER_FAILURE);
51		}
52		log_debug2("%s: removing header: %s", a->name, hdr);
53	45
54		ARRAY_FREE(&m->wrapped);
55		m->wrapchar = '\0';
56		fill_wrapped(m);
	46	for (j = 0; j < ARRAY_LENGTH(data->hdrs); j++) {
	47	hdr = replacestr(
	48	&ARRAY_ITEM(data->hdrs, j), m->tags, m, &m->rml);
	49	if (hdr == NULL \|\| *hdr == '\0') {
	50	if (hdr != NULL)
	51	xfree(hdr);
	52	log_warnx("%s: empty header", a->name);
	53	return (DELIVER_FAILURE);
	54	}
	55	log_debug2("%s: removing header: %s", a->name, hdr);
57	56
58		set_wrapped(m, ' ');
	57	ARRAY_FREE(&m->wrapped);
	58	m->wrapchar = '\0';
	59	fill_wrapped(m);
59	60
60		while ((ptr = match_header(m, hdr, &len, 0)) != NULL) {
61		log_debug3("%s: found header to remove: %.*s", a->name,
62		(int) len, ptr);
	61	set_wrapped(m, ' ');
63	62
64		/* Remove the header. */
65		memmove(ptr, ptr + len, m->size - len - (ptr - m->data));
66		m->size -= len;
67		m->body -= len;
	63	while ((ptr = match_header(m, hdr, &len, 0)) != NULL) {
	64	log_debug3("%s: found header to remove: %.*s", a->name,
	65	(int) len, ptr);
68	66
69		/* Fix up the wrapped array. */
70		off = ptr - m->data;
71		i = 0;
72		while (i < ARRAY_LENGTH(&m->wrapped)) {
73		wrap = ARRAY_ITEM(&m->wrapped, i);
74		if (wrap >= off + len) {
75		ARRAY_SET(&m->wrapped, i, wrap - len);
76		i++;
77		} else if (wrap >= off)
78		ARRAY_REMOVE(&m->wrapped, i);
79		else
80		i++;
	67	/* Remove the header. */
	68	memmove(
	69	ptr, ptr + len, m->size - len - (ptr - m->data));
	70	m->size -= len;
	71	m->body -= len;
	72
	73	/* Fix up the wrapped array. */
	74	off = ptr - m->data;
	75	i = 0;
	76	while (i < ARRAY_LENGTH(&m->wrapped)) {
	77	wrap = ARRAY_ITEM(&m->wrapped, i);
	78	if (wrap >= off + len) {
	79	ARRAY_SET(&m->wrapped, i, wrap - len);
	80	i++;
	81	} else if (wrap >= off)
	82	ARRAY_REMOVE(&m->wrapped, i);
	83	else
	84	i++;
	85	}
81	86	}
82	87	}
83	88
84		/* invalidate the match data since stuff may have moved */
	89	/* Invalidate the match data since stuff may have moved. */
85	90	m->rml.valid = 0;
86	91
87	92	set_wrapped(m, '\n');

92	97	deliver_remove_header_desc(struct actitem ti, char buf, size_t len)
93	98	{
94	99	struct deliver_remove_header_data *data = ti->data;
	100	char *hdrs;
95	101
96		xsnprintf(buf, len, "remove-header \"%s\"", data->hdr.str);
	102	hdrs = fmt_replstrs("remove-headers ", data->hdrs);
	103	strlcpy(buf, hdrs, len);
	104	xfree(hdrs);
97	105	}

+1

-1

deliver-rewrite.c less more

115	115	error_cause:
116	116	log_warnx("%s: %s: %s", a->name, s, cause);
117	117	xfree(cause);
118
	118
119	119	error:
120	120	if (cmd != NULL)
121	121	cmd_free(cmd);

+1

-1

deliver.h less more

87	87
88	88	/* Deliver remove-header data. */
89	89	struct deliver_remove_header_data {
90		struct replstr hdr;
	90	struct replstrs *hdrs;
91	91	};
92	92
93	93	/* Deliver write data. */

+10

-10

fetch-mbox.c less more

257	257	struct fetch_mbox_data *data = a->data;
258	258	struct fetch_mbox_mbox *fmbox;
259	259	u_int i;
260
	260
261	261	for (i = 0; i < ARRAY_LENGTH(&data->fmboxes); i++) {
262	262	fmbox = ARRAY_ITEM(&data->fmboxes, i);
263
	263
264	264	if (fmbox->base != NULL)
265	265	munmap(fmbox->base, fmbox->size);
266	266	if (fmbox->fd != -1)

278	278	fetch_mbox_state_init(struct account a, struct fetch_ctx fctx)
279	279	{
280	280	struct fetch_mbox_data *data = a->data;
281
	281
282	282	if (fetch_mbox_make(a) != 0)
283	283	return (FETCH_ERROR);
284	284	if (ARRAY_EMPTY(&data->fmboxes)) {

328	328	return (FETCH_ERROR);
329	329	}
330	330	fmbox->size = size;
331
	331
332	332	log_debug3("%s: opening mbox, size %ju", a->name, size);
333	333	used = 0;
334	334	do {

342	342	goto error;
343	343	}
344	344	} while (fmbox->fd < 0);
345
	345
346	346	/* mmap the file. */
347	347	fmbox->base = mmap(
348	348	NULL, fmbox->size, PROT_READ\|PROT_WRITE, MAP_SHARED, fmbox->fd, 0);

452	452	data->off = fmbox->size;
453	453	} else
454	454	data->off += ptr - line + 1;
455
	455
456	456	/* Check if the line is "From ". */
457	457	if (line > fmbox->base &&
458	458	ptr - line >= 5 && strncmp(line, "From ", 5) == 0) {

469	469	lptr++;
470	470	llen--;
471	471	}
472
	472
473	473	if (llen >= 5 && strncmp(lptr, "From ", 5) == 0)
474	474	line++;
475	475	}

496	496	aux->size -= 2;
497	497	m->size -= 2;
498	498	}
499
	499
500	500	return (FETCH_MAIL);
501	501	}
502	502

511	511	if (fetch_mbox_save(a, ARRAY_ITEM(&data->fmboxes, i)) != 0)
512	512	return (FETCH_ERROR);
513	513	}
514
515		fetch_mbox_abort(a);
	514
	515	fetch_mbox_abort(a);
516	516	return (FETCH_EXIT);
517	517	}
518	518

+9

-9

fetch-nntp.c less more

59	59	struct fetch fetch_nntp = {
60	60	"nntp",
61	61	fetch_nntp_state_connect,
62
	62
63	63	fetch_nntp_fill,
64	64	NULL,
65	65	fetch_nntp_abort,

116	116	va_list ap;
117	117	u_int i;
118	118	int code;
119
	119
120	120	if (codep == NULL)
121	121	codep = &code;
122	122

318	318	void
319	319	fetch_nntp_fill(struct account a, struct iolist iol)
320	320	{
321		struct fetch_nntp_data *data = a->data;
	321	struct fetch_nntp_data *data = a->data;
322	322
323	323	if (data->io != NULL)
324	324	ARRAY_ADD(iol, data->io);

428	428	return (FETCH_BLOCK);
429	429	}
430	430	} while (ARRAY_ITEM(&data->groups, data->group)->ignore);
431
	431
432	432	fctx->state = fetch_nntp_state_group;
433	433	return (FETCH_AGAIN);
434	434	}

478	478	struct fetch_nntp_group *group;
479	479	char *line;
480	480	u_int n;
481
	481
482	482	group = ARRAY_ITEM(&data->groups, data->group);
483	483
484	484	if (fetch_nntp_check(a, fctx, &line, NULL, 1, 211) != 0)

621	621	fctx->state = fetch_nntp_state_line;
622	622	return (FETCH_AGAIN);
623	623	}
624
	624
625	625	/* Line state. */
626	626	int
627	627	fetch_nntp_state_line(struct account a, struct fetch_ctx fctx)

634	634	line = io_readline2(data->io, &fctx->lbuf, &fctx->llen);
635	635	if (line == NULL)
636	636	return (FETCH_BLOCK);
637
	637
638	638	if (line[0] == '.') {
639	639	if (line[1] == '\0')
640	640	break;
641	641	line++;
642	642	}
643
	643
644	644	if (data->flushing)
645	645	continue;
646
	646
647	647	if (append_line(m, line, strlen(line)) != 0) {
648	648	log_warn("%s: failed to resize mail", a->name);
649	649	return (FETCH_ERROR);

+8

-8

fetch-pop3.c less more

55	55	int fetch_pop3_state_retr(struct account , struct fetch_ctx );
56	56	int fetch_pop3_state_line(struct account , struct fetch_ctx );
57	57	int fetch_pop3_state_quit(struct account , struct fetch_ctx );
58
	58
59	59	struct fetch fetch_pop3 = {
60	60	"pop3",
61	61	fetch_pop3_state_init,

223	223	xasprintf(&src, "%s%s", line, data->pass);
224	224	MD5(src, strlen(src), digest);
225	225	xfree(src);
226
	226
227	227	for (i = 0; i < MD5_DIGEST_LENGTH; i++)
228	228	xsnprintf(out + i * 2, 3, "%02hhx", digest[i]);
229	229

316	316	/* Handle dropped mail here. */
317	317	if (!TAILQ_EMPTY(&data->dropped)) {
318	318	aux = TAILQ_FIRST(&data->dropped);
319
	319
320	320	io_writeline(data->io, "DELE %u", aux->idx);
321	321	fctx->state = fetch_pop3_state_delete;
322	322	return (FETCH_BLOCK);

336	336	if (data->cur <= data->num)
337	337	data->cur++;
338	338
339		/*
	339	/*
340	340	* If this is the last mail, wait until everything has been committed
341	341	* back, then quit.
342	342	*/

535	535	line = io_readline2(data->io, &fctx->lbuf, &fctx->llen);
536	536	if (line == NULL)
537	537	return (FETCH_BLOCK);
538
	538
539	539	if (line[0] == '.') {
540	540	if (line[1] == '\0')
541	541	break;
542	542	line++;
543	543	}
544
	544
545	545	if (data->flushing)
546	546	continue;
547
	547
548	548	if (append_line(m, line, strlen(line)) != 0) {
549	549	log_warn("%s: failed to resize mail", a->name);
550	550	return (FETCH_ERROR);

563	563	{
564	564	struct fetch_pop3_data *data = a->data;
565	565	char *line;
566
	566
567	567	line = io_readline2(data->io, &fctx->lbuf, &fctx->llen);
568	568	if (line == NULL)
569	569	return (FETCH_BLOCK);

+1

-1

fetch-stdin.c less more

127	127	out:
128	128	if (io != NULL)
129	129	io_free(io);
130
	130
131	131	fctx->state = fetch_stdin_state_exit;
132	132	return (FETCH_MAIL);
133	133

+2

-2

fetch.h less more

34	34	struct fetch_ctx {
35	35	int (state)(struct account , struct fetch_ctx *);
36	36	int flags;
37
	37
38	38	struct mail *mail;
39	39
40	40	size_t llen;

74	74
75	75	ARRAY_DECL(, struct fetch_mbox_mbox *) fmboxes;
76	76	u_int index;
77
	77
78	78	size_t off;
79	79
80	80	TAILQ_HEAD(, fetch_mbox_mail) kept;

+5

-5

imap-common.c less more

713	713	return (FETCH_ERROR);
714	714	if (line == NULL)
715	715	return (FETCH_BLOCK);
716
	716
717	717	if (data->flushing)
718	718	continue;
719	719

722	722	size = strlen(line);
723	723	if (used + size + 2 > data->size)
724	724	break;
725
	725
726	726	if (append_line(m, line, size) != 0) {
727	727	log_warn("%s: failed to resize mail", a->name);
728	728	return (FETCH_ERROR);

739	739	return (imap_invalid(a, line));
740	740	if (line[left] != ')' \|\| line[left + 1] != '\0')
741	741	return (imap_invalid(a, line));
742
	742
743	743	/* If there was data left, add it as a new line without trailing \n. */
744	744	if (left > 0) {
745	745	if (append_line(m, line, left) != 0) {
746	746	log_warn("%s: failed to resize mail", a->name);
747	747	return (FETCH_ERROR);
748		}
	748	}
749	749	data->lines++;
750
	750
751	751	/* Wipe out the trailing \n. */
752	752	m->size--;
753	753	}

+3

-3

io.c less more

142	142	*rio = io;
143	143	if (io == NULL)
144	144	continue;
145
	145
146	146	switch (io_before_poll(io, &pfds[i])) {
147	147	case 0:
148	148	/* Found a closed io. */

187	187	if (io_after_poll(io, &pfds[i]) == -1)
188	188	goto error;
189	189	}
190
	190
191	191	xfree(pfds);
192	192	return (1);
193	193

218	218	pfd->events \|= POLLIN;
219	219	if (io->wr != NULL && (BUFFER_USED(io->wr) != 0 \|\|
220	220	(io->flags & (IOF_NEEDFILL\|IOF_NEEDPUSH\|IOF_MUSTWR)) != 0))
221		pfd->events \|= POLLOUT;
	221	pfd->events \|= POLLOUT;
222	222
223	223	return (1);
224	224	}

+2

-0

lex.c less more

119	119	{ "kilobyte", TOKKILOBYTES },
120	120	{ "kilobytes", TOKKILOBYTES },
121	121	{ "lock-file", TOKLOCKFILE },
	122	{ "lock-type", TOKLOCKTYPES },
122	123	{ "lock-types", TOKLOCKTYPES },
123	124	{ "m", TOKMEGABYTES },
124	125	{ "maildir", TOKMAILDIR },

153	154	{ "queue-high", TOKQUEUEHIGH },
154	155	{ "queue-low", TOKQUEUELOW },
155	156	{ "remove-header", TOKREMOVEHEADER },
	157	{ "remove-headers", TOKREMOVEHEADERS },
156	158	{ "returns", TOKRETURNS },
157	159	{ "rewrite", TOKREWRITE },
158	160	{ "second", TOKSECONDS },

+1

-1

mail.c less more

329	329	xfree(hdr);
330	330	}
331	331	}
332
	332
333	333	line_next(m, &ptr, len);
334	334	}
335	335	if (ptr == NULL)

+2

-1

parse-fn.c less more

405	405	xfree(data->path.str);
406	406	} else if (ti->deliver == &deliver_remove_header) {
407	407	struct deliver_remove_header_data *data = ti->data;
408		xfree(data->hdr.str);
	408	free_replstrs(data->hdrs);
	409	ARRAY_FREEALL(data->hdrs);
409	410	} else if (ti->deliver == &deliver_add_header) {
410	411	struct deliver_add_header_data *data = ti->data;
411	412	xfree(data->hdr.str);

+63

-19

parse.y less more

125	125	%token TOKMEGABYTES TOKGIGABYTES TOKBYTES TOKATTACHMENT TOKCOUNT TOKTOTALSIZE
126	126	%token TOKANYTYPE TOKANYNAME TOKANYSIZE TOKEQ TOKNE TOKNNTP TOKNNTPS TOKCACHE
127	127	%token TOKGROUP TOKGROUPS TOKPURGEAFTER TOKCOMPRESS TOKNORECEIVED TOKFILEUMASK
128		%token TOKFILEGROUP TOKVALUE TOKTIMEOUT TOKREMOVEHEADER TOKSTDOUT TOKNOVERIFY
129		%token TOKADDHEADER TOKQUEUEHIGH TOKQUEUELOW TOKVERIFYCERTS TOKEXPIRE
130		%token TOKTOCACHE TOKINCACHE TOKKEY TOKNOAPOP
	128	%token TOKFILEGROUP TOKVALUE TOKTIMEOUT TOKREMOVEHEADER TOKREMOVEHEADERS
	129	%token TOKSTDOUT TOKNOVERIFY TOKADDHEADER TOKQUEUEHIGH TOKQUEUELOW
	130	%token TOKVERIFYCERTS TOKEXPIRE TOKTOCACHE TOKINCACHE TOKKEY TOKNOAPOP
131	131	%token LCKFLOCK LCKFCNTL LCKDOTLOCK
132	132
133	133	%union

186	186	%type <gid> gid
187	187	%type <locks> lock locklist
188	188	%type <number> size time numv retrc expire
189		%type <replstrs> actions actionslist
	189	%type <replstrs> actions actionslist rmheaders rmheaderslist
190	190	%type <rule> perform
191	191	%type <server> server
192	192	%type <string> port to folder xstrv strv replstrv retre replpathv val optval

230	230	/** MBOXP */
231	231	mboxp: TOKMBOX
232	232	\| TOKMBOXES
	233	/** RMHEADERP */
	234	rmheaderp: TOKREMOVEHEADER
	235	\| TOKREMOVEHEADERS
	236	/** HEADERP */
	237	headerp: TOKHEADER
	238	\| TOKHEADERS
	239	/** DOMAINP */
	240	domainp: TOKDOMAIN
	241	\| TOKDOMAINS
233	242
234	243	/** VAL: <string> (char ) /
235	244	val: TOKVALUE strv

703	712	}
704	713
705	714	/** DOMAINS: <strings> (struct strings ) /
706		domains: TOKDOMAIN replstrv
	715	domains: domainp replstrv
707	716	/** [$2: replstrv (char )] /
708	717	{
709	718	char *cp;

717	726	cp = tolower((u_char) cp);
718	727	ARRAY_ADD($$, $2);
719	728	}
720		\| TOKDOMAINS '{' domainslist '}'
	729	\| domainp '{' domainslist '}'
721	730	/** [$3: domainslist (struct strings )] /
722	731	{
723	732	$$ = weed_strings($3);

753	762	}
754	763
755	764	/** HEADERS: <strings> (struct strings ) /
756		headers: TOKHEADER replstrv
	765	headers: headerp replstrv
757	766	/** [$2: replstrv (char )] /
758	767	{
759	768	char *cp;

767	776	cp = tolower((u_char) cp);
768	777	ARRAY_ADD($$, $2);
769	778	}
770		\| TOKHEADERS '{' headerslist '}'
	779	\| headerp '{' headerslist '}'
771	780	/** [$3: headerslist (struct strings )] /
772	781	{
773	782	$$ = weed_strings($3);

800	809	for (cp = $1; *cp != '\0'; cp++)
801	810	cp = tolower((u_char) cp);
802	811	ARRAY_ADD($$, $1);
	812	}
	813
	814	/** RMHEADERS: <replstrs> (struct replstrs ) /
	815	rmheaders: rmheaderp strv
	816	/** [$2: strv (char )] /
	817	{
	818	if (*$2 == '\0')
	819	yyerror("invalid header");
	820
	821	$$ = xmalloc(sizeof *$$);
	822	ARRAY_INIT($$);
	823	ARRAY_EXPAND($$, 1);
	824	ARRAY_LAST($$).str = $2;
	825	}
	826	\| rmheaderp '{' rmheaderslist '}'
	827	/** [$3: rmheaderslist (struct replstrs )] /
	828	{
	829	$$ = $3;
	830	}
	831
	832	/** RMHEADERSLIST: <replstrs> (struct replstrs ) /
	833	rmheaderslist: rmheaderslist strv
	834	/** [$1: rmheaderslist (struct replstrs )] [$2: strv (char )] */
	835	{
	836	if (*$2 == '\0')
	837	yyerror("invalid header");
	838
	839	$$ = $1;
	840	ARRAY_EXPAND($$, 1);
	841	ARRAY_LAST($$).str = $2;
	842	}
	843	\| replstrv
	844	/** [$1: replstrv (char )] /
	845	{
	846	if (*$1 == '\0')
	847	yyerror("invalid header");
	848
	849	$$ = xmalloc(sizeof *$$);
	850	ARRAY_INIT($$);
	851	ARRAY_EXPAND($$, 1);
	852	ARRAY_LAST($$).str = $1;
803	853	}
804	854
805	855	/** PATHSLIST: <strings> (struct strings ) /

1171	1221
1172	1222	data->path.str = $2;
1173	1223	}
1174		\| TOKREMOVEHEADER strv
1175		/** [$2: strv (char )] /
	1224	\| rmheaders
	1225	/** [$1: rmheaders (struct replstrs )] /
1176	1226	{
1177	1227	struct deliver_remove_header_data *data;
1178		char *cp;
1179
1180		if (*$2 == '\0')
1181		yyerror("invalid header");
1182	1228
1183	1229	$$ = xcalloc(1, sizeof *$$);
1184	1230	$$->deliver = &deliver_remove_header;

1186	1232	data = xcalloc(1, sizeof *data);
1187	1233	$$->data = data;
1188	1234
1189		for (cp = $2; *cp != '\0'; cp++)
1190		cp = tolower((u_char) cp);
1191		data->hdr.str = $2;
	1235	data->hdrs = $1;
1192	1236	}
1193	1237	\| TOKADDHEADER strv val
1194	1238	/** [$2: strv (char )] [$3: val (char )] */

2231	2275	/** FETCHTYPE: <fetch> (struct { ... } fetch) */
2232	2276	fetchtype: poptype server userpassnetrc apop verify
2233	2277	/** [$1: poptype (int)] [$2: server (struct { ... } server)] */
2234		/** [$3: userpassnetrc (struct { ... } userpass)] [$4: verify (int)] */
2235		/** [$5: apop (int)] */
	2278	/** [$3: userpassnetrc (struct { ... } userpass)] [$4: apop (int)] */
	2279	/** [$5: verify (int)] */
2236	2280	{
2237	2281	struct fetch_pop3_data *data;
2238	2282

+19

-0

regress/deliver/test-remove-header7.sh less more

	0	#!/bin/sh
	1	# $Id$
	2
	3	. ./test-deliver.subr && test_init
	4
	5	cat <<EOF\|test_in
	6	Header: Test
	7	Header2: Test
	8	Header: Test
	9
	10	EOF
	11
	12	cat <<EOF\|test_out
	13
	14	EOF
	15
	16	cat <<EOF\|test_run
	17	match all action remove-headers { "Header" "Header2" } continue
	18	EOF