ezmlmx 0.69
ezmlmx
Loading...
Searching...
No Matches
ezmlm-archive.c
Go to the documentation of this file.
1#include <sys/types.h>
2#include <sys/stat.h>
3#include <unistd.h>
4#include "alloc.h"
5#include "error.h"
6#include "stralloc.h"
7#include "str.h"
8#include "sig.h"
9#include "getconf.h"
10#include "logmsg.h"
11#include "getln.h"
12#include "buffer.h"
13#include "readwrite.h"
14#include "fmt.h"
15#include "getoptb.h"
16#include "idxthread.h"
17#include "makehash.h"
18#include "lock.h"
19#include "open.h"
20#include "scan.h"
21#include "idx.h"
22#include "errtxt.h"
23#include "lockfile.h"
24#include "auto_version.h"
25
26#define WHO "ezmlm-archive"
27
32
33int rename(const char *,const char *); // stdio.h
34
35buffer bi;
36char inbuf[1024];
37buffer bo;
38char outbuf[1024];
39buffer bn;
40char numbuf[16];
41
42stralloc line = {0};
43stralloc num = {0};
44stralloc fn = {0};
45stralloc fnn = {0};
46
47char strnum[FMT_ULONG];
48int flagerror = 0;
49int flagsync = 1; /* sync() by default, not for -c or -f or -t */
50char *dir;
51
52struct ca {
53 char *s; /* start */
54 unsigned int l; /* length */
56
57static void die_nomem() { logmsg(WHO,111,FATAL,ERR_NOMEM); }
58static void die_usage() { logmsg(WHO,100,USAGE,"ezmlm-archive [-cCFsSTvV] [-f min_msg] [-t max_msg] dir"); }
59
65
66void close_proper(buffer *bo,char *s,char *sn)
67{
68 if (buffer_flush(bo) == -1)
69 logmsg(WHO,111,FATAL,B(ERR_FLUSH,dir,"/",s));
70 if (flagsync)
71 if (fsync(bo->fd) == -1)
72 logmsg(WHO,111,FATAL,B(ERR_SYNC,dir,"/",s));
73 if (close(bo->fd) == -1)
74 logmsg(WHO,111,FATAL,B(ERR_CLOSE,dir,"/",s));
75 if (rename(sn,s) == -1)
76 logmsg(WHO,111,FATAL,B(ERR_MOVE,dir,"/",s));
77}
78
84
85void write_threads(msgentry *msgtable,subentry *subtable,authentry *authtable,dateentry *datetable,unsigned long from,unsigned long to)
86{
87 msgentry *pmsgt;
88 subentry *psubt, *psubtm, *psubtlast;
89 subentry *presubt = (subentry *)0;
90 authentry *pautht;
91 dateentry *pdatet;
92 const char *cp;
93 const char *cp1;
94 unsigned long msg;
95 unsigned long ulmsginthread;
96 unsigned long subnum;
97 unsigned long authnum;
98 unsigned long msgnum;
99 unsigned int pos;
100 unsigned int startdate, nextdate;
101 unsigned int startmsg, nextmsg;
102 int fd = -1;
103 int fdn = -1;
104 int match;
105 int ffound;
106 int lineno;
107 int res;
108 int r;
109
110 psubtm = subtable; /* now for new threads */
111 pdatet = datetable;
112 nextmsg = 0L;
113 nextdate = pdatet->date;
114
115 while (psubtm->sub) { /* these are in msgnum order */
116 if (!presubt) /* for rewind */
117 if (psubtm->lastmsg >= nextmsg)
118 presubt = psubtm; /* this thread extends beyond current month */
119
120 if (psubtm->firstmsg >= nextmsg) { /* done with this month */
121 if (fdn != -1) close_proper(&bo,fn.s,fnn.s);
122 psubtlast = psubtm; /* last thread done */
123 if (presubt) /* need to rewind? */
124 psubtm = presubt; /* do it */
125 psubt = psubtm; /* tmp pointer to reset done flag */
126 presubt = (subentry *)0; /* reset rewind pointer */
127 pdatet++; /* next month */
128 startdate = nextdate; /* startdate */
129 nextdate = pdatet->date; /* end date */
130 startmsg = nextmsg; /* first message in month */
131 nextmsg = pdatet->msg; /* first message in next month */
132
133 if (!stralloc_copys(&fn,"archive/threads/")) die_nomem();
134 if (!stralloc_catb(&fn,strnum,fmt_uint(strnum,startdate))) die_nomem();
135 if (!stralloc_copy(&fnn,&fn)) die_nomem();
136 if (!stralloc_0(&fn)) die_nomem();
137 if (!stralloc_cats(&fnn,"n")) die_nomem();
138 if (!stralloc_0(&fnn)) die_nomem();
139 if ((fdn = open_trunc(fnn.s)) == -1)
140 logmsg(WHO,111,FATAL,B(ERR_CREATE,dir,"/",fnn.s));
141 buffer_init(&bo,buffer_unixwrite,fdn,outbuf,sizeof(outbuf));
142 if ((fd = open_read(fn.s)) == -1) {
143 if (errno != ENOENT)
144 logmsg(WHO,111,FATAL,B(ERR_OPEN,dir,"/",fnn.s));
145 } else {
146 buffer_init(&bi,buffer_unixread,fd,inbuf,sizeof(inbuf));
147
148 for (;;) {
149 if (getln(&bi,&line,&match,'\n') == -1)
150 logmsg(WHO,111,FATAL,B(ERR_READ,dir,"/",fnn.s));
151 if (!match) break;
152 pos = scan_ulong(line.s,&msgnum);
153 pos++; /* skip ':' */
154 if (msgnum >= from) continue;/* ignore entries from threading range */
155 if (line.len < pos + HASHLEN) {
156 flagerror = -1; /* and bad ones */
157 continue;
158 }
159 psubt = subtable;
160 cp = line.s + pos;
161 ffound = 0; /* search among already known subjects */
162
163 for (;;) {
164 res = str_diffn(psubt->sub,cp,HASHLEN);
165 if (res < 0) {
166 if (psubt->higher)
167 psubt = psubt->higher;
168 else
169 break;
170 } else if (res > 0) {
171 if (psubt->lower)
172 psubt = psubt->lower;
173 else
174 break;
175 } else {
176 ffound = 1;
177 break;
178 }
179 }
180
181 if (!ffound) {
182 if (buffer_put(&bo,line.s,line.len) == -1)
183 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
184 } else { /* new # of msg in thread */
185 cp += HASHLEN; /* HASHLEN [#] Subject always \n at end */
186 if (*(cp++) == ' ' && *(cp++) == '[') {
187 cp += scan_ulong(cp,&ulmsginthread);
188 if (*cp == ']') {
189 psubt->msginthread += (unsigned char) (ulmsginthread & 0xff);
190 }
191 } else
192 flagerror = -5;
193 }
194 }
195 close(fd);
196 }
197 continue;
198 } /* done month if */
199
200 if (psubtm->firstmsg < nextmsg && psubtm->lastmsg >= startmsg) {
201 if (!stralloc_copyb(&line,strnum,fmt_ulong(strnum,psubtm->lastmsg)))
202 die_nomem();
203 if (!stralloc_cats(&line,":")) die_nomem();
204 if (!stralloc_catb(&line,psubtm->sub,HASHLEN)) die_nomem();
205 if (!stralloc_cats(&line," [")) die_nomem();
206 if (!stralloc_catb(&line,strnum,fmt_ulong(strnum,(unsigned long) psubtm->msginthread)))
207 die_nomem();
208 if (!stralloc_cats(&line,"]")) die_nomem();
209 if (!stralloc_catb(&line,psubtm->sub + HASHLEN,psubtm->sublen - HASHLEN))
210 die_nomem(); /* has \n */
211 if (buffer_put(&bo,line.s,line.len) == -1)
212 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
213
214 }
215 psubtm++;
216 }
217 if (fdn != -1)
218 close_proper(&bo,fn.s,fnn.s);
219
220 psubt = subtable;
221 while (psubt->sub) { /* now the threads */
222 if (!stralloc_copys(&fn,"archive/subjects/")) die_nomem();
223 if (!stralloc_catb(&fn,psubt->sub,2)) die_nomem();
224 if (!stralloc_0(&fn)) die_nomem();
225 if (mkdir(fn.s,0755) == -1)
226 if (errno != EEXIST)
227 logmsg(WHO,111,FATAL,B(ERR_CREATE,dir,"/",fnn.s));
228 fn.s[fn.len - 1] = '/';
229 if (!stralloc_catb(&fn,psubt->sub+2,HASHLEN-2)) die_nomem();
230 if (!stralloc_copy(&fnn,&fn)) die_nomem();
231 if (!stralloc_cats(&fnn,"n")) die_nomem();
232 if (!stralloc_0(&fn)) die_nomem();
233 if (!stralloc_0(&fnn)) die_nomem();
234 if ((fdn = open_trunc(fnn.s)) == -1)
235 logmsg(WHO,111,FATAL,B(ERR_CREATE,fnn.s));
236 buffer_init(&bo,buffer_unixwrite,fdn,outbuf,sizeof(outbuf));
237 if ((fd = open_read(fn.s)) == -1) {
238 if (errno != ENOENT)
239 logmsg(WHO,111,FATAL,B(ERR_OPEN,fn.s));
240 if (buffer_puts(&bo,psubt->sub) == -1) /* write subject */
241 logmsg(WHO,111,FATAL,B(ERR_WRITE,fnn.s));
242 } else { /* copy data */
243 buffer_init(&bi,buffer_unixread,fd,inbuf,sizeof(inbuf));
244 lineno = 0;
245
246 for (;;) { // STOP
247 if (getln(&bi,&line,&match,'\n') == -1)
248 logmsg(WHO,111,FATAL,B(ERR_READ,dir,"/",fnn.s));
249 if (!match) break;
250 if (!lineno) { /* write subject */
251 if (line.len < HASHLEN + 1 || line.s[HASHLEN] != ' ')
252 flagerror = -3;
253 if (buffer_put(&bo,line.s,line.len) == -1)
254 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
255 lineno = 1;
256 continue;
257 }
258 r = scan_ulong(line.s,&msgnum);
259 if (msgnum >= from) break;
260 if (buffer_put(&bo,line.s,line.len) == -1)
261 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
262 }
263 r = close(fd); /* close old index */
264 }
265
266 subnum = (unsigned long) (psubt - subtable + 1); /* idx of this subj */
267 pmsgt = msgtable + psubt->firstmsg - from; /* first message entry */
268 for (msg = psubt->firstmsg; msg <= psubt->lastmsg; msg++) {
269 if (pmsgt->subnum == subnum) {
270 if (!stralloc_copyb(&line,strnum,fmt_ulong(strnum,msg))) die_nomem();
271 if (!stralloc_cats(&line,":")) die_nomem();
272 if (!stralloc_catb(&line,strnum,fmt_uint(strnum,pmsgt->date)))
273 die_nomem();
274 if (!stralloc_cats(&line,":")) die_nomem();
275 if (pmsgt->authnum) {
276 pautht = authtable + pmsgt->authnum - 1;
277 cp = pautht->auth;
278 cp1 = cp + str_chr(cp,' ');
279 if (cp + HASHLEN != cp1)
280 logmsg(WHO,100,ERROR,ERR_BAD_INDEX);
281 if (!stralloc_cats(&line,cp))
282 die_nomem(); /* hash */
283 } else
284 if (!stralloc_cats(&line,"\n")) die_nomem();
285 if (buffer_put(&bo,line.s,line.len) == -1)
286 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
287 }
288 pmsgt++;
289 }
290 close_proper(&bo,fn.s,fnn.s);
291 psubt++;
292 }
293
294 /* (no master author index) */
295
296 pautht = authtable;
297
298 while (pautht->auth) { /* now the authors */
299 if (!stralloc_copys(&fn,"archive/authors/")) die_nomem();
300 if (!stralloc_catb(&fn,pautht->auth,2)) die_nomem();
301 if (!stralloc_0(&fn)) die_nomem();
302 if (mkdir(fn.s,0755) == -1)
303 if (errno != EEXIST)
304 logmsg(WHO,111,FATAL,B(ERR_CREATE,dir,"/",fn.s));
305 fn.s[fn.len - 1] = '/';
306 if (!stralloc_catb(&fn,pautht->auth+2,HASHLEN-2)) die_nomem();
307 if (!stralloc_copy(&fnn,&fn)) die_nomem();
308 if (!stralloc_cats(&fnn,"n")) die_nomem();
309 if (!stralloc_0(&fn)) die_nomem();
310 if (!stralloc_0(&fnn)) die_nomem();
311 if ((fdn = open_trunc(fnn.s)) == -1)
312 logmsg(WHO,111,FATAL,B(ERR_CREATE,fnn.s));
313 buffer_init(&bo,buffer_unixwrite,fdn,outbuf,sizeof(outbuf));
314 if ((fd = open_read(fn.s)) == -1) {
315 if (errno != ENOENT)
316 logmsg(WHO,111,FATAL,B(ERR_OPEN,fn.s));
317 else { /* didn't exist before: write author */
318 if (buffer_put(&bo,pautht->auth,pautht->authlen) == -1)
319 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
320 }
321 } else { /* copy data */
322 buffer_init(&bi,buffer_unixread,fd,inbuf,sizeof(inbuf));
323 lineno = 0;
324
325 for (;;) {
326 if (getln(&bi,&line,&match,'\n') == -1)
327 logmsg(WHO,111,FATAL,B(ERR_READ,dir,"/",fn.s));
328 if (!match) break;
329 if (!lineno) { /* write author */
330 if (line.len < HASHLEN + 1 || line.s[HASHLEN] != ' ')
331 flagerror = - 4;
332 if (buffer_put(&bo,line.s,line.len) == -1)
333 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
334 lineno = 1;
335 continue;
336 }
337 scan_ulong(line.s,&msgnum);
338 if (msgnum >= from) break;
339 if (buffer_put(&bo,line.s,line.len) == -1)
340 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
341 }
342 close(fd); /* close old index */
343 }
344
345 authnum = (unsigned long) (pautht - authtable + 1); /* idx of this auth */
346 pmsgt = msgtable + pautht->firstmsg - from; /* first message entry */
347 for (msg = pautht->firstmsg; msg <= to; msg++) {
348 if (pmsgt->authnum == authnum) {
349 if (!stralloc_copyb(&line,strnum,fmt_ulong(strnum,msg))) die_nomem();
350 if (!stralloc_cats(&line,":")) die_nomem();
351 if (!stralloc_catb(&line,strnum,fmt_uint(strnum,pmsgt->date)))
352 die_nomem();
353 if (!stralloc_cats(&line,":")) die_nomem();
354 if (pmsgt->subnum) {
355 psubt = subtable + pmsgt->subnum - 1;
356 if (!stralloc_catb(&line,psubt->sub,psubt->sublen))
357 die_nomem();
358 }
359 if (buffer_put(&bo,line.s,line.len) == -1)
360 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
361 }
362 pmsgt++;
363 }
364 close_proper(&bo,fn.s,fnn.s);
365 pautht++;
366 }
367}
368
369int main(int argc,char **argv)
370{
371 unsigned long archnum = 0L;
372 unsigned long to = 0L;
373 unsigned long max;
374 int fd;
375 int fdlock;
376 int flagcreate = 0;
377 int flagsyncall = 0;
378 int opt;
379 msgentry *msgtable;
380 subentry *subtable;
381 authentry *authtable;
382 dateentry *datetable;
383
384 umask(022);
385 sig_pipeignore();
386
387 while ((opt = getoptb(argc,argv,"cCf:FsSt:TvV")) != opteof)
388 switch (opt) {
389 case 'c': flagcreate = 1; flagsync = 0; break; /* start at beginning of archive */
390 case 'C': flagcreate = 0; break; /* Do only archnum+1 => num */
391 case 'f': if (optarg) { scan_ulong(optarg,&archnum); archnum = (archnum / 100) * 100; }
392 flagsync = 0; break;
393 case 'F': archnum = 0; break;
394 case 's': flagsyncall = 1; break;
395 case 'S': flagsyncall = 0; break;
396 case 't': if (optarg) { scan_ulong(optarg,&to); } flagsync = 0; break;
397 case 'T': to = 0; break;
398 case 'v':
399 case 'V': logmsg(WHO,0,VERSION,auto_version);
400 default: die_usage();
401 }
402
403 if (flagsyncall) flagsync = 1; /* overrides */
404 dir = argv[optind++];
405 if (!dir) die_usage();
406 if (chdir(dir) == -1)
407 logmsg(WHO,111,FATAL,B(ERR_SWITCH,dir));
408
409 if (mkdir("archive/threads",0755) == -1)
410 if (errno != EEXIST)
411 logmsg(WHO,111,FATAL,B(ERR_CREATE,dir,"/archive/threads"));
412 if (mkdir("archive/subjects",0755) == -1)
413 if (errno != EEXIST)
414 logmsg(WHO,111,FATAL,B(ERR_CREATE,dir,"/archive/subjects"));
415 if (mkdir("archive/authors",0755) == -1)
416 if (errno != EEXIST)
417 logmsg(WHO,111,FATAL,B(ERR_CREATE,dir,"/archive/authors"));
418
419 /* Lock list to assure that no ezmlm-send is working on it */
420 /* and that the "num" message is final */
421
422 fdlock = lockfile("lock");
423 if (!getconf_line(&num,"num",0,dir)) /* get num */
424 logmsg(WHO,100,ERROR,ERR_EMPTY_LIST);
425 close(fdlock);
426
427 if (!stralloc_0(&num)) die_nomem(); /* parse num */
428 scan_ulong(num.s,&max);
429 if (!to || to > max) to = max;
430
431 fdlock = lockfile("archive/lock"); /* lock index */
432 if (!flagcreate && !archnum) { /* adjust archnum (from) / to */
433 if (getconf_line(&num,"archnum",0,dir)) {
434 if (!stralloc_0(&num)) die_nomem();
435 scan_ulong(num.s,&archnum);
436 archnum++;
437 }
438 }
439
440 if (archnum > to) _exit(0); /* nothing to do */
441
442 /* do the subject threading */
443 idx_mkthreads(&msgtable,&subtable,&authtable,&datetable,archnum,to,max,0);
444
445 /* update the index */
446 write_threads(msgtable,subtable,authtable,datetable,archnum,to);
447
448 /* update archnum */
449 if ((fd = open_trunc("archnumn")) == -1)
450 logmsg(WHO,111,FATAL,B(ERR_CREATE,dir,"/archnumn"));
451 buffer_init(&bn,buffer_unixwrite,fd,numbuf,sizeof(numbuf));
452 if (buffer_put(&bn,strnum,fmt_ulong(strnum,to)) == -1)
453 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
454 if (buffer_puts(&bn,"\n") == -1)
455 logmsg(WHO,111,FATAL,B(ERR_WRITE,dir,"/",fnn.s));
456 close_proper(&bn,"archnum","archnumn");
457
458 switch (flagerror) {
459 case 0: _exit(0); /* go bye-bye */
460 case -1: logmsg(WHO,99,WARN,"threads entry with illegal format"); break;
461 case -2: logmsg(WHO,99,WARN,"thread in index, but threadfile mibing"); break;
462 case -3: logmsg(WHO,99,WARN,"a subject file lacks subject"); break;
463 case -4: logmsg(WHO,99,WARN,"an author file lacks author/hash"); break;
464 case -5: logmsg(WHO,99,WARN,"threads entry lacks message count"); break;
465 default: logmsg(WHO,99,WARN,"something happened that isn't quite right");
466 }
467}
int opt
Definition ezmlm-cron.c:55
const char * cp
Definition ezmlm-cron.c:78
int fdlock
Definition ezmlm-cron.c:73
#define WHO
Definition author.c:1
unsigned long max
Definition ezmlm-get.c:85
void die_nomem()
Definition getconf.c:17
int getconf_line(stralloc *sa, const char *fn, int flagrequired, const char *dir)
Definition getconf.c:53
int main()
Definition ezmlm-weed.c:69
unsigned long lineno
Definition ezmlm-split.c:52
int flagerror
stralloc fn
char inbuf[1024]
void close_proper(buffer *bo, char *s, char *sn)
flush,sync,close,move sn->s
int rename(const char *, const char *)
buffer bn
char outbuf[1024]
stralloc fnn
stralloc num
char numbuf[16]
char * dir
void write_threads(msgentry *msgtable, subentry *subtable, authentry *authtable, dateentry *datetable, unsigned long from, unsigned long to)
buffer bi
buffer bo
int flagsync
#define HASHLEN
Definition idxthread.c:25
void idx_mkthreads(msgentry **pmsgtable, subentry **psubtable, authentry **pauthtable, dateentry **pdatetable, unsigned long msg_from, unsigned long msg_to, unsigned long msg_latest, int locked)
Definition idxthread.c:156
unsigned long msgnum
stralloc to
Definition ezmlm-clean.c:97
stralloc from
int fd
Definition ezmlm-cgi.c:141
int match
Definition ezmlm-cgi.c:140
const char auto_version[]
int lockfile(const char *)
Definition lockfile.c:15
Error messages. If you translate these, I would urge you to keep the English version as well....
#define ERR_BAD_INDEX
Definition errtxt.h:64
#define ERR_FLUSH
Definition errtxt.h:20
#define ERR_NOMEM
Definition errtxt.h:14
#define ERR_SYNC
Definition errtxt.h:22
#define ERR_OPEN
Definition errtxt.h:30
#define ERR_MOVE
Definition errtxt.h:29
#define ERR_READ
Definition errtxt.h:18
#define ERR_CREATE
Definition errtxt.h:28
#define ERR_SWITCH
Definition errtxt.h:42
#define ERR_EMPTY_LIST
Definition errtxt.h:62
#define ERR_WRITE
Definition errtxt.h:17
#define ERR_CLOSE
Definition errtxt.h:16
unsigned int l
char * s
unsigned int date
Definition idx.h:330
unsigned long authnum
Definition idx.h:329
unsigned long subnum
Definition idx.h:328
void * higher
Definition idx.h:334
unsigned long firstmsg
Definition idx.h:342
char * sub
Definition idx.h:336
unsigned char msginthread
Definition idx.h:344
unsigned long lastmsg
Definition idx.h:343
void * lower
Definition idx.h:335
unsigned int sublen
Definition idx.h:341
unsigned long firstmsg
Definition idx.h:356
unsigned long authlen
Definition idx.h:355
char * auth
Definition idx.h:350
unsigned int date
Definition idx.h:362
unsigned int msg
Definition idx.h:363
const char * logmsg(const char *dir, unsigned long num, unsigned long listno, unsigned long subs, int done)
Definition loginfo.c:32