1 diff -uN qmail-1.03/EXTTODO qmail-exttodo/EXTTODO

     2 --- qmail-1.03/EXTTODO	Thu Jan  1 01:00:00 1970

     3 +++ qmail-exttodo/EXTTODO	Sun Jan  5 22:12:01 2003

     4 @@ -0,0 +1,114 @@

     5 +EXTTODO by Claudio Jeker <jeker@n-r-g.com> and 

     6 +Andre Oppermann <opi@nrg4u.com>

     7 +(c) 1998,1999,2000,2001,2002 Internet Business Solutions Ltd.

     8 +

     9 +The EXTTODO patch is a part of the qmail-ldap patch.

    10 +This patches for qmail come with NO WARRANTY.

    11 +

    12 +These patches are under the BSD license.

    13 +

    14 +RELEASE: 5. Jan. 2003

    15 +

    16 +EXTTODO:

    17 +======================

    18 +

    19 +TOC:

    20 + WHAT DOES IT DO

    21 + INSTALL

    22 + CONFIG FILES

    23 + SETUP

    24 + BIG PICTURE

    25 +

    26 +NEWS:

    27 + 

    28 + This is the first release of the EXTTODO patch.

    29 +

    30 +================================================================================

    31 +

    32 +WHAT DOES IT DO

    33 +

    34 + The exttodo patch addresses a problem known as the silly qmail (queue)

    35 + problem. This problem is found only on system with high injection rates.

    36 +

    37 + qmail with a big local and remote concurrency could deliver a tremendous 

    38 + amount of messages but normally this can not be achieved because qmail-send

    39 + becomes a bottleneck on those high volumes servers.

    40 + qmail-send preprocesses all new messages before distributing them for local

    41 + or remote delivering. In one run qmail-send does one todo run but has the 

    42 + ability to close multiple jobs. Because of this layout qmail-send can not 

    43 + feed all the new available (local/remote) delivery slots and therefor it is 

    44 + not possible to achieve the maximum throughput.

    45 + This would be a minor problem if one qmail-send run could be done in extreme

    46 + short time but because of many file system calls (fsync and (un)link) a todo

    47 + run is expensive and throttles the throughput.

    48 +

    49 + The exttodo patch tries to solve the problem by moving the todo routine into 

    50 + an external program. This reduces the run time in qmail-send.

    51 +

    52 + exttodo adds a new program to qmail called qmail-todo. qmail-todo prepares

    53 + incoming messages for local and remote delivering (by creating info/<messid>

    54 + local/<messid> and remote/<messid> and removing todo/<messid>). See also

    55 + INTERNALS. As next qmail-todo transmits the <messid> to qmail-send which will

    56 + add this message into the priority queue which schedules the message for 

    57 + delivery. 

    58 +

    59 +INSTALL

    60 +

    61 + To enable the exttodo patch you need to define EXTERNAL_TODO while compiling

    62 + qmail(-ldap) this can be done with the -D flag of cc (e.g. cc -DEXTERNAL_TODO).

    63 +

    64 + NOTE: the exttodo patch can also be used on qmail systems without the 

    65 + qmail-ldap patch.

    66 + 

    67 +================================================================================

    68 +

    69 +CONFIG FILES

    70 +

    71 + No additional control files are used or needed.

    72 +

    73 +================================================================================

    74 +

    75 +SETUP

    76 +

    77 + qmail-todo will be started by qmail-start and therefor no additional setup

    78 + is needed.

    79 +

    80 + To verify that exttodo is running just check if qmail-todo is running.

    81 +

    82 +================================================================================

    83 +

    84 +BIG PICTURE

    85 +

    86 +               +-------+   +-------+

    87 +               | clean |   | clean |

    88 +               +--0-1--+   +--0-1--+       +-----------+

    89 +         trigger  ^ |         ^ |        +->0,1 lspawn |

    90 +            |     | v         | v       /  +-----------+

    91 + +-------+  v  +--2-3--+   +--5-6--+   /

    92 + |       |  |  |       0<--7     1,2<-+

    93 + | queue |--+--| todo  |   | send  |

    94 + |       |  |  |       1-->8     3,4<-+

    95 + +-------+     +-------+   +---0---+   \

    96 +                               |        \  +-----------+

    97 +                               v         +->0,1 rspwan |

    98 +                           +---0---+       +-----------+

    99 +                           | logger|

   100 +                           +-------+

   101 +

   102 +Communication between qmail-send and qmail-todo

   103 +

   104 +todo -> send:

   105 +   D[LRB]<mesgid>\0

   106 +          Start delivery for new message with id <messid>.

   107 +          the character L, R or B defines the type

   108 +          of delivery, local, remote or both respectively.

   109 +   L<string>\0

   110 +          Dump string to the logger without adding additional \n or similar.

   111 +send -> todo:

   112 +   H      Got a SIGHUP reread ~/control/locals and ~/control/virtualdomains

   113 +   X      Quit ASAP.

   114 +

   115 +qmail-todo sends "\0" terminated messages whereas qmail-send just send one

   116 +character to qmail-todo.

   117 +

   118 +

   119 diff -uN qmail-1.03/EXTTODO-INFO qmail-exttodo/EXTTODO-INFO

   120 --- qmail-1.03/EXTTODO-INFO	Thu Jan  1 01:00:00 1970

   121 +++ qmail-exttodo/EXTTODO-INFO	Tue Apr 30 16:49:02 2002

   122 @@ -0,0 +1,11 @@

   123 +Files modified:

   124 +Makefile

   125 +EXTTODO

   126 +FILES

   127 +TARGETS

   128 +qmail-send.c

   129 +qmail-todo.c

   130 +qmail-start.c

   131 +hier.c

   132 +install-big.c

   133 +

   134 diff -uN qmail-1.03/FILES qmail-exttodo/FILES

   135 --- qmail-1.03/FILES	Mon Jun 15 12:53:16 1998

   136 +++ qmail-exttodo/FILES	Mon Apr 22 13:59:28 2002

   137 @@ -431,3 +431,4 @@

   138  tcp-environ.5

   139  constmap.h

   140  constmap.c

   141 +qmail-todo.c

   142 diff -uN qmail-1.03/Makefile qmail-exttodo/Makefile

   143 --- qmail-1.03/Makefile	Mon Jun 15 12:53:16 1998

   144 +++ qmail-exttodo/Makefile	Mon Apr 22 14:55:59 2002

   145 @@ -1,5 +1,7 @@

   146  # Don't edit Makefile! Use conf-* for configuration.

   147  

   148 +DEFINES=-DEXTERNAL_TODO # use to enable external todo

   149 +

   150  SHELL=/bin/sh

   151  

   152  default: it

   153 @@ -703,7 +705,7 @@

   154  

   155  hier.o: \

   156  compile hier.c auto_qmail.h auto_split.h auto_uids.h fmt.h fifo.h

   157 -	./compile hier.c

   158 +	./compile $(DEFINES) hier.c

   159  

   160  home: \

   161  home.sh conf-qmail

   162 @@ -755,7 +757,7 @@

   163  install-big.o: \

   164  compile install-big.c auto_qmail.h auto_split.h auto_uids.h fmt.h \

   165  fifo.h

   166 -	./compile install-big.c

   167 +	./compile $(DEFINES) install-big.c

   168  

   169  install.o: \

   170  compile install.c substdio.h strerr.h error.h open.h readwrite.h \

   171 @@ -808,7 +810,7 @@

   172  forward preline condredirect bouncesaying except maildirmake \

   173  maildir2mbox maildirwatch qail elq pinq idedit install-big install \

   174  instcheck home home+df proc proc+df binm1 binm1+df binm2 binm2+df \

   175 -binm3 binm3+df

   176 +binm3 binm3+df qmail-todo

   177  

   178  load: \

   179  make-load warn-auto.sh systype

   180 @@ -1509,7 +1511,7 @@

   181  scan.h case.h auto_qmail.h trigger.h newfield.h stralloc.h quote.h \

   182  qmail.h substdio.h qsutil.h prioq.h datetime.h gen_alloc.h constmap.h \

   183  fmtqfn.h readsubdir.h direntry.h

   184 -	./compile qmail-send.c

   185 +	./compile $(DEFINES) qmail-send.c

   186  

   187  qmail-showctl: \

   188  load qmail-showctl.o auto_uids.o control.o open.a getln.a stralloc.a \

   189 @@ -1574,7 +1576,7 @@

   190  

   191  qmail-start.o: \

   192  compile qmail-start.c fd.h prot.h exit.h fork.h auto_uids.h

   193 -	./compile qmail-start.c

   194 +	./compile $(DEFINES) qmail-start.c

   195  

   196  qmail-tcpok: \

   197  load qmail-tcpok.o open.a lock.a strerr.a substdio.a error.a str.a \

   198 @@ -1605,6 +1607,20 @@

   199  compile qmail-tcpto.c substdio.h subfd.h substdio.h auto_qmail.h \

   200  fmt.h ip.h lock.h error.h exit.h datetime.h now.h datetime.h

   201  	./compile qmail-tcpto.c

   202 +

   203 +qmail-todo: \

   204 +load qmail-todo.o control.o constmap.o trigger.o fmtqfn.o now.o \

   205 +readsubdir.o case.a ndelay.a getln.a sig.a open.a stralloc.a alloc.a \

   206 +substdio.a error.a str.a fs.a auto_qmail.o auto_split.o

   207 +	./load qmail-todo control.o constmap.o trigger.o fmtqfn.o now.o \

   208 +	readsubdir.o case.a ndelay.a getln.a sig.a open.a stralloc.a \

   209 +	alloc.a substdio.a error.a str.a fs.a auto_qmail.o auto_split.o

   210 +

   211 +qmail-todo.o: \

   212 +compile alloc.h auto_qmail.h byte.h constmap.h control.h direntry.h error.h \

   213 +exit.h fmt.h fmtqfn.h getln.h open.h ndelay.h now.h readsubdir.h readwrite.h \

   214 +scan.h select.h str.h stralloc.h substdio.h trigger.h

   215 +	./compile $(DEFINES) qmail-todo.c

   216  

   217  qmail-upq: \

   218  warn-auto.sh qmail-upq.sh conf-qmail conf-break conf-split

   219 diff -uN qmail-1.03/TARGETS qmail-exttodo/TARGETS

   220 --- qmail-1.03/TARGETS	Mon Jun 15 12:53:16 1998

   221 +++ qmail-exttodo/TARGETS	Mon Apr 22 13:59:32 2002

   222 @@ -385,3 +385,5 @@

   223  man

   224  setup

   225  check

   226 +qmail-todo.o

   227 +qmail-todo

   228 diff -uN qmail-1.03/hier.c qmail-exttodo/hier.c

   229 --- qmail-1.03/hier.c	Mon Jun 15 12:53:16 1998

   230 +++ qmail-exttodo/hier.c	Mon Apr 22 14:01:58 2002

   231 @@ -108,6 +108,9 @@

   232    c(auto_qmail,"bin","qmail-rspawn",auto_uido,auto_gidq,0711);

   233    c(auto_qmail,"bin","qmail-clean",auto_uido,auto_gidq,0711);

   234    c(auto_qmail,"bin","qmail-send",auto_uido,auto_gidq,0711);

   235 +#ifdef EXTERNAL_TODO

   236 +  c(auto_qmail,"bin","qmail-todo",auto_uido,auto_gidq,0711);

   237 +#endif

   238    c(auto_qmail,"bin","splogger",auto_uido,auto_gidq,0711);

   239    c(auto_qmail,"bin","qmail-newu",auto_uido,auto_gidq,0700);

   240    c(auto_qmail,"bin","qmail-newmrh",auto_uido,auto_gidq,0700);

   241 diff -uN qmail-1.03/install-big.c qmail-exttodo/install-big.c

   242 --- qmail-1.03/install-big.c	Mon Jun 15 12:53:16 1998

   243 +++ qmail-exttodo/install-big.c	Mon Apr 22 14:02:11 2002

   244 @@ -108,6 +108,9 @@

   245    c(auto_qmail,"bin","qmail-rspawn",auto_uido,auto_gidq,0711);

   246    c(auto_qmail,"bin","qmail-clean",auto_uido,auto_gidq,0711);

   247    c(auto_qmail,"bin","qmail-send",auto_uido,auto_gidq,0711);

   248 +#ifdef EXTERNAL_TODO

   249 +  c(auto_qmail,"bin","qmail-todo",auto_uido,auto_gidq,0711);

   250 +#endif

   251    c(auto_qmail,"bin","splogger",auto_uido,auto_gidq,0711);

   252    c(auto_qmail,"bin","qmail-newu",auto_uido,auto_gidq,0700);

   253    c(auto_qmail,"bin","qmail-newmrh",auto_uido,auto_gidq,0700);

   254 diff -uN qmail-1.03/qmail-send.c qmail-exttodo/qmail-send.c

   255 --- qmail-1.03/qmail-send.c	Mon Jun 15 12:53:16 1998

   256 +++ qmail-exttodo/qmail-send.c	Sun Jan  5 22:09:42 2003

   257 @@ -1215,6 +1215,7 @@

   258  

   259  /* this file is too long ---------------------------------------------- TODO */

   260  

   261 +#ifndef EXTERNAL_TODO

   262  datetime_sec nexttodorun;

   263  DIR *tododir; /* if 0, have to opendir again */

   264  stralloc todoline = {0};

   265 @@ -1438,6 +1439,143 @@

   266     if (fdchan[c] != -1) close(fdchan[c]);

   267  }

   268  

   269 +#endif

   270 +

   271 +/* this file is too long ------------------------------------- EXTERNAL TODO */

   272 +

   273 +#ifdef EXTERNAL_TODO

   274 +stralloc todoline = {0};

   275 +char todobuf[2048];

   276 +int todofdin;

   277 +int todofdout;

   278 +int flagtodoalive;

   279 +

   280 +void tododied() { log1("alert: oh no! lost qmail-todo connection! dying...\n");

   281 + flagexitasap = 1; flagtodoalive = 0; }

   282 +

   283 +void todo_init()

   284 +{

   285 +  todofdout = 7;

   286 +  todofdin = 8;

   287 +  flagtodoalive = 1;

   288 +  /* sync with external todo */

   289 +  if (write(todofdout, "S", 1) != 1) tododied();

   290 +  

   291 +  return;

   292 +}

   293 +

   294 +void todo_selprep(nfds,rfds,wakeup)

   295 +int *nfds;

   296 +fd_set *rfds;

   297 +datetime_sec *wakeup;

   298 +{

   299 +  if (flagexitasap) {

   300 +    if (flagtodoalive) {

   301 +      write(todofdout, "X", 1);

   302 +    }

   303 +  }

   304 +  if (flagtodoalive) {

   305 +    FD_SET(todofdin,rfds);

   306 +    if (*nfds <= todofdin)

   307 +      *nfds = todofdin + 1;

   308 +  }

   309 +}

   310 +

   311 +void todo_del(char* s)

   312 +{

   313 + int flagchan[CHANNELS];

   314 + struct prioq_elt pe;

   315 + unsigned long id;

   316 + unsigned int len;

   317 + int c;

   318 +

   319 + for (c = 0;c < CHANNELS;++c) flagchan[c] = 0;

   320 + switch(*s++) {

   321 +  case 'L':

   322 +    flagchan[0] = 1;

   323 +    break;

   324 +  case 'R':

   325 +    flagchan[1] = 1;

   326 +    break;

   327 +  case 'B':

   328 +    flagchan[0] = 1;

   329 +    flagchan[1] = 1;

   330 +    break;

   331 +  case 'X':

   332 +    break;

   333 +  default:

   334 +    log1("warning: qmail-send unable to understand qmail-todo\n");

   335 +    return;

   336 + }

   337 + 

   338 + len = scan_ulong(s,&id);

   339 + if (!len || s[len]) {

   340 +  log1("warning: qmail-send unable to understand qmail-todo\n");

   341 +  return;

   342 + }

   343 +

   344 + pe.id = id; pe.dt = now();

   345 + for (c = 0;c < CHANNELS;++c)

   346 +   if (flagchan[c])

   347 +     while (!prioq_insert(&pqchan[c],&pe)) nomem();

   348 +

   349 + for (c = 0;c < CHANNELS;++c) if (flagchan[c]) break;

   350 + if (c == CHANNELS)

   351 +   while (!prioq_insert(&pqdone,&pe)) nomem();

   352 +

   353 + return;

   354 +}

   355 +

   356 +void todo_do(rfds)

   357 +fd_set *rfds;

   358 +{

   359 +  int r;

   360 +  char ch;

   361 +  int i;

   362 +  

   363 +  if (!flagtodoalive) return;

   364 +  if (!FD_ISSET(todofdin,rfds)) return;

   365 +

   366 +  r = read(todofdin,todobuf,sizeof(todobuf));

   367 +  if (r == -1) return;

   368 +  if (r == 0) {

   369 +    if (flagexitasap)

   370 +      flagtodoalive = 0;

   371 +    else

   372 +      tododied();

   373 +    return;

   374 +  }

   375 +  for (i = 0;i < r;++i) {

   376 +    ch = todobuf[i];

   377 +    while (!stralloc_append(&todoline,&ch)) nomem();

   378 +    if (todoline.len > REPORTMAX)

   379 +      todoline.len = REPORTMAX;

   380 +      /* qmail-todo is responsible for keeping it short */

   381 +    if (!ch && (todoline.len > 1)) {

   382 +      switch (todoline.s[0]) {

   383 +	case 'D':

   384 +	  if (flagexitasap) break;

   385 +	  todo_del(todoline.s + 1);

   386 +	  break;

   387 +	case 'L':

   388 +	  log1(todoline.s + 1);

   389 +	  break;

   390 +	case 'X':

   391 +	  if (flagexitasap)

   392 +	    flagtodoalive = 0;

   393 +	  else

   394 +	    tododied();

   395 +	  break;

   396 +	default:

   397 +	  log1("warning: qmail-send unable to understand qmail-todo: report mangled\n");

   398 +	  break;

   399 +      }

   400 +      todoline.len = 0;

   401 +    }

   402 +  }

   403 +}

   404 +

   405 +#endif

   406  

   407  /* this file is too long ---------------------------------------------- MAIN */

   408  

   409 @@ -1504,6 +1642,9 @@

   410     log1("alert: unable to reread controls: unable to switch to home directory\n");

   411     return;

   412    }

   413 +#ifdef EXTERNAL_TODO

   414 + write(todofdout, "H", 1);

   415 +#endif

   416   regetcontrols();

   417   while (chdir("queue") == -1)

   418    {

   419 @@ -1568,8 +1709,12 @@

   420   todo_init();

   421   cleanup_init();

   422  

   423 +#ifdef EXTERNAL_TODO

   424 + while (!flagexitasap || !del_canexit() || flagtodoalive)

   425 +#else

   426   while (!flagexitasap || !del_canexit())

   427 -  {

   428 +#endif

   429 + {

   430     recent = now();

   431  

   432     if (flagrunasap) { flagrunasap = 0; pqrun(); }

   433 diff -uN qmail-1.03/qmail-start.c qmail-exttodo/qmail-start.c

   434 --- qmail-1.03/qmail-start.c	Mon Jun 15 12:53:16 1998

   435 +++ qmail-exttodo/qmail-start.c	Mon Apr 22 13:55:48 2002

   436 @@ -8,6 +8,9 @@

   437  char *(qcargs[]) = { "qmail-clean", 0 };

   438  char *(qlargs[]) = { "qmail-lspawn", "./Mailbox", 0 };

   439  char *(qrargs[]) = { "qmail-rspawn", 0 };

   440 +#ifdef EXTERNAL_TODO

   441 +char *(qtargs[]) = { "qmail-todo", 0};

   442 +#endif

   443  

   444  void die() { _exit(111); }

   445  

   446 @@ -18,13 +21,28 @@

   447  int pi4[2];

   448  int pi5[2];

   449  int pi6[2];

   450 -

   451 -void close23456() { close(2); close(3); close(4); close(5); close(6); }

   452 +#ifdef EXTERNAL_TODO

   453 +int pi7[2];

   454 +int pi8[2];

   455 +int pi9[2];

   456 +int pi10[2];

   457 +#endif

   458 +

   459 +void close23456() { 

   460 +  close(2); close(3); close(4); close(5); close(6); 

   461 +#ifdef EXTERNAL_TODO

   462 +  close(7); close(8);

   463 +#endif

   464 +}

   465  

   466  void closepipes() {

   467    close(pi1[0]); close(pi1[1]); close(pi2[0]); close(pi2[1]);

   468    close(pi3[0]); close(pi3[1]); close(pi4[0]); close(pi4[1]);

   469    close(pi5[0]); close(pi5[1]); close(pi6[0]); close(pi6[1]);

   470 +#ifdef EXTERNAL_TODO

   471 +  close(pi7[0]); close(pi7[1]); close(pi8[0]); close(pi8[1]);

   472 +	close(pi9[0]); close(pi9[1]); close(pi10[0]); close(pi10[1]);

   473 +#endif

   474  }

   475  

   476  void main(argc,argv)

   477 @@ -40,6 +58,10 @@

   478    if (fd_copy(4,0) == -1) die();

   479    if (fd_copy(5,0) == -1) die();

   480    if (fd_copy(6,0) == -1) die();

   481 +#ifdef EXTERNAL_TODO

   482 +  if (fd_copy(7,0) == -1) die();

   483 +  if (fd_copy(8,0) == -1) die();

   484 +#endif

   485  

   486    if (argv[1]) {

   487      qlargs[1] = argv[1];

   488 @@ -70,6 +92,12 @@

   489    if (pipe(pi4) == -1) die();

   490    if (pipe(pi5) == -1) die();

   491    if (pipe(pi6) == -1) die();

   492 +#ifdef EXTERNAL_TODO

   493 +  if (pipe(pi7) == -1) die();

   494 +  if (pipe(pi8) == -1) die();

   495 +  if (pipe(pi9) == -1) die();

   496 +  if (pipe(pi10) == -1) die();

   497 +#endif

   498   

   499    switch(fork()) {

   500      case -1: die();

   501 @@ -105,6 +133,34 @@

   502        execvp(*qcargs,qcargs);

   503        die();

   504    }

   505 +

   506 +#ifdef EXTERNAL_TODO

   507 +  switch(fork()) {

   508 +    case -1: die();

   509 +    case 0:

   510 +      if (prot_uid(auto_uids) == -1) die();

   511 +      if (fd_copy(0,pi7[0]) == -1) die();

   512 +      if (fd_copy(1,pi8[1]) == -1) die();

   513 +      close23456();

   514 +      if (fd_copy(2,pi9[1]) == -1) die();

   515 +      if (fd_copy(3,pi10[0]) == -1) die();

   516 +      closepipes();

   517 +      execvp(*qtargs,qtargs);

   518 +      die();

   519 +  }

   520 +

   521 +  switch(fork()) {

   522 +    case -1: die();

   523 +    case 0:

   524 +      if (prot_uid(auto_uidq) == -1) die();

   525 +      if (fd_copy(0,pi9[0]) == -1) die();

   526 +      if (fd_copy(1,pi10[1]) == -1) die();

   527 +      close23456();

   528 +      closepipes();

   529 +      execvp(*qcargs,qcargs);

   530 +      die();

   531 +  }

   532 +#endif

   533   

   534    if (prot_uid(auto_uids) == -1) die();

   535    if (fd_copy(0,1) == -1) die();

   536 @@ -114,6 +170,10 @@

   537    if (fd_copy(4,pi4[0]) == -1) die();

   538    if (fd_copy(5,pi5[1]) == -1) die();

   539    if (fd_copy(6,pi6[0]) == -1) die();

   540 +#ifdef EXTERNAL_TODO

   541 +  if (fd_copy(7,pi7[1]) == -1) die();

   542 +  if (fd_copy(8,pi8[0]) == -1) die();

   543 +#endif

   544    closepipes();

   545    execvp(*qsargs,qsargs);

   546    die();

   547 diff -uN qmail-1.03/qmail-todo.c qmail-exttodo/qmail-todo.c

   548 --- qmail-1.03/qmail-todo.c	Thu Jan  1 01:00:00 1970

   549 +++ qmail-exttodo/qmail-todo.c	Sun Jan  5 22:16:34 2003

   550 @@ -0,0 +1,688 @@

   551 +#include <sys/types.h>

   552 +#include <sys/stat.h>

   553 +#include "alloc.h"

   554 +#include "auto_qmail.h"

   555 +#include "byte.h"

   556 +#include "constmap.h"

   557 +#include "control.h"

   558 +#include "direntry.h"

   559 +#include "error.h"

   560 +#include "exit.h"

   561 +#include "fmt.h"

   562 +#include "fmtqfn.h"

   563 +#include "getln.h"

   564 +#include "open.h"

   565 +#include "ndelay.h"

   566 +#include "now.h"

   567 +#include "readsubdir.h"

   568 +#include "readwrite.h"

   569 +#include "scan.h"

   570 +#include "select.h"

   571 +#include "str.h"

   572 +#include "stralloc.h"

   573 +#include "substdio.h"

   574 +#include "trigger.h"

   575 +

   576 +/* critical timing feature #1: if not triggered, do not busy-loop */

   577 +/* critical timing feature #2: if triggered, respond within fixed time */

   578 +/* important timing feature: when triggered, respond instantly */

   579 +#define SLEEP_TODO 1500 /* check todo/ every 25 minutes in any case */

   580 +#define SLEEP_FUZZ 1 /* slop a bit on sleeps to avoid zeno effect */

   581 +#define SLEEP_FOREVER 86400 /* absolute maximum time spent in select() */

   582 +#define SLEEP_SYSFAIL 123

   583 +

   584 +stralloc percenthack = {0};

   585 +struct constmap mappercenthack;

   586 +stralloc locals = {0};

   587 +struct constmap maplocals;

   588 +stralloc vdoms = {0};

   589 +struct constmap mapvdoms;

   590 +stralloc envnoathost = {0};

   591 +

   592 +char strnum[FMT_ULONG];

   593 +

   594 +/* XXX not good, if qmail-send.c changes this has to be updated */

   595 +#define CHANNELS 2

   596 +char *chanaddr[CHANNELS] = { "local/", "remote/" };

   597 +

   598 +datetime_sec recent;

   599 +

   600 +void log1(char *x);

   601 +void log3(char* x, char* y, char* z);

   602 +

   603 +int flagstopasap = 0;

   604 +void sigterm(void)

   605 +{

   606 +  if (flagstopasap == 0)

   607 +    log1("status: qmail-todo stop processing asap\n");

   608 +  flagstopasap = 1;

   609 +}

   610 +

   611 +int flagreadasap = 0; void sighup(void) { flagreadasap = 1; }

   612 +int flagsendalive = 1; void senddied(void) { flagsendalive = 0; }

   613 +

   614 +void nomem() { log1("alert: out of memory, sleeping...\n"); sleep(10); }

   615 +void pausedir(dir) char *dir;

   616 +{ log3("alert: unable to opendir ",dir,", sleeping...\n"); sleep(10); }

   617 +

   618 +void cleandied()

   619 +{ 

   620 +  log1("alert: qmail-todo: oh no! lost qmail-clean connection! dying...\n");

   621 +  flagstopasap = 1;

   622 +}

   623 +

   624 +

   625 +/* this file is not so long ------------------------------------- FILENAMES */

   626 +

   627 +stralloc fn = {0};

   628 +

   629 +void fnmake_init(void)

   630 +{

   631 + while (!stralloc_ready(&fn,FMTQFN)) nomem();

   632 +}

   633 +

   634 +void fnmake_info(unsigned long id) { fn.len = fmtqfn(fn.s,"info/",id,1); }

   635 +void fnmake_todo(unsigned long id) { fn.len = fmtqfn(fn.s,"todo/",id,0); }

   636 +void fnmake_mess(unsigned long id) { fn.len = fmtqfn(fn.s,"mess/",id,1); }

   637 +void fnmake_chanaddr(unsigned long id, int c)

   638 +{ fn.len = fmtqfn(fn.s,chanaddr[c],id,1); }

   639 +

   640 +

   641 +/* this file is not so long ------------------------------------- REWRITING */

   642 +

   643 +stralloc rwline = {0};

   644 +

   645 +/* 1 if by land, 2 if by sea, 0 if out of memory. not allowed to barf. */

   646 +/* may trash recip. must set up rwline, between a T and a \0. */

   647 +int rewrite(char *recip)

   648 +{

   649 +  int i;

   650 +  int j;

   651 +  char *x;

   652 +  static stralloc addr = {0};

   653 +  int at;

   654 +

   655 +  if (!stralloc_copys(&rwline,"T")) return 0;

   656 +  if (!stralloc_copys(&addr,recip)) return 0;

   657 +

   658 +  i = byte_rchr(addr.s,addr.len,'@');

   659 +  if (i == addr.len) {

   660 +    if (!stralloc_cats(&addr,"@")) return 0;

   661 +    if (!stralloc_cat(&addr,&envnoathost)) return 0;

   662 +  }

   663 +

   664 +  while (constmap(&mappercenthack,addr.s + i + 1,addr.len - i - 1)) {

   665 +    j = byte_rchr(addr.s,i,'%');

   666 +    if (j == i) break;

   667 +    addr.len = i;

   668 +    i = j;

   669 +    addr.s[i] = '@';

   670 +  }

   671 +

   672 +  at = byte_rchr(addr.s,addr.len,'@');

   673 +

   674 +  if (constmap(&maplocals,addr.s + at + 1,addr.len - at - 1)) {

   675 +    if (!stralloc_cat(&rwline,&addr)) return 0;

   676 +    if (!stralloc_0(&rwline)) return 0;

   677 +    return 1;

   678 +  }

   679 +

   680 +  for (i = 0;i <= addr.len;++i)

   681 +    if (!i || (i == at + 1) || (i == addr.len) || ((i > at) && (addr.s[i] == '.')))

   682 +      if (x = constmap(&mapvdoms,addr.s + i,addr.len - i)) {

   683 +        if (!*x) break;

   684 +        if (!stralloc_cats(&rwline,x)) return 0;

   685 +        if (!stralloc_cats(&rwline,"-")) return 0;

   686 +        if (!stralloc_cat(&rwline,&addr)) return 0;

   687 +        if (!stralloc_0(&rwline)) return 0;

   688 +        return 1;

   689 +      }

   690 + 

   691 +  if (!stralloc_cat(&rwline,&addr)) return 0;

   692 +  if (!stralloc_0(&rwline)) return 0;

   693 +  return 2;

   694 +}

   695 +

   696 +/* this file is not so long --------------------------------- COMMUNICATION */

   697 +

   698 +substdio sstoqc; char sstoqcbuf[1024];

   699 +substdio ssfromqc; char ssfromqcbuf[1024];

   700 +stralloc comm_buf = {0};

   701 +int comm_pos;

   702 +int fdout = -1;

   703 +int fdin = -1;

   704 +

   705 +void comm_init(void)

   706 +{

   707 + substdio_fdbuf(&sstoqc,write,2,sstoqcbuf,sizeof(sstoqcbuf));

   708 + substdio_fdbuf(&ssfromqc,read,3,ssfromqcbuf,sizeof(ssfromqcbuf));

   709 +

   710 + fdout = 1; /* stdout */

   711 + fdin = 0;  /* stdin */

   712 + if (ndelay_on(fdout) == -1)

   713 + /* this is so stupid: NDELAY semantics should be default on write */

   714 +   senddied(); /* drastic, but better than risking deadlock */

   715 +

   716 + while (!stralloc_ready(&comm_buf,1024)) nomem();

   717 +}

   718 +

   719 +int comm_canwrite(void)

   720 +{

   721 + /* XXX: could allow a bigger buffer; say 10 recipients */

   722 + /* XXX: returns true if there is something in the buffer */

   723 + if (!flagsendalive) return 0;

   724 + if (comm_buf.s && comm_buf.len) return 1;

   725 + return 0;

   726 +}

   727 +

   728 +void log1(char* x)

   729 +{

   730 +  int pos;

   731 +  

   732 +  pos = comm_buf.len;

   733 +  if (!stralloc_cats(&comm_buf,"L")) goto fail;

   734 +  if (!stralloc_cats(&comm_buf,x)) goto fail;

   735 +  if (!stralloc_0(&comm_buf)) goto fail;

   736 +  return;

   737 +  

   738 +fail:

   739 +  /* either all or nothing */

   740 +  comm_buf.len = pos;

   741 +}

   742 +

   743 +void log3(char* x, char *y, char *z)

   744 +{

   745 +  int pos;

   746 +  

   747 +  pos = comm_buf.len;

   748 +  if (!stralloc_cats(&comm_buf,"L")) goto fail;

   749 +  if (!stralloc_cats(&comm_buf,x)) goto fail;

   750 +  if (!stralloc_cats(&comm_buf,y)) goto fail;

   751 +  if (!stralloc_cats(&comm_buf,z)) goto fail;

   752 +  if (!stralloc_0(&comm_buf)) goto fail;

   753 +  return;

   754 +  

   755 +fail:

   756 +  /* either all or nothing */

   757 +  comm_buf.len = pos;

   758 +}

   759 +

   760 +void comm_write(unsigned long id, int local, int remote)

   761 +{

   762 +  int pos;

   763 +  char *s;

   764 +  

   765 +  if(local && remote) s="B";

   766 +  else if(local) s="L";

   767 +  else if(remote) s="R";

   768 +  else s="X";

   769 +  

   770 +  pos = comm_buf.len;

   771 +  strnum[fmt_ulong(strnum,id)] = 0;

   772 +  if (!stralloc_cats(&comm_buf,"D")) goto fail;

   773 +  if (!stralloc_cats(&comm_buf,s)) goto fail;

   774 +  if (!stralloc_cats(&comm_buf,strnum)) goto fail;

   775 +  if (!stralloc_0(&comm_buf)) goto fail;

   776 +  return;

   777 +  

   778 +fail:

   779 +  /* either all or nothing */

   780 +  comm_buf.len = pos;

   781 +}

   782 +

   783 +static int issafe(char ch)

   784 +{

   785 + if (ch == '%') return 0; /* general principle: allman's code is crap */

   786 + if (ch < 33) return 0;

   787 + if (ch > 126) return 0;

   788 + return 1;

   789 +}

   790 +

   791 +void comm_info(unsigned long id, unsigned long size, char* from, unsigned long pid, unsigned long uid)

   792 +{

   793 +  int pos;

   794 +  int i;

   795 +  

   796 +  pos = comm_buf.len;

   797 +  if (!stralloc_cats(&comm_buf,"Linfo msg ")) goto fail;

   798 +  strnum[fmt_ulong(strnum,id)] = 0;

   799 +  if (!stralloc_cats(&comm_buf,strnum)) goto fail;

   800 +  if (!stralloc_cats(&comm_buf,": bytes ")) goto fail;

   801 +  strnum[fmt_ulong(strnum,size)] = 0;

   802 +  if (!stralloc_cats(&comm_buf,strnum)) goto fail;

   803 +  if (!stralloc_cats(&comm_buf," from <")) goto fail;

   804 +  i = comm_buf.len;

   805 +  if (!stralloc_cats(&comm_buf,from)) goto fail;

   806 +  for (;i < comm_buf.len;++i)

   807 +    if (comm_buf.s[i] == '\n')

   808 +      comm_buf.s[i] = '/';

   809 +    else

   810 +      if (!issafe(comm_buf.s[i]))

   811 +	comm_buf.s[i] = '_';

   812 +  if (!stralloc_cats(&comm_buf,"> qp ")) goto fail;

   813 +  strnum[fmt_ulong(strnum,pid)] = 0;

   814 +  if (!stralloc_cats(&comm_buf,strnum)) goto fail;

   815 +  if (!stralloc_cats(&comm_buf," uid ")) goto fail;

   816 +  strnum[fmt_ulong(strnum,uid)] = 0;

   817 +  if (!stralloc_cats(&comm_buf,strnum)) goto fail;

   818 +  if (!stralloc_cats(&comm_buf,"\n")) goto fail;

   819 +  if (!stralloc_0(&comm_buf)) goto fail;

   820 +  return;

   821 +  

   822 +fail:

   823 +  /* either all or nothing */

   824 +  comm_buf.len = pos;

   825 +}

   826 +

   827 +void comm_exit(void)

   828 +{

   829 +  int w;

   830 +  

   831 +  /* if it fails exit, we have already stoped */

   832 +  if (!stralloc_cats(&comm_buf,"X")) _exit(1);

   833 +  if (!stralloc_0(&comm_buf)) _exit(1);

   834 +}

   835 +

   836 +void comm_selprep(int *nfds, fd_set *wfds, fd_set *rfds)

   837 +{

   838 +  if (flagsendalive) {

   839 +    if (flagstopasap && comm_canwrite() == 0)

   840 +      comm_exit();

   841 +    if (comm_canwrite()) {

   842 +      FD_SET(fdout,wfds);

   843 +      if (*nfds <= fdout)

   844 +	*nfds = fdout + 1;

   845 +    }

   846 +    FD_SET(fdin,rfds);

   847 +    if (*nfds <= fdin)

   848 +      *nfds = fdin + 1;

   849 +  }

   850 +}

   851 +

   852 +void comm_do(fd_set *wfds, fd_set *rfds)

   853 +{

   854 +  /* first write then read */

   855 +  if (flagsendalive)

   856 +    if (comm_canwrite())

   857 +      if (FD_ISSET(fdout,wfds)) {

   858 +	int w;

   859 +	int len;

   860 +	len = comm_buf.len;

   861 +	w = write(fdout,comm_buf.s + comm_pos,len - comm_pos);

   862 +	if (w <= 0) {

   863 +	  if ((w == -1) && (errno == error_pipe))

   864 +	    senddied();

   865 +	} else {

   866 +	  comm_pos += w;

   867 +	  if (comm_pos == len) {

   868 +	    comm_buf.len = 0;

   869 +	    comm_pos = 0;

   870 +	  }

   871 +	}

   872 +      }

   873 +  if (flagsendalive)

   874 +    if (FD_ISSET(fdin,rfds)) {

   875 +      /* there are only two messages 'H' and 'X' */

   876 +      char c;

   877 +      int r;

   878 +      r = read(fdin, &c, 1);

   879 +      if (r <= 0) {

   880 +	if ((r == -1) && (errno != error_intr))

   881 +	  senddied();

   882 +      } else {

   883 +	switch(c) {

   884 +	  case 'H':

   885 +	    sighup();

   886 +	    break;

   887 +	  case 'X':

   888 +	    sigterm();

   889 +	    break;

   890 +	  default:

   891 +	    log1("warning: qmail-todo: qmail-send speaks an obscure dialect\n");

   892 +	    break;

   893 +	}

   894 +      }

   895 +    }

   896 +}

   897 +

   898 +/* this file is not so long ------------------------------------------ TODO */

   899 +

   900 +datetime_sec nexttodorun;

   901 +DIR *tododir; /* if 0, have to opendir again */

   902 +stralloc todoline = {0};

   903 +char todobuf[SUBSTDIO_INSIZE];

   904 +char todobufinfo[512];

   905 +char todobufchan[CHANNELS][1024];

   906 +

   907 +void todo_init(void)

   908 +{

   909 + tododir = 0;

   910 + nexttodorun = now();

   911 + trigger_set();

   912 +}

   913 +

   914 +void todo_selprep(int *nfds, fd_set *rfds, datetime_sec *wakeup)

   915 +{

   916 + if (flagstopasap) return;

   917 + trigger_selprep(nfds,rfds);

   918 + if (tododir) *wakeup = 0;

   919 + if (*wakeup > nexttodorun) *wakeup = nexttodorun;

   920 +}

   921 +

   922 +void todo_do(fd_set *rfds)

   923 +{

   924 + struct stat st;

   925 + substdio ss; int fd;

   926 + substdio ssinfo; int fdinfo;

   927 + substdio sschan[CHANNELS];

   928 + int fdchan[CHANNELS];

   929 + int flagchan[CHANNELS];

   930 + char ch;

   931 + int match;

   932 + unsigned long id;

   933 + unsigned int len;

   934 + direntry *d;

   935 + int c;

   936 + unsigned long uid;

   937 + unsigned long pid;

   938 +

   939 + fd = -1;

   940 + fdinfo = -1;

   941 + for (c = 0;c < CHANNELS;++c) fdchan[c] = -1;

   942 +

   943 + if (flagstopasap) return;

   944 +

   945 + if (!tododir)

   946 +  {

   947 +   if (!trigger_pulled(rfds))

   948 +     if (recent < nexttodorun)

   949 +       return;

   950 +   trigger_set();

   951 +   tododir = opendir("todo");

   952 +   if (!tododir)

   953 +    {

   954 +     pausedir("todo");

   955 +     return;

   956 +    }

   957 +   nexttodorun = recent + SLEEP_TODO;

   958 +  }

   959 +

   960 + d = readdir(tododir);

   961 + if (!d)

   962 +  {

   963 +   closedir(tododir);

   964 +   tododir = 0;

   965 +   return;

   966 +  }

   967 + if (str_equal(d->d_name,".")) return;

   968 + if (str_equal(d->d_name,"..")) return;

   969 + len = scan_ulong(d->d_name,&id);

   970 + if (!len || d->d_name[len]) return;

   971 +

   972 + fnmake_todo(id);

   973 +

   974 + fd = open_read(fn.s);

   975 + if (fd == -1) { log3("warning: qmail-todo: unable to open ",fn.s,"\n"); return; }

   976 +

   977 + fnmake_mess(id);

   978 + /* just for the statistics */

   979 + if (stat(fn.s,&st) == -1)

   980 +  { log3("warning: qmail-todo: unable to stat ",fn.s,"\n"); goto fail; }

   981 +

   982 + for (c = 0;c < CHANNELS;++c)

   983 +  {

   984 +   fnmake_chanaddr(id,c);

   985 +   if (unlink(fn.s) == -1) if (errno != error_noent)

   986 +    { log3("warning: qmail-todo: unable to unlink ",fn.s,"\n"); goto fail; }

   987 +  }

   988 +

   989 + fnmake_info(id);

   990 + if (unlink(fn.s) == -1) if (errno != error_noent)

   991 +  { log3("warning: qmail-todo: unable to unlink ",fn.s,"\n"); goto fail; }

   992 +

   993 + fdinfo = open_excl(fn.s);

   994 + if (fdinfo == -1)

   995 +  { log3("warning: qmail-todo: unable to create ",fn.s,"\n"); goto fail; }

   996 +

   997 + strnum[fmt_ulong(strnum,id)] = 0;

   998 + log3("new msg ",strnum,"\n");

   999 +

  1000 + for (c = 0;c < CHANNELS;++c) flagchan[c] = 0;

  1001 +

  1002 + substdio_fdbuf(&ss,read,fd,todobuf,sizeof(todobuf));

  1003 + substdio_fdbuf(&ssinfo,write,fdinfo,todobufinfo,sizeof(todobufinfo));

  1004 +

  1005 + uid = 0;

  1006 + pid = 0;

  1007 +

  1008 + for (;;)

  1009 +  {

  1010 +   if (getln(&ss,&todoline,&match,'\0') == -1)

  1011 +    {

  1012 +     /* perhaps we're out of memory, perhaps an I/O error */

  1013 +     fnmake_todo(id);

  1014 +     log3("warning: qmail-todo: trouble reading ",fn.s,"\n"); goto fail;

  1015 +    }

  1016 +   if (!match) break;

  1017 +

  1018 +   switch(todoline.s[0])

  1019 +    {

  1020 +     case 'u':

  1021 +       scan_ulong(todoline.s + 1,&uid);

  1022 +       break;

  1023 +     case 'p':

  1024 +       scan_ulong(todoline.s + 1,&pid);

  1025 +       break;

  1026 +     case 'F':

  1027 +       if (substdio_putflush(&ssinfo,todoline.s,todoline.len) == -1)

  1028 +	{

  1029 +	 fnmake_info(id);

  1030 +         log3("warning: qmail-todo: trouble writing to ",fn.s,"\n"); goto fail;

  1031 +	}

  1032 +	comm_info(id, (unsigned long) st.st_size, todoline.s + 1, pid, uid);

  1033 +       break;

  1034 +     case 'T':

  1035 +       switch(rewrite(todoline.s + 1))

  1036 +	{

  1037 +	 case 0: nomem(); goto fail;

  1038 +	 case 2: c = 1; break;

  1039 +	 default: c = 0; break;

  1040 +        }

  1041 +       if (fdchan[c] == -1)

  1042 +	{

  1043 +	 fnmake_chanaddr(id,c);

  1044 +	 fdchan[c] = open_excl(fn.s);

  1045 +	 if (fdchan[c] == -1)

  1046 +          { log3("warning: qmail-todo: unable to create ",fn.s,"\n"); goto fail; }

  1047 +	 substdio_fdbuf(&sschan[c]

  1048 +	   ,write,fdchan[c],todobufchan[c],sizeof(todobufchan[c]));

  1049 +	 flagchan[c] = 1;

  1050 +	}

  1051 +       if (substdio_bput(&sschan[c],rwline.s,rwline.len) == -1)

  1052 +        {

  1053 +	 fnmake_chanaddr(id,c);

  1054 +         log3("warning: qmail-todo: trouble writing to ",fn.s,"\n"); goto fail;

  1055 +        }

  1056 +       break;

  1057 +     default:

  1058 +       fnmake_todo(id);

  1059 +       log3("warning: qmail-todo: unknown record type in ",fn.s,"\n"); goto fail;

  1060 +    }

  1061 +  }

  1062 +

  1063 + close(fd); fd = -1;

  1064 +

  1065 + fnmake_info(id);

  1066 + if (substdio_flush(&ssinfo) == -1)

  1067 +  { log3("warning: qmail-todo: trouble writing to ",fn.s,"\n"); goto fail; }

  1068 + if (fsync(fdinfo) == -1)

  1069 +  { log3("warning: qmail-todo: trouble fsyncing ",fn.s,"\n"); goto fail; }

  1070 + close(fdinfo); fdinfo = -1;

  1071 +

  1072 + for (c = 0;c < CHANNELS;++c)

  1073 +   if (fdchan[c] != -1)

  1074 +    {

  1075 +     fnmake_chanaddr(id,c);

  1076 +     if (substdio_flush(&sschan[c]) == -1)

  1077 +      { log3("warning: qmail-todo: trouble writing to ",fn.s,"\n"); goto fail; }

  1078 +     if (fsync(fdchan[c]) == -1)

  1079 +      { log3("warning: qmail-todo: trouble fsyncing ",fn.s,"\n"); goto fail; }

  1080 +     close(fdchan[c]); fdchan[c] = -1;

  1081 +    }

  1082 +

  1083 + fnmake_todo(id);

  1084 + if (substdio_putflush(&sstoqc,fn.s,fn.len) == -1) { cleandied(); return; }

  1085 + if (substdio_get(&ssfromqc,&ch,1) != 1) { cleandied(); return; }

  1086 + if (ch != '+')

  1087 +  {

  1088 +   log3("warning: qmail-clean unable to clean up ",fn.s,"\n");

  1089 +   return;

  1090 +  }

  1091 +

  1092 + comm_write(id, flagchan[0], flagchan[1]);

  1093 + 

  1094 + return;

  1095 + 

  1096 + fail:

  1097 + if (fd != -1) close(fd);

  1098 + if (fdinfo != -1) close(fdinfo);

  1099 + for (c = 0;c < CHANNELS;++c)

  1100 +   if (fdchan[c] != -1) close(fdchan[c]);

  1101 +}

  1102 +

  1103 +/* this file is too long ---------------------------------------------- MAIN */

  1104 +

  1105 +int getcontrols(void)

  1106 +{

  1107 + if (control_init() == -1) return 0;

  1108 + if (control_rldef(&envnoathost,"control/envnoathost",1,"envnoathost") != 1) return 0;

  1109 + if (control_readfile(&locals,"control/locals",1) != 1) return 0;

  1110 + if (!constmap_init(&maplocals,locals.s,locals.len,0)) return 0;

  1111 + switch(control_readfile(&percenthack,"control/percenthack",0))

  1112 +  {

  1113 +   case -1: return 0;

  1114 +   case 0: if (!constmap_init(&mappercenthack,"",0,0)) return 0; break;

  1115 +   case 1: if (!constmap_init(&mappercenthack,percenthack.s,percenthack.len,0)) return 0; break;

  1116 +  }

  1117 + switch(control_readfile(&vdoms,"control/virtualdomains",0))

  1118 +  {

  1119 +   case -1: return 0;

  1120 +   case 0: if (!constmap_init(&mapvdoms,"",0,1)) return 0; break;

  1121 +   case 1: if (!constmap_init(&mapvdoms,vdoms.s,vdoms.len,1)) return 0; break;

  1122 +  }

  1123 + return 1;

  1124 +}

  1125 +

  1126 +stralloc newlocals = {0};

  1127 +stralloc newvdoms = {0};

  1128 +

  1129 +void regetcontrols(void)

  1130 +{

  1131 + int r;

  1132 +

  1133 + if (control_readfile(&newlocals,"control/locals",1) != 1)

  1134 +  { log1("alert: qmail-todo: unable to reread control/locals\n"); return; }

  1135 + r = control_readfile(&newvdoms,"control/virtualdomains",0);

  1136 + if (r == -1)