diff -uN qmail-1.03/EXTTODO qmail-exttodo/EXTTODO

--- qmail-1.03/EXTTODO	Thu Jan  1 01:00:00 1970

+++ qmail-exttodo/EXTTODO	Sun Jan  5 22:12:01 2003

@@ -0,0 +1,114 @@

+EXTTODO by Claudio Jeker <jeker@n-r-g.com> and 

+Andre Oppermann <opi@nrg4u.com>

+(c) 1998,1999,2000,2001,2002 Internet Business Solutions Ltd.

+

+The EXTTODO patch is a part of the qmail-ldap patch.

+This patches for qmail come with NO WARRANTY.

+

+These patches are under the BSD license.

+

+RELEASE: 5. Jan. 2003

+

+EXTTODO:

+======================

+

+TOC:

+ WHAT DOES IT DO

+ INSTALL

+ CONFIG FILES

+ SETUP

+ BIG PICTURE

+

+NEWS:

+ 

+ This is the first release of the EXTTODO patch.

+

+================================================================================

+

+WHAT DOES IT DO

+

+ The exttodo patch addresses a problem known as the silly qmail (queue)

+ problem. This problem is found only on system with high injection rates.

+

+ qmail with a big local and remote concurrency could deliver a tremendous 

+ amount of messages but normally this can not be achieved because qmail-send

+ becomes a bottleneck on those high volumes servers.

+ qmail-send preprocesses all new messages before distributing them for local

+ or remote delivering. In one run qmail-send does one todo run but has the 

+ ability to close multiple jobs. Because of this layout qmail-send can not 

+ feed all the new available (local/remote) delivery slots and therefor it is 

+ not possible to achieve the maximum throughput.

+ This would be a minor problem if one qmail-send run could be done in extreme

+ short time but because of many file system calls (fsync and (un)link) a todo

+ run is expensive and throttles the throughput.

+

+ The exttodo patch tries to solve the problem by moving the todo routine into 

+ an external program. This reduces the run time in qmail-send.

+

+ exttodo adds a new program to qmail called qmail-todo. qmail-todo prepares

+ incoming messages for local and remote delivering (by creating info/<messid>

+ local/<messid> and remote/<messid> and removing todo/<messid>). See also

+ INTERNALS. As next qmail-todo transmits the <messid> to qmail-send which will

+ add this message into the priority queue which schedules the message for 

+ delivery. 

+

+INSTALL

+

+ To enable the exttodo patch you need to define EXTERNAL_TODO while compiling

+ qmail(-ldap) this can be done with the -D flag of cc (e.g. cc -DEXTERNAL_TODO).

+

+ NOTE: the exttodo patch can also be used on qmail systems without the 

+ qmail-ldap patch.

+ 

+================================================================================

+

+CONFIG FILES

+

+ No additional control files are used or needed.

+

+================================================================================

+

+SETUP

+

+ qmail-todo will be started by qmail-start and therefor no additional setup

+ is needed.

+

+ To verify that exttodo is running just check if qmail-todo is running.

+

+================================================================================

+

+BIG PICTURE

+

+               +-------+   +-------+

+               | clean |   | clean |

+               +--0-1--+   +--0-1--+       +-----------+

+         trigger  ^ |         ^ |        +->0,1 lspawn |

+            |     | v         | v       /  +-----------+

+ +-------+  v  +--2-3--+   +--5-6--+   /

+ |       |  |  |       0<--7     1,2<-+

+ | queue |--+--| todo  |   | send  |

+ |       |  |  |       1-->8     3,4<-+

+ +-------+     +-------+   +---0---+   \

+                               |        \  +-----------+

+                               v         +->0,1 rspwan |

+                           +---0---+       +-----------+

+                           | logger|

+                           +-------+

+

+Communication between qmail-send and qmail-todo

+

+todo -> send:

+   D[LRB]<mesgid>\0

+          Start delivery for new message with id <messid>.

+          the character L, R or B defines the type

+          of delivery, local, remote or both respectively.

+   L<string>\0

+          Dump string to the logger without adding additional \n or similar.

+send -> todo:

+   H      Got a SIGHUP reread ~/control/locals and ~/control/virtualdomains

+   X      Quit ASAP.

+

+qmail-todo sends "\0" terminated messages whereas qmail-send just send one

+character to qmail-todo.

+

+

diff -uN qmail-1.03/EXTTODO-INFO qmail-exttodo/EXTTODO-INFO

--- qmail-1.03/EXTTODO-INFO	Thu Jan  1 01:00:00 1970

+++ qmail-exttodo/EXTTODO-INFO	Tue Apr 30 16:49:02 2002

@@ -0,0 +1,11 @@

+Files modified:

+Makefile

+EXTTODO

+FILES

+TARGETS

+qmail-send.c

+qmail-todo.c

+qmail-start.c

+hier.c

+install-big.c

+

diff -uN qmail-1.03/FILES qmail-exttodo/FILES

--- qmail-1.03/FILES	Mon Jun 15 12:53:16 1998

+++ qmail-exttodo/FILES	Mon Apr 22 13:59:28 2002

@@ -431,3 +431,4 @@

 tcp-environ.5

 constmap.h

 constmap.c

+qmail-todo.c

diff -uN qmail-1.03/Makefile qmail-exttodo/Makefile

--- qmail-1.03/Makefile	Mon Jun 15 12:53:16 1998

+++ qmail-exttodo/Makefile	Mon Apr 22 14:55:59 2002

@@ -1,5 +1,7 @@

 # Don't edit Makefile! Use conf-* for configuration.

+DEFINES=-DEXTERNAL_TODO # use to enable external todo

+

 SHELL=/bin/sh

 default: it

@@ -703,7 +705,7 @@

 hier.o: \

 compile hier.c auto_qmail.h auto_split.h auto_uids.h fmt.h fifo.h

-	./compile hier.c

+	./compile $(DEFINES) hier.c

 home: \

 home.sh conf-qmail

@@ -755,7 +757,7 @@

 install-big.o: \

 compile install-big.c auto_qmail.h auto_split.h auto_uids.h fmt.h \

 fifo.h

-	./compile install-big.c

+	./compile $(DEFINES) install-big.c

 install.o: \

 compile install.c substdio.h strerr.h error.h open.h readwrite.h \

@@ -808,7 +810,7 @@

 forward preline condredirect bouncesaying except maildirmake \

 maildir2mbox maildirwatch qail elq pinq idedit install-big install \

 instcheck home home+df proc proc+df binm1 binm1+df binm2 binm2+df \

-binm3 binm3+df

+binm3 binm3+df qmail-todo

 load: \

 make-load warn-auto.sh systype

@@ -1509,7 +1511,7 @@

 scan.h case.h auto_qmail.h trigger.h newfield.h stralloc.h quote.h \

 qmail.h substdio.h qsutil.h prioq.h datetime.h gen_alloc.h constmap.h \

 fmtqfn.h readsubdir.h direntry.h

-	./compile qmail-send.c

+	./compile $(DEFINES) qmail-send.c

 qmail-showctl: \

 load qmail-showctl.o auto_uids.o control.o open.a getln.a stralloc.a \

@@ -1574,7 +1576,7 @@

 qmail-start.o: \

 compile qmail-start.c fd.h prot.h exit.h fork.h auto_uids.h

-	./compile qmail-start.c

+	./compile $(DEFINES) qmail-start.c

 qmail-tcpok: \

 load qmail-tcpok.o open.a lock.a strerr.a substdio.a error.a str.a \

@@ -1605,6 +1607,20 @@

 compile qmail-tcpto.c substdio.h subfd.h substdio.h auto_qmail.h \

 fmt.h ip.h lock.h error.h exit.h datetime.h now.h datetime.h

 	./compile qmail-tcpto.c

+

+qmail-todo: \

+load qmail-todo.o control.o constmap.o trigger.o fmtqfn.o now.o \

+readsubdir.o case.a ndelay.a getln.a sig.a open.a stralloc.a alloc.a \

+substdio.a error.a str.a fs.a auto_qmail.o auto_split.o

+	./load qmail-todo control.o constmap.o trigger.o fmtqfn.o now.o \

+	readsubdir.o case.a ndelay.a getln.a sig.a open.a stralloc.a \

+	alloc.a substdio.a error.a str.a fs.a auto_qmail.o auto_split.o

+

+qmail-todo.o: \

+compile alloc.h auto_qmail.h byte.h constmap.h control.h direntry.h error.h \

+exit.h fmt.h fmtqfn.h getln.h open.h ndelay.h now.h readsubdir.h readwrite.h \

+scan.h select.h str.h stralloc.h substdio.h trigger.h

+	./compile $(DEFINES) qmail-todo.c

 qmail-upq: \

 warn-auto.sh qmail-upq.sh conf-qmail conf-break conf-split

diff -uN qmail-1.03/TARGETS qmail-exttodo/TARGETS

--- qmail-1.03/TARGETS	Mon Jun 15 12:53:16 1998

+++ qmail-exttodo/TARGETS	Mon Apr 22 13:59:32 2002

@@ -385,3 +385,5 @@

 man

 setup

 check

+qmail-todo.o

+qmail-todo

diff -uN qmail-1.03/hier.c qmail-exttodo/hier.c

--- qmail-1.03/hier.c	Mon Jun 15 12:53:16 1998

+++ qmail-exttodo/hier.c	Mon Apr 22 14:01:58 2002

@@ -108,6 +108,9 @@

   c(auto_qmail,"bin","qmail-rspawn",auto_uido,auto_gidq,0711);

   c(auto_qmail,"bin","qmail-clean",auto_uido,auto_gidq,0711);

   c(auto_qmail,"bin","qmail-send",auto_uido,auto_gidq,0711);

+#ifdef EXTERNAL_TODO

+  c(auto_qmail,"bin","qmail-todo",auto_uido,auto_gidq,0711);

+#endif

   c(auto_qmail,"bin","splogger",auto_uido,auto_gidq,0711);

   c(auto_qmail,"bin","qmail-newu",auto_uido,auto_gidq,0700);

   c(auto_qmail,"bin","qmail-newmrh",auto_uido,auto_gidq,0700);

diff -uN qmail-1.03/install-big.c qmail-exttodo/install-big.c

--- qmail-1.03/install-big.c	Mon Jun 15 12:53:16 1998

+++ qmail-exttodo/install-big.c	Mon Apr 22 14:02:11 2002

@@ -108,6 +108,9 @@

   c(auto_qmail,"bin","qmail-rspawn",auto_uido,auto_gidq,0711);

   c(auto_qmail,"bin","qmail-clean",auto_uido,auto_gidq,0711);

   c(auto_qmail,"bin","qmail-send",auto_uido,auto_gidq,0711);

+#ifdef EXTERNAL_TODO

+  c(auto_qmail,"bin","qmail-todo",auto_uido,auto_gidq,0711);

+#endif

   c(auto_qmail,"bin","splogger",auto_uido,auto_gidq,0711);

   c(auto_qmail,"bin","qmail-newu",auto_uido,auto_gidq,0700);

   c(auto_qmail,"bin","qmail-newmrh",auto_uido,auto_gidq,0700);

diff -uN qmail-1.03/qmail-send.c qmail-exttodo/qmail-send.c

--- qmail-1.03/qmail-send.c	Mon Jun 15 12:53:16 1998

+++ qmail-exttodo/qmail-send.c	Sun Jan  5 22:09:42 2003

@@ -1215,6 +1215,7 @@

 /* this file is too long ---------------------------------------------- TODO */

+#ifndef EXTERNAL_TODO

 datetime_sec nexttodorun;

 DIR *tododir; /* if 0, have to opendir again */

 stralloc todoline = {0};

@@ -1438,6 +1439,143 @@

    if (fdchan[c] != -1) close(fdchan[c]);

 }

+#endif

+

+/* this file is too long ------------------------------------- EXTERNAL TODO */

+

+#ifdef EXTERNAL_TODO

+stralloc todoline = {0};

+char todobuf[2048];

+int todofdin;

+int todofdout;

+int flagtodoalive;

+

+void tododied() { log1("alert: oh no! lost qmail-todo connection! dying...\n");

+ flagexitasap = 1; flagtodoalive = 0; }

+

+void todo_init()

+{

+  todofdout = 7;

+  todofdin = 8;

+  flagtodoalive = 1;

+  /* sync with external todo */

+  if (write(todofdout, "S", 1) != 1) tododied();

+  

+  return;

+}

+

+void todo_selprep(nfds,rfds,wakeup)

+int *nfds;

+fd_set *rfds;

+datetime_sec *wakeup;

+{

+  if (flagexitasap) {

+    if (flagtodoalive) {

+      write(todofdout, "X", 1);

+    }

+  }

+  if (flagtodoalive) {

+    FD_SET(todofdin,rfds);

+    if (*nfds <= todofdin)

+      *nfds = todofdin + 1;

+  }

+}

+

+void todo_del(char* s)

+{

+ int flagchan[CHANNELS];

+ struct prioq_elt pe;

+ unsigned long id;

+ unsigned int len;

+ int c;

+

+ for (c = 0;c < CHANNELS;++c) flagchan[c] = 0;

+ switch(*s++) {

+  case 'L':

+    flagchan[0] = 1;

+    break;

+  case 'R':

+    flagchan[1] = 1;

+    break;

+  case 'B':

+    flagchan[0] = 1;

+    flagchan[1] = 1;

+    break;

+  case 'X':

+    break;

+  default:

+    log1("warning: qmail-send unable to understand qmail-todo\n");

+    return;

+ }

+ 

+ len = scan_ulong(s,&id);

+ if (!len || s[len]) {

+  log1("warning: qmail-send unable to understand qmail-todo\n");

+  return;

+ }

+

+ pe.id = id; pe.dt = now();

+ for (c = 0;c < CHANNELS;++c)

+   if (flagchan[c])

+     while (!prioq_insert(&pqchan[c],&pe)) nomem();

+

+ for (c = 0;c < CHANNELS;++c) if (flagchan[c]) break;

+ if (c == CHANNELS)

+   while (!prioq_insert(&pqdone,&pe)) nomem();

+

+ return;

+}

+

+void todo_do(rfds)

+fd_set *rfds;

+{

+  int r;

+  char ch;

+  int i;

+  

+  if (!flagtodoalive) return;

+  if (!FD_ISSET(todofdin,rfds)) return;

+

+  r = read(todofdin,todobuf,sizeof(todobuf));

+  if (r == -1) return;

+  if (r == 0) {

+    if (flagexitasap)

+      flagtodoalive = 0;

+    else

+      tododied();

+    return;

+  }

+  for (i = 0;i < r;++i) {

+    ch = todobuf[i];

+    while (!stralloc_append(&todoline,&ch)) nomem();

+    if (todoline.len > REPORTMAX)

+      todoline.len = REPORTMAX;

+      /* qmail-todo is responsible for keeping it short */

+    if (!ch && (todoline.len > 1)) {

+      switch (todoline.s[0]) {

+	case 'D':

+	  if (flagexitasap) break;

+	  todo_del(todoline.s + 1);

+	  break;

+	case 'L':

+	  log1(todoline.s + 1);

+	  break;

+	case 'X':

+	  if (flagexitasap)

+	    flagtodoalive = 0;

+	  else

+	    tododied();

+	  break;

+	default:

+	  log1("warning: qmail-send unable to understand qmail-todo: report mangled\n");

+	  break;

+      }

+      todoline.len = 0;

+    }

+  }

+}

+

+#endif

 /* this file is too long ---------------------------------------------- MAIN */

@@ -1504,6 +1642,9 @@

    log1("alert: unable to reread controls: unable to switch to home directory\n");

    return;

   }

+#ifdef EXTERNAL_TODO

+ write(todofdout, "H", 1);

+#endif

  regetcontrols();

  while (chdir("queue") == -1)

   {

@@ -1568,8 +1709,12 @@

  todo_init();

  cleanup_init();

+#ifdef EXTERNAL_TODO

+ while (!flagexitasap || !del_canexit() || flagtodoalive)

+#else

  while (!flagexitasap || !del_canexit())

-  {

+#endif

+ {

    recent = now();

    if (flagrunasap) { flagrunasap = 0; pqrun(); }

diff -uN qmail-1.03/qmail-start.c qmail-exttodo/qmail-start.c

--- qmail-1.03/qmail-start.c	Mon Jun 15 12:53:16 1998

+++ qmail-exttodo/qmail-start.c	Mon Apr 22 13:55:48 2002

@@ -8,6 +8,9 @@

 char *(qcargs[]) = { "qmail-clean", 0 };

 char *(qlargs[]) = { "qmail-lspawn", "./Mailbox", 0 };

 char *(qrargs[]) = { "qmail-rspawn", 0 };

+#ifdef EXTERNAL_TODO

+char *(qtargs[]) = { "qmail-todo", 0};

+#endif

 void die() { _exit(111); }

@@ -18,13 +21,28 @@

 int pi4[2];

 int pi5[2];

 int pi6[2];

-

-void close23456() { close(2); close(3); close(4); close(5); close(6); }

+#ifdef EXTERNAL_TODO

+int pi7[2];

+int pi8[2];

+int pi9[2];

+int pi10[2];

+#endif

+

+void close23456() { 

+  close(2); close(3); close(4); close(5); close(6); 

+#ifdef EXTERNAL_TODO

+  close(7); close(8);

+#endif

+}

 void closepipes() {

   close(pi1[0]); close(pi1[1]); close(pi2[0]); close(pi2[1]);

   close(pi3[0]); close(pi3[1]); close(pi4[0]); close(pi4[1]);

   close(pi5[0]); close(pi5[1]); close(pi6[0]); close(pi6[1]);

+#ifdef EXTERNAL_TODO

+  close(pi7[0]); close(pi7[1]); close(pi8[0]); close(pi8[1]);

+	close(pi9[0]); close(pi9[1]); close(pi10[0]); close(pi10[1]);

+#endif

 }

 void main(argc,argv)

@@ -40,6 +58,10 @@

   if (fd_copy(4,0) == -1) die();

   if (fd_copy(5,0) == -1) die();

   if (fd_copy(6,0) == -1) die();

+#ifdef EXTERNAL_TODO

+  if (fd_copy(7,0) == -1) die();

+  if (fd_copy(8,0) == -1) die();

+#endif

   if (argv[1]) {

     qlargs[1] = argv[1];

@@ -70,6 +92,12 @@

   if (pipe(pi4) == -1) die();

   if (pipe(pi5) == -1) die();

   if (pipe(pi6) == -1) die();

+#ifdef EXTERNAL_TODO

+  if (pipe(pi7) == -1) die();

+  if (pipe(pi8) == -1) die();

+  if (pipe(pi9) == -1) die();

+  if (pipe(pi10) == -1) die();

+#endif

   switch(fork()) {

     case -1: die();

@@ -105,6 +133,34 @@

       execvp(*qcargs,qcargs);

       die();

   }

+

+#ifdef EXTERNAL_TODO

+  switch(fork()) {

+    case -1: die();

+    case 0:

+      if (prot_uid(auto_uids) == -1) die();

+      if (fd_copy(0,pi7[0]) == -1) die();

+      if (fd_copy(1,pi8[1]) == -1) die();

+      close23456();

+      if (fd_copy(2,pi9[1]) == -1) die();

+      if (fd_copy(3,pi10[0]) == -1) die();

+      closepipes();

+      execvp(*qtargs,qtargs);

+      die();

+  }

+

+  switch(fork()) {

+    case -1: die();

+    case 0:

+      if (prot_uid(auto_uidq) == -1) die();

+      if (fd_copy(0,pi9[0]) == -1) die();

+      if (fd_copy(1,pi10[1]) == -1) die();

+      close23456();

+      closepipes();

+      execvp(*qcargs,qcargs);