ANU News Patch
Patch ID: 940524_nntp_server.c!bailey@genetics.upenn.edu
Date: 24-May-1994
Author: Charles Bailey  bailey@genetics.upenn.edu
News Version: 6.1beta9
News Files: NNTP_Server.C, NewsSite.H
Description: Incorporates (with some changes) those optimizations found in
patch 930723_nntp_server.c!saul@hnrc.tufts.edu which were not already folded
into the 6.1beta9 distribution.  This includes the following:
1. Adds const to various constant string declarations.
2. Moves various compile-time constants to NewsSite.H
3. Calculates initial allocation quantity for various files.
4. Allows specification of maximum incoming batch file size (in bytes) via
   the logical name NEWS_NNTP_SERVER_BATCH_SIZE, subject to the limits imposed
   by compile-time constants.  This value is used when processing transferred
   items only; processing of posted items always uses NEWS_BATCH_SIZE_LOWLIM.
5. Allows sites to keep batch files open until full, using fflush() and fsync()
   to flush buffers.  This is the default behavior, unless disabled by
   defining the logical name NEWS_NNTP_SERVER_BATCH_FLUSH_DISABLE to translate
   to a value considered true by DCL (i.e. odd decimal number or string
   beginning with one of TtYy.
Replaces: 930723_nntp_server.c!saul@hnrc.tufts.edu for use with version
6.1beta9. 930723_nntp_server.c!saul@hnrc.tufts.edu remains current for use
with previous versions.
*** nntp_server.c
--- nntp_server_new.c
**************
*** 219,221
  **
  **--
  **/
--- 219,224 -----
+ **	V6.1b9	24-May-1994  Charles Bailey  bailey@genetics.upenn.edu
+ **	  - added further file optimizations based on patch noted in
+ **	    previous comment (930723_nntp_server.c!saul@hnrc.tufts.edu).
  **
  **--
  **/
**************
*** 231,236
                              /* system definitions */
  #include "newsinclude.h"
  #include "newsdefine.h"
  
  #if 0/*vax11c*/
  #include maildef
--- 234,240 -----
                              /* system definitions */
  #include "newsinclude.h"
  #include "newsdefine.h"
+ #include "newssite.h"
  
  #if 0/*vax11c*/
  #include maildef
**************
*** 269,282
  void newscmd() {}   /* hack to substitute for globaldef - see above */
  #endif
  
- #define LOGGING 1 /* set to 1 to produce LOGGING output to file NEWS_NNTP_LOG */
- 
- #define LOG_TABLE    "LNM$SYSTEM"
- 
- #define NEWSBATCH    "NEWS_MANAGER:NNTP_%s_%X.%sBATCH"          /* file names */
- 
- #define NEWS_BATCH_SIZE 1000000
- 
  /*
   * Define macros READ_MBF and WRITE_MBF as empty to disable multibuffering,
   * or specify appropriate MBF options
--- 273,278 -----
  void newscmd() {}   /* hack to substitute for globaldef - see above */
  #endif
  
  /*
   * Define macros READ_MBF and WRITE_MBF as empty to disable multibuffering,
   * or specify appropriate MBF options
**************
*** 379,385
  DIST_ENTRY_T *distfile = 0;
  
                              /* server response table */
! static 
  char *msg[] =        {"100 help text follows\r\n",
                        "199 debug output\r\n",
                        "200 ANU NEWS/NNTP server %s (V%s). Ready at %s (%s).\r\n",
--- 375,381 -----
  DIST_ENTRY_T *distfile = 0;
  
                              /* server response table */
! static const
  char *msg[] =        {"100 help text follows\r\n",
                        "199 debug output\r\n",
                        "200 ANU NEWS/NNTP server %s (V%s). Ready at %s (%s).\r\n",
**************
*** 425,431
  		      "503 fatal program fault - closing connection\r\n",
                        ".\r\n"};
  
! static
  char *help_cmds[] = {
      "ARTICLE <message-id> Send article referenced by id.\r\n",
      "ARTICLE <nnn>        Send article, number <nnn> from current newsgroup.\r\n",
--- 421,427 -----
  		      "503 fatal program fault - closing connection\r\n",
                        ".\r\n"};
  
! static const
  char *help_cmds[] = {
      "ARTICLE <message-id> Send article referenced by id.\r\n",
      "ARTICLE <nnn>        Send article, number <nnn> from current newsgroup.\r\n",
**************
*** 636,642
  
  #define MAX_UNK  50
  
! static
  char *header_keyw[] = {
            "path",
            "newsgroups",
--- 632,638 -----
  
  #define MAX_UNK  50
  
! static const
  char *header_keyw[] = {
            "path",
            "newsgroups",
**************
*** 1257,1262
      if (!*itm_fname) {
        int header = 1;
  
        sprintf(itm_fname,Itm_template,util_dir(newsgrp.grp_name),cre_itm[i]);
        if (!(ofile = fopen(itm_fname,OPEN_WRITE_OPT1,
                                      "deq=16","fop=cbt,tef"))) {
--- 1253,1266 -----
      if (!*itm_fname) {
        int header = 1;
  
+       /* 7/21/93 saul@hnrc.tufts.edu - calculate approximate output size
+        * based upon the size in bytes of the input file and pre-extend
+        * the file */
+ 
+       char alloc_size[32];
+       sprintf (alloc_size, "alq=%d", ((sbuffer.st_size+511)/512));
+ 
+ 
        sprintf(itm_fname,Itm_template,util_dir(newsgrp.grp_name),cre_itm[i]);
        if (!(ofile = fopen(itm_fname,OPEN_WRITE_OPT1,alloc_size,
                                      "deq=16","fop=cbt,tef"))) {
**************
*** 1258,1264
        int header = 1;
  
        sprintf(itm_fname,Itm_template,util_dir(newsgrp.grp_name),cre_itm[i]);
!       if (!(ofile = fopen(itm_fname,OPEN_WRITE_OPT1,
                                      "deq=16","fop=cbt,tef"))) {
          strcpy(no_new_item,"Cannot open output file (write access)");
          failure = 1;
--- 1262,1268 -----
  
  
        sprintf(itm_fname,Itm_template,util_dir(newsgrp.grp_name),cre_itm[i]);
!       if (!(ofile = fopen(itm_fname,OPEN_WRITE_OPT1,alloc_size,
                                      "deq=16","fop=cbt,tef"))) {
          strcpy(no_new_item,"Cannot open output file (write access)");
          failure = 1;
**************
*** 1411,1416
    int  approval_required = 0,
         posted = 0, cur_time;
    struct tm *stme;
  
    time(&cur_time);
    p = ctime(&cur_time);
--- 1415,1421 -----
    int  approval_required = 0,
         posted = 0, cur_time;
    struct tm *stme;
+   struct stat stat_buffer;
  
    time(&cur_time);
    p = ctime(&cur_time);
**************
*** 1446,1451
      while (!strchr(xfrbuf,'\n') && fgets(xfrbuf,sizeof(xfrbuf),fpr));
      }
    fseek(fpr,0,0);
  
    while (fgets(xfrbuf,sizeof(xfrbuf),fpr)) {
      if (*xfrbuf == '\n') break;
--- 1451,1459 -----
      while (!strchr(xfrbuf,'\n') && fgets(xfrbuf,sizeof(xfrbuf),fpr));
      }
    fseek(fpr,0,0);
+   /* save the size for pre-extending the output file
+    * saul@hnrc.tufts.edu 7/22/93 */
+     fstat (fileno(fpr), &stat_buffer);
  
    while (fgets(xfrbuf,sizeof(xfrbuf),fpr)) {
      if (*xfrbuf == '\n') break;
**************
*** 1522,1530
    get_post_defaults(&newsgroups_header,&distribution_header,&followup_to_header);
  
    mail_add_expiry = parse_expiry_date(expires_header);
! 
!   fpw = fopen(scrfile,OPEN_WRITE_OPT1,"deq=16","fop=cbt,tef");
!   _ck_open_w(fpw,scrfile);
    fprintf(fpw,"Relay-Version: %s %s VAX/VMS; site %s\n",
                NEWS_VERSION,NEWS_DDATE,news_node);
  
--- 1530,1543 -----
    get_post_defaults(&newsgroups_header,&distribution_header,&followup_to_header);
  
    mail_add_expiry = parse_expiry_date(expires_header);
!   {
!     /* open the file and prextend it to the size of existing file 
!      * saul@hnrc.tufts.edu 7/22/93 */
!     char alloc_size[32];
!     sprintf(alloc_size,"alq=%d",((stat_buffer.st_size+511)/512));
!     fpw = fopen(scrfile,OPEN_WRITE_OPT2,alloc_size,"fop=cbt,tef");
!     _ck_open_w(fpw,scrfile);
!     }
    fprintf(fpw,"Relay-Version: %s %s VAX/VMS; site %s\n",
                NEWS_VERSION,NEWS_DDATE,news_node);
  
**************
*** 1613,1619
      }
    else {
      if (control_header || always_spool) {
!       posted = spool_add_item(scrfile,0,msg,stm);
        }
      else {
        if (!sys_local_accept(newsgroups_header,distribution_header))
--- 1626,1632 -----
      }
    else {
      if (control_header || always_spool) {
!       posted = spool_add_item(scrfile,0,msg,stm,POST);
        }
      else {
        if (!sys_local_accept(newsgroups_header,distribution_header))
**************
*** 1642,1648
   * spool_add_item
   *
   */
! spool_add_item(filename,buf,msg,stm)
    char *filename;	/* name of temp file containing article (1st char NUL if none) */
    char *buf;		/* ptr to article in memory (only used if no temp file) */
    char *msg;		/* where to store an error message */
--- 1655,1664 -----
   * spool_add_item
   *
   */
! 
! static int max_batch = 0;
! 
! spool_add_item(filename,buf,msg,stm,mode)
    char *filename;	/* name of temp file containing article (1st char NUL if none) */
    char *buf;		/* ptr to article in memory (only used if no temp file) */
    char *msg;		/* where to store an error message */
**************
*** 1647,1652
    char *buf;		/* ptr to article in memory (only used if no temp file) */
    char *msg;		/* where to store an error message */
    int stm;
  {
    time_t time_now;
    char xfrbuf[IO_SIZE];
--- 1663,1671 -----
    char *buf;		/* ptr to article in memory (only used if no temp file) */
    char *msg;		/* where to store an error message */
    int stm;
+   int mode;		/* POST - we're spooling a posting
+ 			   FEED - we're spooling a newsfeed  - used only to
+ 			      estimate spool file size */
  {
    time_t time_now;
    char *batch_logical = NULL;
**************
*** 1649,1654
    int stm;
  {
    time_t time_now;
    char xfrbuf[IO_SIZE];
    FILE *fpr = 0;
    int spool_flush_batch();
--- 1668,1674 -----
  			      estimate spool file size */
  {
    time_t time_now;
+   char *batch_logical = NULL;
    char xfrbuf[IO_SIZE];
    FILE *fpr = 0;
    int spool_flush_batch();
**************
*** 1659,1664
        return(0);
        }
      }
  
    if (leave_spool_open) {
      sys$cantim(spool_flush_batch, 0);
--- 1679,1699 -----
        return(0);
        }
      }
+   max_batch = 0;
+   if (mode == FEED) { /* if we're spooling an incoming newsfeed, calculate
+ 			 the batch size */
+     if ((batch_logical = news_getenv("NEWS_NNTP_SERVER_BATCH_SIZE",0)) != NULL)
+               sscanf(batch_logical,"%d",&max_batch);
+     if (max_batch == 0) max_batch = NEWS_BATCH_SIZE;
+     else if (max_batch < NEWS_BATCH_SIZE_LOWLIM)
+       max_batch = NEWS_BATCH_SIZE_LOWLIM;
+     else if (max_batch > NEWS_BATCH_SIZE_HIGHLIM)
+       max_batch = NEWS_BATCH_SIZE_HIGHLIM;
+     }
+   else {     /* otherwise, we're spooling a single posting, so use a
+ 		more conservative size */
+       max_batch = NEWS_BATCH_SIZE_LOWLIM;
+     }
  
    if (leave_spool_open) {
      sys$cantim(spool_flush_batch, 0);
**************
*** 1662,1669
  
    if (leave_spool_open) {
      sys$cantim(spool_flush_batch, 0);
!     if (fpb && (batch_size > NEWS_BATCH_SIZE))
!       spool_flush_batch();
      if (!fpb) {
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
--- 1697,1703 -----
  
    if (leave_spool_open) {
      sys$cantim(spool_flush_batch, 0);
!     if (fpb) spool_flush_batch();
      if (!fpb) {
        char alloc_size[32];
        sprintf (alloc_size, "alq=%d", (max_batch+511)/512);
**************
*** 1665,1670
      if (fpb && (batch_size > NEWS_BATCH_SIZE))
        spool_flush_batch();
      if (!fpb) {
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
        fpb = fopen(batchname_tmp,OPEN_WRITE_OPT2,"fop=cbt");
--- 1699,1706 -----
      sys$cantim(spool_flush_batch, 0);
      if (fpb) spool_flush_batch();
      if (!fpb) {
+       char alloc_size[32];
+       sprintf (alloc_size, "alq=%d", (max_batch+511)/512);
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
        fpb = fopen(batchname_tmp,OPEN_WRITE_OPT2,alloc_size,"fop=cbt");
**************
*** 1667,1673
      if (!fpb) {
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
!       fpb = fopen(batchname_tmp,OPEN_WRITE_OPT2,"fop=cbt");
        batch_size = 0;
        }
      }
--- 1703,1709 -----
        sprintf (alloc_size, "alq=%d", (max_batch+511)/512);
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
!       fpb = fopen(batchname_tmp,OPEN_WRITE_OPT2,alloc_size,"fop=cbt");
        batch_size = 0;
        }
      }
**************
*** 1678,1684
        }
  
      rename(batchname,batchname_tmp);
!     if (!*batchname || (batch_size > NEWS_BATCH_SIZE) ||
         !(fpb = fopen(batchname_tmp,OPEN_APPEND_OPT2))) {
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
--- 1714,1720 -----
        }
  
      rename(batchname,batchname_tmp);
!     if (!*batchname || (batch_size > max_batch) ||
         !(fpb = fopen(batchname_tmp,OPEN_APPEND_OPT2))) {
        char alloc_size[32];
        sprintf (alloc_size, "alq=%d", (max_batch+511)/512);
**************
*** 1680,1685
      rename(batchname,batchname_tmp);
      if (!*batchname || (batch_size > NEWS_BATCH_SIZE) ||
         !(fpb = fopen(batchname_tmp,OPEN_APPEND_OPT2))) {
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
        fpb = fopen(batchname_tmp,OPEN_WRITE_OPT2);
--- 1716,1723 -----
      rename(batchname,batchname_tmp);
      if (!*batchname || (batch_size > max_batch) ||
         !(fpb = fopen(batchname_tmp,OPEN_APPEND_OPT2))) {
+       char alloc_size[32];
+       sprintf (alloc_size, "alq=%d", (max_batch+511)/512);
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
        fpb = fopen(batchname_tmp,OPEN_WRITE_OPT2,alloc_size,"fop=cbt");
**************
*** 1682,1688
         !(fpb = fopen(batchname_tmp,OPEN_APPEND_OPT2))) {
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
!       fpb = fopen(batchname_tmp,OPEN_WRITE_OPT2);
        batch_size = 0;
        }
      }
--- 1720,1726 -----
        sprintf (alloc_size, "alq=%d", (max_batch+511)/512);
        sprintf(batchname,NEWSBATCH,time_str(),getpid(),"");
        sprintf(batchname_tmp,NEWSBATCH,time_str(),getpid(),"INCOMING_");
!       fpb = fopen(batchname_tmp,OPEN_WRITE_OPT2,alloc_size,"fop=cbt");
        batch_size = 0;
        }
      }
**************
*** 1705,1712
      sys$setimr(0, daytim, spool_flush_batch, spool_flush_batch, 0);
      }
    else {
!     fclose(fpb);
!     rename(batchname_tmp,batchname);
      }
    strcpy(msg,"Item successfully spooled.");
    return(1);
--- 1743,1749 -----
      sys$setimr(0, daytim, spool_flush_batch, spool_flush_batch, 0);
      }
    else {
!     spool_flush_batch();
      }
    strcpy(msg,"Item successfully spooled.");
    return(1);
**************
*** 1714,1720
  
  spool_flush_batch()
  {
!     if (fpb) {
        if (fclose(fpb)) _ck_close(fpb);
        rename(batchname_tmp,batchname);
        fpb = NULL;
--- 1751,1772 -----
  
  spool_flush_batch()
  {
!   int always_close;
!   char *noflush;
! 
!   if (noflush = news_getenv("NEWS_NNTP_SERVER_BATCH_FLUSH_DISABLE",0)) {
!     if (isdigit(*noflush))
!       always_close = (1 & strtol(noflush, NULL, 0));
!     else {
!       noflush[1] = '\0';
!       noflush[0] = tolower(noflush[0]);
!       always_close = strspn(noflush, "ty");
!       }
!     }
!   else always_close = !leave_spool_open;
! 
!   if (fpb) {
!     if (always_close || batch_size >= max_batch) {
        if (fclose(fpb)) _ck_close(fpb);
        rename(batchname_tmp,batchname);
        fpb = NULL;
**************
*** 1719,1724
        rename(batchname_tmp,batchname);
        fpb = NULL;
        }
  }
  
  /*
--- 1771,1781 -----
        rename(batchname_tmp,batchname);
        fpb = NULL;
        }
+     else {
+       fflush(fpb);
+       fsync(fileno(fpb));
+       }
+     }
  }
  
  /*
**************
*** 1744,1750
      }
    else *filename = '\0';
    if (mode == POST) retval = post_add_item(filename,buf,msgbuf,stm);
!   else retval = spool_add_item(filename,buf,msgbuf,stm);
    if (*filename) while (!delete(filename));
    return(retval);
  }
--- 1801,1807 -----
      }
    else *filename = '\0';
    if (mode == POST) retval = post_add_item(filename,buf,msgbuf,stm);
!   else retval = spool_add_item(filename,buf,msgbuf,stm,FEED);
    if (*filename) while (!delete(filename));
    return(retval);
  }
**************
*** 3781,3787
  
  server_shut()
  {
!   if (leave_spool_open) spool_flush_batch();
    if (grp_file_open)  { sys_close(&grpfab);  grp_file_open = 0; }
    if (itm_file_open)  { sys_close(&itmfab);  itm_file_open = 0; }
    if (hist_file_open) { sys_close(&histfab); hist_file_open = 0; }
--- 3838,3844 -----
  
  server_shut()
  {
!   if (leave_spool_open) { max_batch = 0; spool_flush_batch(); }
    if (grp_file_open)  { sys_close(&grpfab);  grp_file_open = 0; }
    if (itm_file_open)  { sys_close(&itmfab);  itm_file_open = 0; }
    if (hist_file_open) { sys_close(&histfab); hist_file_open = 0; }
*** newssite.h
--- newssite_new.h
**************
*** 30,37
  #define GRP_TIME        60
  
  /*
!  *  NEWS_BATCH_SIZE
!  *  This is the maximum size of news batch files (in bytes - not blocks)
   */
  
  #define NEWS_BATCH_SIZE 250000
--- 30,38 -----
  #define GRP_TIME        60
  
  /*
!  *  NEWSBATCH
!  *  This is the template used by the NNTP server to create file
!  *  names for incoming batches.
   */
  
  #ifndef NEWSBATCH
**************
*** 34,40
   *  This is the maximum size of news batch files (in bytes - not blocks)
   */
  
! #define NEWS_BATCH_SIZE 250000
  
  /*
   *  RECHECK_TIMER
--- 35,68 -----
   *  names for incoming batches.
   */
  
! #ifndef NEWSBATCH
! #define NEWSBATCH    "NEWS_MANAGER:NNTP_%s_%X.%sBATCH"          /* file names */
! #endif
! /*
!  *  NEWS_BATCH_SIZE
!  *  This is the default maximum size of news batch files (in bytes -
!  *  not blocks).  This value may be overridden via the logical name
!  *  NEWS_NNTP_SERVER_BATCH_SIZE.
!  *  NEWS_BATCH_SIZE_LOWLIM
!  *  This is the lowest value that can be used as a maximum batch file size,
!  *  regardless of the value specified by the NEWS_NNTP_SERVER_BATCH_SIZE
!  *  logical name.  It is also the maximum batch file size used when spooling
!  *  single postings.
!  *  NEWS_BATCH_SIZE_HIGHLIM
!  *  This is the highest value that can be used as a maximum batch file size,
!  *  regardless of the value specified by the News_NNTP_Server_Batch_Size
!  *  logical name.
!  */
! 
! #ifndef NEWS_BATCH_SIZE
! #define NEWS_BATCH_SIZE (500*512)
! #endif
! #ifndef NEWS_BATCH_SIZE_LOWLIM
! #define NEWS_BATCH_SIZE_LOWLIM (10*512)
! #endif
! #ifndef NEWS_BATCH_SIZE_HIGHLIM
! #define NEWS_BATCH_SIZE_HIGHLIM (10000*512)
! #endif
  
  /*
   *  RECHECK_TIMER
*** patchlist.h;-1
--- patchlist.h
**************
*** 1,1
  =+=+= End =+=+=
--- 1,2 -----
+ 940524_nntp_server.c!bailey@genetics.upenn.edu
  =+=+= End =+=+=