$ ! N E W S S K I M . C O M - DECUS UUCP Distribution version $ GOTO START !'f$verify(f$integer(f$trnlnm("NEWS_NEWSSKIM_VERIFY")))' $ ! $ ! NEWSSKIM - the daily news management code. This code is submitted into $ ! the batch queues under the "news manager"'s account by news_systartup.com, $ ! and resubmits itself. The current code is expected to run hourly. This $ ! frequency should probably not be modified, since all that is done at NON $ ! midnight runs is to forward down stream news and to process any news that $ ! arrived by mail. This processing is minimal, and if particular efforts $ ! at regulating flow are desired, this should be managed by restricting $ ! uucp calling times to adjacent nodes. Doing things this way, will end up $ ! with a savings in the max disk space used since once batches are "forwarded" $ ! they are usually compressed, hence less space used. $ ! $ ! The code performs the following actions: $ ! 1 - read all items MAILED to RNEWS or NEWSMAIL, and add them $ ! 2 - delete (skim) all expired news items and expired newsgroups (daily) $ ! 3 - Generate IHAVE control messages to down stream sites as appropriate $ ! 4 - Distribute posted items to remote sites $ ! a) COPY post items to NEWSMGR proxies on remote DECNET nodes $ ! b) UUX_RNEWS via UUCP to send post items to adjacent UUCP sites $ ! 5 - If new UUCP maps have arrived, process them into a new paths $ ! database for proper mail routing (daily) $ ! 6 - Arbitron and Inpaths reports are generated (once a month) $ ! 7 - Log file analysis/reporting, and cleanup $ ! 8 - rebuilds the news index files on Sunday nights. (mornings) $ ! 9 - resubmit for next run $ ! $ ! Author: $ ! G Huston $ ! Computer Services Centre $ ! Australian National University $ ! $ ! Version: $ ! V1.0 17-Jul-1986 $ ! V2.0 16-Jul-1987 $ ! V4.3 12-Jan-1987 GIH $ ! V5.2 26-Apr-1988 GIH $ ! V5.6 3-Nov-1988 GIH $ ! V5.7 12-Feb-1989 Mark Pizzolato - mark@infopiz.UUCP $ ! V5.7A 15-Apr-1989 Jamie Hanrahan - jeh@simpact.UUCP $ ! V5.7A-1 31-May-1989 Mark Pizzolato - mark@infopiz.UUCP $ ! V5.8 1-Jul-1989 Mark Pizzolato - mark@infopiz.UUCP $ ! V5.9 15-Oct-1989 Mark Pizzolato - mark@infopiz.UUCP $ ! V5.9B 2-Dec-1989 Mark Pizzolato - mark@infopiz.UUCP $ ! V5.9C 20-Dec-1989 Mark Pizzolato - mark@infopiz.UUCP $ ! Jamie Hanrahan - jeh@simpact.com $ ! Terry Poot - tp@mccall.com $ ! Earle Ake - fac2@dayton.saic.com $ ! V6.0 30-Mar-1990 Geoff Huston - gih900@sao.aarnet.edu.au $ ! (note, this also works with V5.9C -- jeh) $ ! V6.1-A4 29-Mar-1992 Mark Pizzolato - mark@infocomm.com $ ! V6.1-B5 19-Sep-1992 Mark Pizzolato - mark@infocomm.com $ ! $ !********************************************************************** $ ! $ ! Control Logicals: Units: Default: $ ! ================================================================ $ ! NEWS_NEWSSKIM_LOG_LIFETIME days 4 $ ! NEWS_NEWSSKIM_SET_ITEMHOLD boolean false $ ! NEWS_NEWSSKIM_ARBITRON boolean false $ ! NEWS_NEWSSKIM_INPATHS boolean false $ ! NEWS_NEWSSKIM_FEEDCHECK boolean true $ ! NEWS_NEWSSKIM_SKIM_HISTORY days 45 $ ! NEWS_NEWSSKIM_SKIM_QUALIFIERS string "" $ ! NEWS_NEWSSKIM_NIGHTLY_BUCKET_RECLAIM boolean true $ ! $ ! start code $ $START: $ say = "write sys$output" $ set protection=(S:RWED,O:RWED,G:R,W:R)/default $ start_time = f$time() $ SS$_DEADLOCK = "%X10000E0A" $ rnews_job_prio = f$getsyi("MAXQUEPRI")-1 $ say "''f$time()' NEWSSKIM starting" $ define/nolog newsrc nla0: $ $ old_prcnam = f$getjpi("","PRCNAM") $ if f$mode().eqs."BATCH" then set process/name="NEWSSKIM_''f$element(1,"_",old_prcnam)'" $ $ if f$search("news_manager:last_skim.time").eqs."" $ then $ new_day = "Y" ! file missing, assume first run ever $ else $ last_time = "" $ define/user sys$output nla0: $ define/user sys$error nla0: $ last_time = f$cvtime(f$file("news_manager:last_skim.time","CDT"),"ABSOLUTE") $ create NLA0: $ new_day = f$element(0, " ", last_time) .nes. f$element(0, " ", f$cvtime(start_time,"ABSOLUTE")) $ endif $ if (f$mode().eqs."BATCH").and.(f$environment("DEPTH").eq.1).and.(p1.eqs."") $ then $ ! $ ! To avoid clutter in other guys log (output) files, and also to $ ! provide a consistent place to look for the information reported by $ ! this procedure, namely NEWS_LOG:NEWSSKIM.LOG, we recurse with output $ ! directed to NEWS_LOG:NEWSSKIM.LOG (Note, we rename to a file of the $ ! form NEWSSKIM.LOG_mmdd-hh at the end of this procedure (after all $ ! useful work has been done). $ ! $ set noon $ log = f$cvtime(,,"MONTH")+f$cvtime(,,"DAY")+"-"+f$cvtime(,,"HOUR") $ say "''f$time()' Redirecting NEWSSKIM output to NEWS_LOG:NEWSSKIM.LOG." $ open/write/share=read LOGFILE NEWS_LOG:NEWSSKIM.LOG $ @'f$environment("PROCEDURE")'/output=LOGFILE $ close LOGFILE $ ! $ ! Rename our log file to NEWSSKIM.LOG_mmdd-hh to avoid large version numbers $ ! $ rename news_log:newsskim.log;0 news_log:newsskim.log_'log';0 $ say "''f$time()' NEWSSKIM Complete." $ goto all_done $ endif $ if p3.nes."" then new_day = p3 ! Override for interactive testing $ if p1.nes."" then goto 'p1' ! Option for interactive testing $ ! $ ! Make sure the disks are there $ ! $ if f$search("news_root:news.items") .eqs. "" .or. - f$search("news_root:news.groups") .eqs. "" .or. - f$search("news_device:[*...]*.itm") .eqs. "") $ then $ say "Aborting, can't see required files" $ goto abort $ endif $ $ ! $ ! Log the time ... $ ! $record_skim: $ open/read/write/Error=skim_conflict last_skim news_manager:last_skim.time;1 $ create news_manager:last_skim.time $ goto time_logged ! Note: this file is kept open to serialize SKIM operations $skim_conflict: $ if $status.eqs."%X10018292" ! File Not Found $ then $ create news_manager:last_skim.time;1 $ goto record_skim $ endif $ say "''f$time()' Skim Processing already in progress, Exiting." $ goto fin $ $time_logged: $ NEWS = "$UUCP_BIN:NEWS" $ execqual = "" $ IF (f$logi("NEWS_EXECUTE_CONTROL")) THEN execqual = "/EXECUTE" $ cregrpqual = "" $ IF (f$logi("NEWS_AUTO_CREGRP")) THEN cregrpqual = "/CREGRP" $ $ on error then goto abort $ oldprv = f$setprv("SYSPRV") $ if .not. f$privilege("SYSPRV") then exit $ set output_rate = :00:15 ! Allow a close watch to be kept $ $ ! $ ! Read all and remove locally addressed mail to NEWSMGR $ ! $ set noon $ say "''f$time()' Looking for items mailed to RNEWS or NEWSMAIL" $ mail select newmail extract/all news_manager:newmail.mail delete/all exit $ ! $ ! Decode extracted UUENCODEd mail $ ! $ if f$search("uucp_bin:newsuud.exe").eqs."" then goto mail_add $ newsuud = "$uucp_bin:newsuud" $decode_uue: $ ufile = f$search("news_manager:newmail.mail;*",1) $ if ufile .nes. "" $ then $ newsuud 'ufile' $ goto decode_uue $ endif $ ! $ ! Place all extracted mail into the news data base $ ! $mail_add: $ if (f$search("news_manager:newmail.mail").eqs."") then goto uuemail_add $ rename news_manager:newmail.mail news_manager:newmail.mail_batched $ batch = f$search("news_manager:newmail.mail_batched") $ batch_log = "NEWS_LOG:RNEWS_MAIL_''f$cvtime(,"COMPARISON","MONTH")'-''f$cvtime(,,"DAY")'-''f$cvtime(,,"HOUR")'-''f$cvtime(,,"MINUTE")'-''f$cvtime(,,"SECOND")'" $ submit/noprint/log='batch_log' - /parameter=('batch',"mail") - /queue=NEWS_BATCH_QUEUE - /priority='rnews_job_prio' - /restart UUCP_BIN:NEWS_RNEWS $uuemail_add: $ if (f$search("news_manager:uudnews.batch").eqs."") then goto nntp_add $ rename news_manager:uudnews.batch news_manager:uudnews.batch_batched $ batch = f$search("news_manager:uudnews.batch_batched") $ batch_log = "NEWS_LOG:RNEWS_MAIL_''f$cvtime(,"COMPARISON","MONTH")'-''f$cvtime(,,"DAY")'-''f$cvtime(,,"HOUR")'-''f$cvtime(,,"MINUTE")'-''f$cvtime(,,"SECOND")'" $ submit/noprint/log='batch_log' - /parameter=('batch',"uuemail") - /queue=NEWS_BATCH_QUEUE - /priority='rnews_job_prio' - /restart UUCP_BIN:NEWS_RNEWS $nntp_add: $ if (f$search("news_manager:nntp_*.batch").eqs."") then goto other_add $ rename news_manager:nntp.batch news_manager:nntp.batch_batched $ batch = f$search("news_manager:nntp.batch_batched") $ batch_log = "NEWS_LOG:RNEWS_NNTP_''f$cvtime(,"COMPARISON","MONTH")'-''f$cvtime(,,"DAY")'-''f$cvtime(,,"HOUR")'-''f$cvtime(,,"MINUTE")'-''f$cvtime(,,"SECOND")'" $ submit/noprint/log='batch_log' - /parameter=('batch',"nntp") - /queue=NEWS_BATCH_QUEUE - /priority='rnews_job_prio' - /restart UUCP_BIN:NEWS_RNEWS $ goto nntp_add $other_add: $ if (f$search("news_manager:news.batch").eqs."") then goto local_add $ rename news_manager:news.batch news_manager:news.batch_batched $ batch = f$search("news_manager:news.batch_batched") $ batch_log = "NEWS_LOG:RNEWS_MAIL_''f$cvtime(,"COMPARISON","MONTH")'-''f$cvtime(,,"DAY")'-''f$cvtime(,,"HOUR")'-''f$cvtime(,,"MINUTE")'-''f$cvtime(,,"SECOND")'" $ submit/noprint/log='batch_log' - /parameter=('batch',"uuemail") - /queue=NEWS_BATCH_QUEUE - /priority='rnews_job_prio' - /restart UUCP_BIN:NEWS_RNEWS $ goto other_add $local_add: $ if f$search("UUCP_BIN:NEWS_LOCAL_RECEIVE.COM").nes."" $ then $ ! $ ! This callout allows for a site specific "other" method for arrival $ ! of news batches. It is recommended that a technique similar to the $ ! above examples for "mail_add", "uuemail_add", and "other_add" be $ ! employed within such a procedure to actually process any inbound $ ! batches (i.e. passing them to UUCP_BIN:NEWS_RNEWS). This will allow $ ! for as much parallelism as possible and also avoids duplication of $ ! resource checking code, and keeps skim and forwarding separate from $ ! the actual inbound processing. $ ! $ @UUCP_BIN:NEWS_LOCAL_RECEIVE $ endif $local_done: $ say "''f$time()' Processing of mailed/forwarded items complete" $ $ ! ------------------- $ ! This section optionally invokes a procedure that sets the default $ ! newsgroup itemhold according an appropriate site specific policy. $ ! $ enable_this = f$trnlnm("NEWS_NEWSSKIM_SET_ITEMHOLD") $ if (enable_this .and. new_day .and. f$search("UUCP_BIN:NEWS_SET_ITEMHOLD.COM") .nes. "") $ then $ set noon $ @UUCP_BIN:NEWS_SET_ITEMHOLD $ endif $ $skim: $ ! --------------------- $ ! Skim -- remove expired items (the function for which this proc was $ ! originally named). $ ! It is a waste of time to perform more than 1 skim of the NEWS files per $ ! day (since the skim operation is solely concerned with deleting expired $ ! items, and items only expire based on time kept in whole days). $ $ if (.not. new_day) then goto skim_done $ ! $ ! Skim off all expired news $ ! $ say "''f$time()' Nightly SKIM Starting" $ history = f$trnlnm("NEWS_NEWSSKIM_SKIM_HISTORY") $ if history.eqs."" then history = "45" $ skimquals = f$trnlnm("NEWS_NEWSSKIM_SKIM_QUALIFIERS") $ news/noscreen skim/history='history' 'skimquals' $ say "''f$time()' Nightly SKIM Completed" $ ! $ ! Daily recovery of buckets in the history and newsitems files tends to $ ! smooth the variance in the size of these files, since these freed $ ! buckets can be reused immediately. This done with other news $ ! processing locked out if the necessary facility (NEWSSHUTDOWN) is $ ! available. (Note SYSLCK priv is required by the NEWSMANAGER account) $ ! $ newsshutdown = f$search("UUCP_BIN:NEWSSHUTDOWN.EXE") $ if newsshutdown.eqs."" then newsshutdown = f$search("UUCP_BIN:NEWSSHUTDOWN.EXE") $ if newsshutdown.nes."" $ then $ old_privs = f$setprv("SYSLCK") $ if .not.f$privilege("SYSLCK") $ then $ say "''f$time()' SYSLCK privilege required to ensure database consistency $ say "''f$time()' Continuing anyway . . ." $ goto do_reclaim $ endif $ define/user NEWS_LOCKED_COMMAND "@''f$environment("PROCEDURE")' do_reclaim ''f$verify()'" $ run 'newsshutdown $ old_privs = f$setprv(old_privs) $ goto reclaim_done $ endif $do_reclaim: $ convert = "convert" $ do_convert = f$trnlnm("NEWS_NEWSSKIM_NIGHTLY_BUCKET_RECLAIM") $ if do_convert.eqs."" then do_convert = "TRUE" $ if .not.do_convert then goto reclaim_done $ say "''f$time()' Reclaiming Free Space in News Items file" $ convert/reclaim/statistics news_root:news.items $ if (f$search("news_root:history.v60").nes."") $ then $ say "''f$time()' Reclaiming Free Space in News History file" $ convert/reclaim/statistics news_root:history.v60 $ endif $ if p1.nes."" then goto fin $reclaim_done: $ junk = f$getqui("CANCEL_OPERATION") $qloop: $ qname = f$getqui("DISPLAY_QUEUE","QUEUE_NAME","*","BATCH") $ if qname.eqs."" then goto release_done $ if qname.nes.f$trnlnm("NEWS_BATCH_QUEUE") then goto qloop $jloop: $ jname = f$getqui("DISPLAY_JOB","JOB_NAME",,"ALL_JOBS") $ if jname.eqs."" then goto release_done $ if jname.nes."NEWS_RNEWS" then goto jloop $ if .not.f$getqui("DISPLAY_JOB","JOB_HOLDING",,"ALL_JOBS,FREEZE_CONTEXT") then goto jloop $ jentry = F$GETQUI("DISPLAY_JOB","ENTRY_NUMBER",,"ALL_JOBS,FREEZE_CONTEXT") $ say "''f$time()' Releasing Job Entry # ''jentry'" $ set entry 'jentry'/release $ goto jloop $release_done: $ if p1.nes."" then goto fin $ say "''f$time()' Skim Completed" $ ! $skim_done: $ $ !********************************************************************** $ ! Post IHAVE, SENDME Control messages to other sites $ ! $ say "''f$time()' Looking for IHAVE control messages to send to other sites" $ mynode = f$trnlnm("NEWS_NODE") $isearch_1: $ sayit = say $ nodedir = f$search("news_manager:ihave_*.dir",2) $ if nodedir .eqs. "" then goto ihave_done $ node = f$parse(nodedir,,,"NAME","SYNTAX_ONLY") - "IHAVE_" $ savedfiles = "news_manager_dev:[ihave_" + node + "]*.*;*" $isearch_2: $ file = f$search(savedfiles,4) $ if file .eqs. "" then goto isearch_1 $ cfile = f$element(0,";",file) $ sayit "''f$time()' ...Posting 'ihave' messages to newsgroup to.''node'" $ sayit = "!" $ on error then goto isearch_2 $ on error then goto ino_copy_2 $ open/read/error=ino_copy_2 access_check 'file' $iadd_file: NEWS/NOSCREEN POST/NEWS=to.'node'/SUBJ="ihave ''mynode'" - /NOEDIT/CONTROL="ihave ''mynode'" 'file YES YES $ if $status.eqs.SS$_DEADLOCK $ then $ say "''f$time()' NEWS Processing is LOCKED - Waiting 10 minutes $ wait 00:10:00 $ goto iadd_file $ endif $ delete 'file $ino_copy_2: $ close/nolog access_check $ goto isearch_2 $ihave_done: $ say "''f$time()' IHAVE scan complete" $ ! $ !********************************************************************** $ ! Copy news items to other sites $ ! $ ! Search for all directories of the form: $ ! news_manager:post_.dir $ ! Copy all items in the node specific directory to the remote node $ ! Once all post directories have been found, delete all items in the $ ! post directory. this code assumes that for DECnet copies, proxies exist $ ! on remote nodes. $ ! $ ! news_manager:post_<[prio-][FILELIST[$st$et]-]transmission_mode_>.dir $ ! $ ! NOTE: The transmission mode may optionally be prepended with a "scan $ ! priority", which is specified as: prio-transmission. (Note the "-" $ ! being lexically significant). The ONLY purpose of this priority $ ! is to reorder which POST directories get flushed first. For example, $ ! if you feed two sites: AAAA and ZZZZ, and it is more important that $ ! outbound traffic for size ZZZZ gets sent before that for AAAA, then $ ! using the priority mechanism is appropriate. $ ! The transmission mode, can be prefixed by a storage methodology. $ ! This storage methodology defaults to traditional batch files, $ ! but it can also be specified as "FILELIST", which indicates that $ ! the file in the posting directory actually contains a list of $ ! file names of the items which are to be batched only at the $ ! time of transmission. $ ! The FILELIST specification, may optionally be followed by a flush $ ! time specification. The "flush time" specification is a time range $ ! which indicates the $st (start hour), and $et (end hour). If a $ ! flush time range is specified, then the FILELIST will only be batched $ ! for transmission when NEWSSKIM is run between the start hour and $ ! the end hour. $ ! $ ! The currently supported transmission modes are: $ ! a) Default - DECNET ala Base ANU NEWS distribution $ ! b) DECNET - Same as above, but explicitly stated. $ ! in both of the above cases, 'node' is the DECnet nodename. $ ! c) UUCP - UUCP copies to 'rnews' on the remote site. $ ! d) UUCP$CMPRS - Compressed batch UUCP copies to 'rnews' on the $ ! remote site. $ ! in both of the above cases, 'node' is the uucp hostname. $ ! e) COMMAND - Local site provided transport implemented by the $ ! file NEWS_TRANSPORT.COM in the directory $ ! NEWS_MANAGER_DEV:[POST_COMMAND_node]. This $ ! procedure is invoked with two arguments: $ ! P1 = node $ ! P2 = file to transport $ ! f) MAIL - Used for mail based feeds. The appropriate logical $ ! name must be defined in NEWSSKIM_LOGICALS.COM $ ! for the mail address of the news_manager account on $ ! the remote system. $ ! g) MLIST - Used for mailing list gateways. The appropriate $ ! logical name must be defined in NEWSSKIM_LOGICALS.COM $ ! for the mail address of the mailing list. $ ! h) UUEMAIL - Used for mail based feeds with EMBL-UUEn/Decoding. $ ! The appropriate logical name must be defined in $ ! NEWSSKIM_LOGICALS.COM for the mail address of the $ ! news_manager account on the remote system. $ ! in the above three cases, 'node' is the uucp hostname. $ ! $ say "''f$time()' Searching for items to forward to other sites" $ set on $ compress = "$uucp_bin:compress" $ uux_rnews = "$uucp_bin:uux_rnews" $ batchnews = "$uucp_bin:batchnews" $ gatenews = "$uucp_bin:gatenews" $ mx_enter = "$mx_exe:mx_site_in" $ newsuue = "$uucp_bin:newsuue" $ if f$search("uucp_bin:newsskim_logicals.com").nes."" then - @uucp_bin:newsskim_logicals $search_1: $ sayit = say $ nodedir = f$search("news_manager:post_*.dir",2) $ if nodedir .eqs. "" then goto transport_done $ node = f$parse(nodedir,,,"NAME","SYNTAX_ONLY") - "POST_" $ savedfiles = "news_manager_dev:[post_" + node + "]*.*;" $ method = f$element(0,"_",node) + "_" $ node = node - method $ compressed = "NO" $ prio = f$string(f$integer(f$element(0,"-",method))) $ if (prio.eqs.f$element(0,"-",method)) then method = (method - prio) - "-" $ if f$element(1,"-",method).nes."-" $ then $ storage_class = f$element(0,"-",method) $ method = (method - storage_class) - "-" $ start_hour = f$element(1,"$",storage_class) $ end_hour = f$element(2,"$",storage_class) $ storage_class = f$element(0,"$",storage_class) $ if (start_hour.nes."$").and.(end_hour.nes."$") $ then $ hour = f$integer(f$cvtime(,,"HOUR")) $ start_hour = f$integer(start_hour) $ end_hour = f$integer(end_hour) $ if (((start_hour.gt.end_hour).and. - ((hour.lt.end_hour).or.(hour.gt.start_hour))).or. - ((start_hour.le.end_hour).and. - ((hour.lt.start_hour).or.(hour.gt.end_hour)))) $ then $ say "''f$time()' Deferring ''storage_class' batching for ''node' due to time restrictions ''start_hour'-''end_hour'" $ goto search_1 $ endif $ endif $ else $ storage_class = "BATCH" $ endif $ if f$element(1,"-",method).nes."-" then method = f$element(1,"-",method) $search_3: $ file = f$search(savedfiles,4) $ if file .eqs. "" then goto search_1 $ cfile = f$element(0,";",file) $ versions = f$integer(f$parse(file,,,"VERSION","SYNTAX_ONLY")-";") $ ver = 0 $search_2: ver = ver + 1 $ if (ver.gt.versions) then goto search_3 $ file = cfile + ";" + f$string(ver) $ if f$search(file).eqs."" then goto search_2 $ sayit "''f$time()' ...forwarding items to ''node' via ''storage_class'-''method'" $ sayit = "!" $ on error then goto no_copy_2 $ on warning then goto no_copy_2 $ open/read/error=no_copy_2 access_check 'file' $ if storage_class.eqs."FILELIST" $ then $merge_versions: v1 = ver + 1 $ open/read/error=no_merge access_check_1 'cfile';'f$string(v1)' $ close access_check $ append access_check_1: 'file' $ open/read access_check 'file' $ delete 'cfile;'f$string(v1)' $ ver = ver + 1 $ close access_check_1 $ goto merge_versions $no_merge: $ endif $ goto 'method'METHOD $_METHOD: ! Default (Original ANU NEWS) Method is DECnet $DECNET_METHOD: $ if storage_class.eqs."FILELIST" $ then $ batch_file = file - f$parse(file,,,"VERSION") + "-BATCH" $ batchnews /FILELIST='file' /SIZE=1000000 /BATCHFILE='batch_file' $ copy/log 'batch_file'* 'node'::; $ delete 'batch_file'*;* $ else $ copy/log 'file 'node'::; $ endif $ goto delete_it $UUCP$CMPRS_METHOD: ! Compressed UUCP NEWS batch $ compressed = "YES" $UUCP_METHOD: ! Uncompressed UUCP NEWS batch $ call uucp_disk_check "''file'" "''compressed'" "''node'" "''storage_class'" "''start_hour'" "''end_hour'" $ batch_file = file - f$parse(file,,,"VERSION") + "-BATCH" $ if (storage_class.eqs."FILELIST") $ then $ batchnews /FILELIST='file' /MAX_SIZE='total_batch_size' /SIZE='batch_size' /BATCHFILE='batch_file' $ else $ copy 'file' 'batch_file' $ endif $ if compressed then compress -fzpv 'batch_file'* $ uux_rnews -j 'node' 'batch_file'* $ delete 'batch_file'*;* $ goto delete_it $COMMAND_METHOD: ! User Supplied Command file to effect delivery $ command = f$search(f$parse("NEWS_TRANSPORT.COM",,file)) $ if command.eqs."" then command = f$search("NEWS_MANAGER:NEWS_TRANSPORT_''node'.COM") $ if command.eqs."" then command = f$search("NEWS_MANAGER:NEWS_TRANSPORT.COM") $ if f$search(command).nes."" $ then $ if file.nes.(file-"]NEWS_TRANSPORT.COM;") then goto no_copy_2 $ if storage_class.eqs."FILELIST" $ then $ batch_file = file - f$parse(file,,,"VERSION") + "-BATCH" $ batchnews /FILELIST='file' /SIZE=1000000 /BATCHFILE='batch_file' $ @'command' 'node' 'batch_file'* $ delete 'batch_file'*;* $ else $ @'command' 'node' 'file' $ endif $ else $ say "No Delivery procedure NEWS_TRANSPORT.COM for ''NODE'" $ say "''file' preserved" $ goto no_copy_2 $ endif $ goto delete_it $MAIL_METHOD: ! Outgoing mail based feeds $ mail/subject=newsfeed/noedit/noself 'file' 'node' $ say "''f$time()' News batch mailed to ''node'" $ goto delete_it $MLIST_METHOD: ! Outgoing news gateway to mailing lists $ if (f$search("uucp_bin:gatenews.exe").nes."") .and. (f$search("mx_exe:mx_site_in.exe").nes."") $ then $ gatenews 'file' 'f$trnlnm(node)' 'f$trnlnm(node)' $ else $ say "No news/mlist gateway (uucp_bin:gatenews.exe or mx_exe:mx_site_in.exe) found" $ say "''file' preserved" $ goto no_copy_2 $ endif $ goto delete_it $UUEMAIL_METHOD: ! Outgoing mail based feeds with EMBL UUEncoding $ if f$search("uucp_bin:newsuue.exe").nes."" $ then $ newsuue 'file' $ mail/subject=news_uue_feed/noedit/noself uuenews.uue 'node' $ del uuenews.uue;* $ say "UUENCODEd News batch mailed to ''node'" $ else $ say "No EMBL-UUencoder (uucp_bin:newsuue.exe) found" $ say "''file' preserved" $ goto no_copy_2 $ endif $ goto delete_it $delete_it: $ delete 'file $no_copy_2: $ close/nolog access_check $ goto search_2 $transport_done: $ if p1.nes."" then goto fin $ say "''f$time()' Item forwarding scan complete" $ ! $ ! $map_process: $ ! $ ! This section deals with the special case news group comp.mail.maps $ ! and assumes that an entry exists in the NEWS.SYS file of the following $ ! form: $ ! $ ! mapsink:world,na,usa,comp.mail.maps:B:NEWS_MANAGER_DEV/[UUCP_MAPS]mapsaver^M $ ! $ ! A related entry must exist in NEWS.DISTRIBUTION: (See the comments in $ ! both files) $ ! $ ! node_we_get_maps_from mapsink $ ! $ ! If no such entries exist, then this code is a noop. $ ! $ ! We only do the maps once a day, since rebuilding paths. is such a big $ ! job; it seems silly to do it when more maps might arrive later the same $ ! day. If maps arrive after the day's paths. our paths. will be at most $ ! a day "old". Since it takes days or weeks for new map entries to filter $ ! through rutgers, to the regional map coordinators, back to rutgers, and $ ! finally through innumerable uucp news links to us, this is a non-problem. $ ! $ ! new_day was set earlier, when we decided whether or not to do the $ ! NEWS SKIM operation. $ $ if .not. new_day then goto map_done $ map_mail_subject = "" $ ! $ ! any maps to do? $ ! $ if f$search("NEWS_MANAGER_DEV:[UUCP_MAPS]mapsaver.*").eqs."" - then goto map_cleanup $ ! $ say "''f$time()' Processing New UUCP Map Entries" $ ! $ ! Lets first make sure we have enough disk space to complete this function $ ! $ new_maps_size = 0 $map_size: $ mapfile = f$search("NEWS_MANAGER_DEV:[UUCP_MAPS]mapsaver.*;*",1) $ if mapfile .eqs. "" then goto map_size_done $ new_maps_size = new_maps_size + f$file_attributes(mapfile, "ALQ") $ goto map_size $map_size_done: $ space_required = f$integer(f$trnlnm("UUCP_MINIMUM_DISKSPACE")) $ if space_required.eq.0 then space_required = 5000 $ spool_space = f$getdvi("UUCP_SPOOL","FREEBLOCKS")-space_required $ paths_size = 4000 ! Assume big $ if f$search("UUCP_DATA:PATHS.").nes."" then - paths_size = f$file_attributes("UUCP_DATA:PATHS.","ALQ") $ if (F$GETDVI("UUCP_SPOOL","DEVNAM").eqs.F$GETDVI("NEWS_MANAGER_DEV","DEVNAM")) $ then $ if (2*new_maps_size .gt. spool_space) then goto map_disk_full $ if (paths_size .gt. spool_space) then goto map_disk_full $ else $ if (f$getdvi("NEWS_MANAGER_DEV","FREEBLOCKS").lt.new_maps_size) - then goto map_disk_full $ if (spool_space .lt. (new_maps_size + paths_size)) - then goto map_disk_full $ endif $ on warning then goto map_parse_error $ ! $ ! Extract the newly arrived newsgroup entries (comp.mail.maps) into $ ! the UUCP_PATH_DATA directory. $ ! $ create nla0: $ mapsaver :== $uucp_bin:mapsaver $ mapsaver -vdnews_manager_dev:[uucp_maps] news_manager_dev:[uucp_maps]mapsaver.*;* $ delete news_manager_dev:[uucp_maps]mapsaver.*;* $ copy news_manager_dev:[uucp_maps]*.* uucp_path_data: $ delete news_manager_dev:[uucp_maps]*.*;* $ on warning then goto map_done $ ! $ ! Rebuild the PATHS database considering the new map entry data $ ! just provided. $ ! $ say "''f$time()' ...starting MAKEPATHS" $ @uucp_bin:makepaths $ say "''f$time()' ...MAKEPATHS complete" $ ! what I really want to do is DIR /excl=;0, but that never finds ANY $ ! files for some reason. $ directory/outp=news_log:temp.txt uucp_data:paths,uucp_path_data:*.*;-1,;-2,;-3,;-4,;-5,;-6/dates/size $ set noon $ assign/user nla0: sys$output $ assign/user nla0: sys$error $ if f$search("uucp_log:pathalias.log").nes."" $ then $ copy uucp_log:pathalias.log news_log:temp.txt $ append news_log:temp.txt;-1 news_log:temp.txt $ delete news_log:temp.txt;-1,uucp_log:pathalias.log;* $ endif $ create nla0: ! Get rid of stray sys$output and sys$error redirection $ set on $ map_mail_subject = "Obsolete uucp maps; purge after inspection" $ $map_cleanup: $ ! adjust retention period of map entries in the newsgroup so they'll be $ ! deleted by tomorrow's SKIM. (They arrive with an Expires: header and will $ ! stick around for a long time otherwise, and they've already been $ ! "forwarded" to the map collection mechanism, and we don't need TWO $ ! copies of the maps!) We intentionally do this here, under control $ ! of the "on warning then goto map_done", so that if there are any $ ! problems in the map processing we won't delete any maps from the newsgroup $ ! until after the problems are fixed. $ $ if f$search("news_device:[comp.mail.maps]*.itm") .nes. "" $ then $ news/noscreen select comp.mail.maps set item/hold=1 * exit $ if $status.eqs.SS$_DEADLOCK $ then $ say "NEWS Processing is LOCKED - Waiting 10 minutes $ wait 00:10:00 $ goto map_cleanup $ endif $ endif $ goto map_mail_results $ $map_parse_error: $ $ say "''f$time()' Error Processing Newly Arrived Maps" $ map_mail_subject = "Error Processing Newly Arrived Maps" $ create news_log:temp.txt An error occured while extracting newly arrived maps prior to passing them to pathalias. This usually happens when a map arrives from the net in a mangled form. You should review the NEWSSKIM log for this time period, and examine the possibly malformed map indicated in the log. Usually, you may simply delete the offending map NEWS_MANAGER_DEV:[UUCP_MAPS]MAPSAVER.TXT file and let the normal nightly processing of newly arrived maps process the remaining maps at that time. $ goto map_mail_results $ $map_disk_full: $ directory/size=all/date/output=news_log:temp.txt NEWS_MANAGER_DEV:[UUCP_MAPS] $ say "''f$time()' ***** Deferring processing maps due to insufficient disk space" $ map_mail_subject = "Processing of the following UUCP maps has been deferred until tomorrow due to insufficient disk space" $ $map_mail_results: $ if (f$search("news_log:temp.txt").nes."").and.(map_mail_subject.nes."") $ then $ mail news_log:temp.txt uucp_postmaster /subject="''map_mail_subject'" $ delete news_log:temp.txt; $ endif $ $map_done: $ $ ! --------------------- $ ! Check for errors in yesterday's rnews batch log files, mail results to $ ! local news manager $ ! $ if (new_day) $ then $ set noon $ define/user sys$error nl: $ define/user sys$output nl: $ search news_log:rnews*.log/since=yesterday/before=today - "cannot open", "error:" - /output=news_log:temp.txt $ if $status .eq. 1 $ then $ mail news_log:temp.txt usenet - /subject="Errors adding yesterday's news batches" $ endif $ if f$search("news_log:temp.txt;").nes."" then delete news_log:temp.txt; $ endif $ $ ! $ ! Report on the activity and "quality" of your news feed(s). $ ! $ $ do_feedcheck = f$trnlnm("NEWS_NEWSSKIM_FEEDCHECK") $ if do_feedcheck.eqs."" then do_feedcheck = "TRUE" $ if (do_feedcheck.and.new_day.and.(f$search("uucp_bin:feedcheck.exe").nes."")) $ then $ say "''f$time()' Analyzing NEWS Log files" $ set noon $ feedcheck = "$uucp_bin:feedcheck" $ feedfiles = "news_log:NEWSSKIM.*;* " $ if f$search("news_log:rnews*.*;*").nes."" $ then $ feedfiles = feedfiles + ",news_log:rnews*.*;*" $ endif $ feedcheck/Since=YESTERDAY/Before/Output=news_log:temp.txt 'feedfiles' $ if f$file_attributes("news_log:temp.txt","EOF").gt.0) $ then $ feed_date = f$element(0," ",f$cvtime("YESTERDAY","ABSOLUTE")) $ mail news_log:temp.txt - /subject="News Feed Analysis for ''feed_date'" - usenet $ endif $ delete news_log:temp.txt; $ endif $ ! $ ! delete log files that are more than four days old. $ ! $ delete_files = 0 $ delete_total = 0 $ delete_age = f$trnlnm("NEWS_NEWSSKIM_LOG_LIFETIME") ! age in days $ if delete_age.eqs."" then delete_age = "4" ! 4 day default $ if delete_age.lts."2" then delete_age = "2" ! 2 day minimum $ delete_age = "-"+delete_age+"-00:00:00" $rnews_loop: $ rnews_file = f$search("news_log:rnews*.*;*") $ if rnews_file.nes."" $ then $ if (f$cvtime(f$file_attributes(rnews_file,"CDT")).les. - f$cvtime(delete_age)) $ then $ delete_files = delete_files + 1 $ delete_total = delete_total + f$file_attributes(rnews_file,"ALQ") $ delete/log 'rnews_file' $ endif $ goto rnews_loop $ endif $newsskim_loop: $ newsskim_file = f$search("news_log:newsskim.*;*") $ if newsskim_file.nes."" $ then $ if (f$cvtime(f$file_attributes(newsskim_file,"CDT")).les. - f$cvtime(delete_age)) $ then $ delete_files = delete_files + 1 $ delete_total = delete_total + f$file_attributes(newsskim_file,"ALQ") $ delete/log 'newsskim_file' $ endif $ goto newsskim_loop $ endif $ if delete_files.gt.1 then say "%DELETE-I-TOTAL, ''delete_files' files deleted (''delete_total' blocks)" $ $ ! --------------------- $ ! If it's close to the end of the month, run the arbitron and inpaths $ ! programs and mail the results to decwrl. $ do_arbitron = f$trnlnm("NEWS_NEWSSKIM_ARBITRON") $ if do_arbitron.eqs."" then do_arbitron = "FALSE" $ if .not.(new_day .and. do_arbitron .and. (f$search("uucp_bin:arbitron.exe").nes."")) $ then goto arbitron_inpaths_done $ endif $ $ host = f$logi("uucp_host_name") $ this_mo = f$cvtime(,"ABSOLUTE","MONTH") $ mmm_yyyy = f$edit(this_mo,"LOWERCASE")+" "+f$cvtime(,"ABSOLUTE","YEAR") $ $ ! Are we close enough to the end of the month? Find out how far away $ ! decwrl is (in hops), assume one day per hop, and see if that many $ ! days added to the current time would take us into the next month. $ $ uupath = "$uucp_bin:checkaddr" $ define/user sys$output temp.txt $ uupath foo@decwrl.dec.com $ if uucp_path_length .eq. 0 ! this symbol is set by uupath $ then $ day_of_month_ok = 0 ! no path to decwrl, can't do this stuff $ else $ ndays = f$string(uucp_path_length) ! assume one day per hop $ day_of_month_ok = this_mo .nes. - f$cvtime("+''ndays'-0:0:0", "ABSOLUTE", "MONTH") $ endif $ $ ! Have we done arbitron this month already? $ $ if (f$search("news_log:arbitron.out").eqs."") $ then $ ! ok if haven't done it before, at all. $ month_ok = 1 $ else $ ! ok if last month run is different from current month. $ month_ok = this_mo .nes. - f$cvtime(f$file("news_log:arbitron.out","CDT"),"ABSOLUTE","MONTH") $ endif $ $ if (day_of_month_ok .and. month_ok) $ then $ say "''f$time()' Arbitron Beginning" $ run uucp_bin:arbitron $ backup/delete arbitron.out news_log:/new_version $ purge news_log:arbitron.out $ rename news_log:arbitron.out ;1 $ mail news_log:arbitron.out - /subj="Arbitron stats from site: ''host' for ''mmm_yyyy'" - uucp%"""netsurvey@decwrl.dec.com""",usenet $ say "''f$time()' Arbitron Completed" $ endif $ $ ! Pretty much the same for inpaths. File handling is slightly different. $ ! Have we done inpaths this month already? $ $ do_inpaths = f$trnlnm("NEWS_NEWSSKIM_INPATHS") $ if do_inpaths.eqs."" then do_inpaths = "FALSE" $ if (do_inpaths .and. new_day .and. (f$search("uucp_bin:inpaths.exe").nes."")) $ then $ if (f$search("news_log:inpaths.out").eqs."") $ then $ ! ok if haven't done it before, at all. $ month_ok = 1 $ else $ ! ok if last month run is different from current month. $ month_ok = this_mo .nes. - f$cvtime(f$file("news_log:inpaths.out","CDT"),"ABSOLUTE","MONTH") $ endif $ $ if (day_of_month_ok .and. month_ok) $ then $ say "''f$time()' Inpaths beginning" $ inpaths := $uucp_bin:inpaths $ define/user sys$output news_log:inpaths.out $ inpaths 'host' $ purge news_log:inpaths.out $ mail/subject="INPATHS report from site: ''host' for ''mmm_yyyy'" - news_log:inpaths.out uucp%"""pathsurvey@decwrl.dec.com""",usenet $ say "''f$time()' Inpaths completed" $ endif $ endif $arbitron_inpaths_done: $ $ ! $ ! On Sunday Mornings, we clean up the index file structures by rebuilding them. $ ! $ ! never do this except on the first run of the day. $ ! force it to be done if we haven't done it for three weeks (in case $ ! system has been down on weekends, or something). $ ! under normal conditions, do this on the first run on sunday . $ $ if ( (f$cvtime(f$file_attributes("news_root:news.items","CDT")).gts. - f$cvtime("-21-00:00:00")) .and. - (f$cvtime("TODAY",,"WEEKDAY") .nes. "Sunday") ) .or. (.not.new_day) $ then goto fin $ endif $ newsshutdown = f$search("UUCP_BIN:NEWSSHUTDOWN.EXE") $ if newsshutdown.eqs."" then newsshutdown = f$search("UUCP_BIN:NEWSSHUTDOWN.EXE") $ if newsshutdown.nes."" $ then $ old_privs = f$setprv("SYSLCK") $ if .not.f$privilege("SYSLCK") $ then $ say "''f$time()' SYSLCK privilege required to ensure database consistency $ say "''f$time()' Continuing anyway . . ." $ goto do_rebuild $ endif $ define/user NEWS_LOCKED_COMMAND "@''f$environment("PROCEDURE")' do_rebuild ''f$verify()'" $ run 'newsshutdown $ old_privs = f$setprv(old_privs) $ goto fin $ endif $do_rebuild: $ say "''f$time()' Starting rebuild of indexed files" $ convert = "convert" $ if f$search("UUCP_BIN:NEWS_SITE_REBUILD.COM") $ then ! If a site specific rebuild procedure exists then use it $ set noon $ @UUCP_BIN:NEWS_SITE_REBUILD $ goto rebuild_done $ endif ! else take the default rebuild action. $ on warning then goto groups_failed $ convert/reclaim/statistics news_root:news.groups $ goto do_items $groups_failed: $do_items: $ on warning then goto items_failed $ analyze/rms/fdl/output=news_root:newsitems.fdl news_root:news.items $ edit/fdl/analyze=news_root:newsitems.fdl/nointer news_root:newsitems.fdl $ convert/statistics/fdl=news_root:newsitems.fdl - news_root:news.items news_root:news.items_new $ rename news_root:news.items_new news_root:news.items; $ purge news_root:news.items $ delete news_root:newsitems.fdl;* $ goto do_history $items_failed: $ if f$search("news_root:news.items_new").nes."" then - delete/log news_root:news.items_new;* $ if f$search("news_root:news.fdl").nes."" then - delete/log news_root:news.fdl;* $ purge/log news_root:news.items $do_history: $ on warning then goto history_failed $ if f$search("news_root:history.v60").eqs."" then goto history_failed $ analyze/rms/fdl/output=news_root:history.fdl news_root:history.v60 $ edit/fdl/analyze=news_root:history.fdl/nointer news_root:history.fdl $ convert/statistics/fdl=news_root:history.fdl - news_root:history.v60 news_root:history.v60_new $ rename news_root:history.v60_new news_root:history.v60; $ purge news_root:history.v60 $ delete news_root:history.fdl;* $ goto rebuild_done $history_failed: $ if f$search("news_root:history.v60_new").nes."" then - delete/log news_root:history.v60_new;* $ if f$search("news_root:history.fdl").nes."" then - delete/log news_root:history.fdl;* $ purge/log news_root:history.v60 $rebuild_done: $ say "''f$time()' Indexed file rebuild complete" $ if p1.nes."" then goto fin $ ! $abort: $fin: $ ! $ ! Normal and error exit - schedule next run $ ! $ ! only resubmit actual NEWSSKIM batch jobs $ ! $ if f$environment("DEPTH").ne.0 .or. p1.nes."" then goto all_done $ ! $ ! Rename our log file to NEWSSKIM.LOG_mmdd-hh to avoid large version numbers $ ! $ last_time = f$cvtime(f$file("news_manager:last_skim.time","CDT"),"ABSOLUTE") $ log = f$cvtime(last_time,,"MONTH")+f$cvtime(last_time,,"DAY")+"-"+f$cvtime(last_time,,"HOUR") $ rename news_log:newsskim.log;0 news_log:newsskim.log_'log';0 $ ! $ ! This code will resubmit with a 1-hour delay. $ ! $ Next_Time = f$cvtime("''start_time'+01:00:00","ABSOLUTE") $ submit/noprint/log=NEWS_LOG:/priority='f$getsyi("MAXQUEPRI")' - /queue=NEWS_BATCH_QUEUE/after="''Next_Time'" - uucp_bin:newsskim.com $all_done: $ if f$trnlnm("LAST_SKIM").nes."" $ then $ close/nolog last_skim $ purge news_manager:last_skim.time $ rename news_manager:last_skim.time ;1 $ endif $ set process/name="''old_prcnam'" $ exit $ $uucp_disk_check: subroutine $ space_required = f$integer(f$trnlnm("UUCP_MINIMUM_DISKSPACE")) $ if space_required.eq.0 then space_required = 5000 $ spool_space = f$getdvi("UUCP_SPOOL","FREEBLOCKS")-space_required $ if (p4.eqs."FILELIST") $ then $ uustat := $uucp_bin:uustat $ UUCP_SITE_BYTES_QUEUED_'p3' = 0 $ UUCP_SITE_JOB_AGE_'p3' = 0 $ UUCP_SITE_CONNECT_STATUS_'p3' = 0 $ uustat -m -q -s 'p3' >NLA0: $ current_usage = (f$integer(UUCP_SITE_BYTES_QUEUED_'p3')+511)/512 $ batch_size = f$integer(f$trnlnm("NEWS_SITE_UUCP_BATCH_SIZE_''p3'")) $ if batch_size.le.0 then batch_size = f$integer(f$trnlnm("NEWS_DEFAULT_UUCP_BATCH_SIZE")) $ if batch_size.le.0 then batch_size = 500000 $ batch_size == batch_size $ limit = f$integer(f$trnlnm("UUCP_SITE_QUEUE_LIMIT_''p3'")) $ if limit.le.0 then limit = f$integer(f$trnlnm("UUCP_SITE_QUEUE_LIMIT")) $ if limit.le.0 then limit = 14000 ! 1 hour's worth of fast traffic $ if limit.gt.spool_space then limit = spool_space $ limit = limit - current_usage $ if limit.le.0 $ then $ say "''f$time()' Insufficient Disk Space to forward batch ''p1'" $ limit = limit + current_usage $ say "''f$time()' ''current_usage' blocks already queued, ''limit' blocks allowed" $ exit %x10000002 $ endif $ age = f$integer(UUCP_SITE_JOB_AGE_'p3')/60 $ if ((p5.eqs."$").and.(p6.eqs."$").and.((age/60).ge.2).and. - (9.ne.f$integer(UUCP_SITE_CONNECT_STATUS_'p3')).and. - (4.ne.f$integer(UUCP_SITE_CONNECT_STATUS_'p3'))) $ then $ say "''f$time()' Jobs more than 2 hours old exist, batch forwarding deferred" $ say "''f$time()' ''current_usage' blocks already queued, Oldest job is ''age' minutes" $ exit %x10000002 $ endif $ if (p2) $ then !