$ !							NEWS_RNEWS.COM
$ GOTO START		! 'f$verify(f$trnlnm("NEWS_RNEWS_VERIFY"))'
$ !
$ !	This command procedure executes in an independent batch process
$ !	usually initiated by UUXQT_DCL.COM which is running as a spawned
$ !	subprocess of UUXQT.EXE .
$ !	
$ !	P1	- the input file spec of a NEWS batch file to be processed
$ !	P2	- the hostname of the site supplying (this is optional)
$ !
$ !	Author:	Mark Pizzolato     2/5/89	mark@infopiz.UUCP
$ !	Modified:
$ !		Mark Pizzolato	   11/16/92	mark@infocomm.com
$ !		      - Generalized batch file type determination to allow for
$ !			the case where an input news batch file is compressed
$ !			and there are more than about 1000 bytes at the
$ !			beginning of the file without a NEWLINE.  This problem
$ !			was reported along with the fix method by James 
$ !			Thompson (jimmy_t@verifone.com)
$ !		Mark Pizzolato	   11/16/92	mark@infocomm.com
$ !		      - Added a more appropriate explanation in the mail
$ !			message that is generated when the NEWS_DEVICE is 
$ !			too full.
$ !		Mark Pizzolato	   10/12/92	mark@infocomm.com
$ !		      - Downstream storage computations have been adjusted
$ !			to accomodate the new "FILELIST" storage method
$ !			available with ANU-NEWS (V6.1 Beta 6 or later).
$ !		      - NEWS_RNEWS jobs are now submitted at priority
$ !			SYSGEN(MAXQUEPRI)-1, and NEWSSKIM jobs are run at
$ !			SYSGEN(MAXQUEPRI). This eliminates the need
$ !			to run NEWSSKIM at the end of each NEWS_RNEWS job, 
$ !			so this is not done anymore.
$ !		      - If insufficient disk resources to process the current
$ !			job are not available, this job now resubmits itself
$ !			for a 15 minute delay (up to 4 times), and eventually
$ !			will put itself on hold as before.  This allows other
$ !			useful work to get done to hopefully clean things up.
$ !		Mark Pizzolato	   9/19/92	mark@infocomm.com
$ !		      - Added support for batches that may have been compressed
$ !			with the ever popular LZCOMP program.
$ !		Mark Pizzolato	   3/28/92	mark@infocomm.com
$ !		      - Added appropriate code to handle the case when NEWS
$ !			either aborted add processing or couldn't even initiate
$ !			it due to the news system being shutdown via an
$ !			exclusive lock on the news data base files (NEWS_LOCK).
$ !		Mark Pizzolato	   2/16/92	mark@infocomm.com
$ !		      - integrated a few changes found in the ANU-NEWS
$ !			distribution from V6.1-Alpha, and changed f$logi to
$ !			f$trnlnm.
$ !		      - Added code to report Missing Batch files as missing.
$ !		Mark Pizzolato	   2/7/92	mark@infocomm.com
$ !		      - Call NEWSSKIM after processing of the inbound batch.
$ !		Mark Pizzolato	   1/29/92	mark@infocomm.com
$ !		      - Added counting of downstream sites to get a better
$ !			approximation of how much space will be required to
$ !			process the batch for the disk check.
$ !		Mark Pizzolato	   6/3/90	mark@infocomm.com
$ !		      - Added checks for disk space on the disk used for 
$ !			target and workfile processing for news batches.
$ !			If there is not sufficient space, we FIRST try to free
$ !			up some space by executing a NEWSSKIM in line with the
$ !			current procedure.  After the NEWSSKIM, if there is
$ !			still insufficient space, we wait up to an hour to give
$ !			any outbound queues a chance to flush.  If there is
$ !			still insufficient space, we basically flow control 
$ !			things by not processing the current batch and 
$ !			resubmitting it as a held job
$ !		Mark Pizzolato	   4/15/90	mark@infopiz.UUCP
$ !		      - Added optional argument for hostname of the
$ !			site that the batch file being processed originated
$ !			The purpose of this argument is to allow some realistic
$ !			processing of news batch log files to determine the
$ !			"quality" of a site's feed.  This would only be needed
$ !			if a site gets fed by more than one source, and there
$ !			is overlap (my site fits this case, but as more DECUS
$ !			sites start running this stuff, the need for this info
$ !			will grow)
$ !
$ !		Mark Pizzolato	   9/26/89	mark@infopiz.UUCP
$ !		      - Added interpretation of NEWS_AUTO_CREGRP and
$ !			NEWS_KEEP_BATCH logical name flags to assist in
$ !			in debugging and to generalize this procedure.
$ !		      - Merged in changes made by Lenny
$ !			Glassmann (glassmann@ccavax.camb.com) that dealt with
$ !			compressed batch files that don't start with
$ !			"#! cunbatch".
$ !	
$
$ START:
$	infile = p1
$	hostname = p2
$	IF P2.NES."" THEN hostname = "- h:''hostname'"
$	SS$_DEADLOCK = "%X10000E0A"
$	SAY = "WRITE SYS$OUTPUT"
$	SAY "...Processing BATCH ''hostname' starting at ''f$time()', infile ''infile'"
$	SET PROCESS/NAME="RNEWS_''f$element(1,"_",f$getjpi("","PRCNAM"))'"
$	COMPRESS :== $UUCP_BIN:COMPRESS.EXE
$	LZDCMP :== $UUCP_BIN:LZDCMP.EXE
$	NEWS :== $UUCP_BIN:NEWS
$	DEFINE/NOLOG NEWSRC NLA0:
$	MAGIC = "9D1F"
$	MAGIC_LZCOMP = "9E1F"
$	lz_comp = 0
$	pflag = ""
$	batch_file = ""
$
$      	delete_batch = "/DELETE"
$	IF f$trnlnm("NEWS_KEEP_BATCH") THEN delete_batch = ""
$	execqual = ""
$	IF (f$trnlnm("NEWS_EXECUTE_CONTROL")) THEN execqual = "/EXECUTE"
$	cregrpqual = ""
$	IF (f$trnlnm("NEWS_AUTO_CREGRP")) THEN cregrpqual = "/CREGRP"
$
$ CHECK_NEWS_LOCK:
$	IF F$TRNLNM("NEWS_STOP").NES.""
$	THEN
$		SAY "''f$time()'	NEWS Processing is LOCKED(''F$TRNLNM("NEWS_STOP")') - Waiting 10 minutes
$		WAIT 00:10:00
$		GOTO CHECK_NEWS_LOCK
$	ENDIF
$	SET PROTECTION=(OWNER:RWED,GROUP:RWED,SYSTEM:RWED,WORLD:R)/DEFAULT
$	IF F$SEARCH(infile).EQS."" THEN GOTO MISSING_NEWSBATCH
$	ON WARNING THEN GOTO NOT_NEWSBATCH
$	work_file = "NEWS_TEMP.BATCH"+F$GETJPI("","PID")
$	DUMP/WORD/RECORDS=(START=1,END=1)/OUTPUT='work_file' 'infile'
$	OPEN/READ TEMP 'work_file'
$	READ/END=NEWS_PROBLEM/ERROR=NEWS_PROBLEM TEMP LINE
$	READ/END=NEWS_PROBLEM/ERROR=NEWS_PROBLEM TEMP LINE
$	READ/END=NEWS_PROBLEM/ERROR=NEWS_PROBLEM TEMP LINE
$	READ/END=NEWS_PROBLEM/ERROR=NEWS_PROBLEM TEMP LINE
$	READ/END=NEWS_PROBLEM/ERROR=NEWS_PROBLEM TEMP LINE
$	READ/END=NEWS_PROBLEM/ERROR=NEWS_PROBLEM TEMP LINE
$	READ/END=NEWS_PROBLEM/ERROR=NEWS_PROBLEM TEMP LINE
$	CLOSE TEMP
$	DELETE 'work_file';*
$	IF F$EXTRACT(81,11,LINE)  .EQS."#! cunbatch" THEN GOTO DO_COMPRESSED_UNBATCH
$	IF F$EXTRACT(76, 4, LINE) .EQS. MAGIC THEN GOTO DO_COMPRESSED_BATCH
$	IF F$EXTRACT(76, 4, LINE) .EQS. MAGIC_LZCOMP
$	THEN
$		lz_comp = 1
$		GOTO DO_COMPRESSED_BATCH
$	ENDIF
$ DO_UNCOMPRESSED:
$	CALL DISK_CHECK	"''p1'" "''p2'" NO ! Check for Sufficient Disk Space
$	NEWS/NOSCREEN ADD FILE/NETFEED'execqual''cregrpqual' 'infile' 'delete_batch'
$	return_status == $status
$	GOTO RETURN_IT
$
$ DO_COMPRESSED_UNBATCH:
$	pflag = "p"
$ DO_COMPRESSED_BATCH:
$	CALL DISK_CHECK	"''p1'" "''p2'" YES ! Check for Sufficient Disk Space
$	batch_file = "NEWS.BATCH"+F$GETJPI("","PID")
$	SAY "''f$time()'	File is compressed, decompressing."
$	IF lz_comp
$	THEN
$		LZDCMP 'infile' 'batch_file'
$	ELSE
$		COMPRESS -dzc'pflag' 'infile' >'batch_file'
$	ENDIF
$	ON WARNING THEN GOTO NEWS_PROBLEM
$	NEWS/NOSCREEN ADD FILE/NETFEED'execqual''cregrpqual' 'batch_file' 'delete_batch'
$	DELETE 'infile'
$	return_status == $status
$	GOTO RETURN_IT
$
$ NEWS_PROBLEM:
$	return_status == $status
$	IF F$SEARCH(batch_file).NES."" THEN DELETE 'batch_file';
$	IF return_status .eqs. SS$_DEADLOCK
$	THEN
$		SAY "''f$time()'	NEWS Processing is LOCKED - Waiting 10 minutes
$		WAIT 00:10:00
$		GOTO CHECK_NEWS_LOCK
$	ENDIF
$	SET NOON
$ 	MAIL NL: usenet/subject="NEWS_RNEWS - trouble inserting Batch File: ''infile'"
$	SAY "''f$time()'	...''infile' Couldn't be processed by NEWS!"
$	IF F$SEARCH(work_file).NES."" THEN DELETE 'work_file';*
$	GOTO RETURN_IT
$
$ NOT_NEWSBATCH:
$	return_status == $status
$	IF return_status .eqs. SS$_DEADLOCK
$	THEN
$		SAY "''f$time()'	NEWS Processing is LOCKED - Waiting 10 minutes
$		WAIT 00:10:00
$		GOTO CHECK_NEWS_LOCK
$	ENDIF
$show sym return_status
$ 	MAIL NL: usenet/subject="NEWS - Not a News Batch File: ''infile'"
$	SAY "''f$time()'	...''infile' is not a Compressed News Batch File!"
$	GOTO RETURN_IT
$
$ MISSING_NEWSBATCH:
$	return_status == $status
$ 	MAIL NL: usenet/subject="NEWS - Missing Batch File: ''infile'"
$	SAY "''f$time()'	...''infile' is Missing!"
$
$ RETURN_IT:
$!
$! NEWSSKIM runs at a higher job scheduling priority than the NEWS_RNEWS add
$! processing, this allows outflow processing to down stream sites at 
$! reasonable intervals (this defaults to once per hour, but that is dealt
$! with directly within NEWSSKIM).  
$!
$	EXIT 'return_status'
$
$ DISK_CHECK: SUBROUTINE
$	batch_blocks = F$FILE_ATTRIBUTES(p1,"ALQ")
$!
$! There are definately some worse case assumptions being made 
$! here, but the whole point of this exercise is to make sure
$! we never overfill the disk, so we always want to error conservatively.
$! We assume that every incoming batch will actually be fed to every site 
$! that we have a NEWS_MANAGER_DEV:[POST*] directory for.  This is in general
$! never true, since a properly configured system should never send the
$! contents of the arriving batch back to the site it arrived from, but there
$! are other cases that this assumption compensates for, namely extra copies of
$! things that go elsewhere like the uucp map processing. 
$! We also assume that an arriving batch that is uncompressed ends up with a 
$! resulting disk usage of twice the batch size when uncompressed.  Reasonable
$! observations indicate that this might actually be about 5/3 times, but 2
$! times errors on the conservative side.  Compressed batches are assumed to
$! take twice the space of their uncompressed form, also accounting for the 
$! temporary uncompressed copy which needs to exist during the processing.
$!
$	downstream = 0
$ COUNT_DOWNSTREAM:
$	site = f$search("NEWS_MANAGER_DEV:[000000]POST*.DIR;1",1)
$	IF (site.NES."")
$	    THEN
$		node = f$parse(site,,,"NAME","SYNTAX_ONLY") - "POST_"
$		method = f$element(0,"_",node) + "_"
$		node = node - method
$		prio = f$string(f$integer(f$element(0,"-",method)))
$		if (prio.eqs.f$element(0,"-",method)) then method = (method - prio) - "-"
$		if f$element(1,"-",method).nes."-"
$		    then
$			storage_class = f$element(0,"-",method)
$			method = (method - storage_class) - "-"
$			storage_class = f$element(0,"$",storage_class)
$		    else
$			storage_class = "BATCH"
$		endif
$		if f$element(1,"-",method).nes."-" then method = f$element(1,"-",method)
$		if storage_class.eqs."BATCH" then downstream = downstream + 1
$		GOTO COUNT_DOWNSTREAM
$	ENDIF
$	IF (p3)
$	    THEN	! Peak Compressed usage
$		required_blocks = (batch_blocks * 2) * (1 + downstream + 2) 
$	    ELSE	! Peak Uncompressed usage
$		required_blocks = batch_blocks * (downstream + 2)
$	ENDIF
$	news_space = F$GETDVI("NEWS_DEVICE","FREEBLOCKS")
$	IF required_blocks.LE.news_space THEN EXIT
$!
$!
$	JOB_P1 = f$getqui("DISPLAY_ENTRY", "PARAMETER_1", "*", "THIS_JOB")
$	JOB_P2 = f$getqui("DISPLAY_ENTRY", "PARAMETER_2", "*", "THIS_JOB")
$	JOB_P3 = f$getqui("DISPLAY_ENTRY", "PARAMETER_3", "*", "THIS_JOB")
$	JOB_P3 = f$integer(JOB_P3) + 1
$	JOB_P4 = f$integer(JOB_P4)
$!
$! Not enough space yet, wait for up to an hour for any output queues to 
$! flush.  Then hold the job and send mail to that effect.
$!
$	IF JOB_P3.LE.5
$	    THEN
$		SAY "''f$time()'	News Device too Full, Waiting 15 minutes for NEWSSKIM and downstream queues to flush"
$		after_time = "/AFTER=""+00:15:00"""
$	    ELSE
$	 	MAIL SYS$INPUT: usenet/subject="News Device too Full, Suspended processing of NEWS Batch ''p1' from ''p2'"
Processing of the inbound NEWS batch mentioned in the subject of this mail
message has been suspended due to insufficient space on the NEWS_DEVICE.

You may optionally free disk space on the disk specified by NEWS_DEVICE and
release any NEWS_RNEWS jobs that are on the NEWS_BATCH queue in the "Holding"
state.  This is done with the following commands:

	$ SHOW QUEUE/BATCH NEWS_BATCH
	$ SET ENTRY/RELEASE entry_number[,...]

Without any action on your part, this job will automatically be restarted 
after the next nightly NEWSSKIM job.

$		JOB_P3 = 0
$		JOB_P4 = JOB_P4 + 1
$		SAY "''f$time()'	News Device too Full, Suspended processing of NEWS Batch ''p1' from ''p2'"
$		after_time = "/HOLD"
$	ENDIF
$ RESUBMIT_IT:
$	batch_log = "NEWS_LOG:RNEWS_"+(f$parse(p1,,,"TYPE")-".")
$	rnews_job_prio = f$getsyi("MAXQUEPRI")-1
$	SUBMIT/NOPRINT/LOG='batch_log''after_time'			-
	       	/PARAMETER=("''JOB_P1'","''JOB_P2'","''JOB_P3'","''JOB_P4'")	-
		/QUEUE=NEWS_BATCH_QUEUE					-
		/PRIORITY='rnews_job_prio'				-
		UUCP_BIN:NEWS_RNEWS	
$	LOGOUT/FULL
$	ENDSUBROUTINE