#!/usr/local/bin/perl5

$testTape = 0;   # = 0 ignore test run (=1000)
                 # = 1 use test run only

# taper.pl
# Hiroyuki Sako, GSI
# 2000 version of taping command
#           31.07.00
# from controlTapeDaemon.pl
# 13.10.99
# first complete version
# ver1.1 -- 18.10.99
#      H. Sako
#
#"Catherine CHARBONNIER" <Catherine.Rafflin@cern.ch> wrote:
#Na45 PC disk servers can talk to the following tape servers:
# shd66 shd67 shd68 shd69 shd73 shd74 shd75
#Each tape server has 4 drives.
#
#Jean-Philippe Baud<tape.support@cern.ch> wrote:
#
#mount time (robot) 15 to 20 seconds
#load time (for the tape to become ready) 45 to 50 seconds
#positionning time 50s in average
#rewind/unload > 60s
#dismount time 15 to 20 seconds.
#If the tape is already mounted and positionned to one file,
#positionning to the next file is immediate.
#That's why it is always good to put several files in a tpwrite.
#But 2 other parameters must be taken ito account:
#flush buffer + write tapemark at end of each f3ile: 10 to 12 seconds
#In the new software being developped "CASTOR" to replace "SHIFT"
#the mount, open and transfers are overlapping, so we get much better
#throughput.
#Best regards.
#                   Jean-Philippe Baud
#executing command: '/usr/local/bin/tpread -v Y27096 -F F -L 32768 -g SD3R -l al -f temp1.dat:temp2.dat:temp3.dat:temp4.dat:temp5.dat -q u10 /cdrShare33/cerescdr/cdr/tape/temp.dat > logs/testReadTape.log 2>&1 &' 
#Since this command works. The output of temp.dat is cancatenation
# of temp1.dat-temp5.dat
#I can specify different 

# Usage: controlTapeDaemon.pl [flag]
# flag = 1 (restart = default)
#        0 (initialize)

# Revision History
# 10.08.00 HS
# added control/signal file for "stop", "pause" and "continue"
# of taper process at the beginning of the main loop
# which is done by "signal" subroutine

# 14.08.00 HS
# added $remoteHost argument on dumpTape.pl command
# to make Y****.final list
# finishTape changed accordingly

# Possible tape drive error
# 1) hardware error
# error message
#readlbl: TP042 - /var/tmp/RTCOPY/PATH11108.1 : read error : Hardware error ASC=44 ASCQ=B7
#Oct 19 13:39:29 tpwrite[1577]: command failed
#TP025 - bad label structure
#
# how to recover-> retry command
#readlbl: TP042 - /var/tmp/RTCOPY/PATH5549.1 : read error : Medium error ASC=14 ASCQ=4

# Another error!!
#na45pc12: Host name lookup failure
# when nameserver failed...

# error when machine is down....
# na45pc14.cern.ch: No route to host

# Philosophycal remarks:
# this program tries to record files in the same tape which are created in close time
# at the same time as much as possible.
# Only a set of $NStreamSendCDR files is recorded.
# Thus, only 1 stream is used when the total file size < 50GB
#
# A) When total size of input data is small (< 50GB)
#    the program works only for one stream (= one tape)
#    the recording rate is 7MB/sec = 25GB/hour (without running receiver)
# B) However, as the total size becomes large enough (>50GB),
#    several streams (=several tapes) are used.
#    No. of streams = (total file size to be recorded)/50GB
#    
#    
# Speed remark:
#  average tape writting speed/stream = 7MB/sec (no other job running)
#  maximum tape stream allowed = 14
#  total bandwidth = 70MB/sec (theoretical)
# Requirement:
# 35MB/sec in off-burst (12 sec) (8 sec is off-burst)
# 420MB/20sec = 21 MB/sec
#
# Test result
# 1) 50 GB files with 20GB/tape limit
#    6.4MB/sec (2streams)
# 2) 50 GB files with 10GB/tape limit
#    9.3MB/sec (4streams) 


# debug flag 1 = debug mode
$debug = 0;
# print level
$printLevel = 30; # larger level -> more printout

# 2000 tapes
#    Y28408 - Y28782 (remaining from 1999)
#    Y30373 - Y30408 (36)
#    Y30511 - Y30545 (35)
#    Y33196 - Y33700 (505)               
#    Y30221 - Y30246 (26) for spare and test, SOR etc.


# principles
# 1) only 1 CDR machine for 1 tape (easy to maintain)
# name of the data file (same on disk and tape)
# run%04d-burst%04d.dat 

#files for taping (2000 version)
#       file                                      description
#   lock/taper.lock                         lock file for this process
#   lock/Y*****.lock                        lock file for tapeDaemon
#
#   control/size-na45pc**-Y*****.daemon     tape size (kB) after daemon file is made
#   control/size-na45pc**-Y*****.written    tape size (kB) after tapeDaemon is finished
#   control/signal                          control signal "stop", "pause", or "continue"
#
#   toTape/na45pc**-run*-burst*.link     path and size(KB) of data file (before listing)
#   Checked/na45pc**-run*-burst*.link   path and size(KB) of data file (checked its existence and size>0)
#   Taping/na45pc**-run*-burst*.link     path and size(KB) of data file (after listing and before finishing taping)
#   toClean/na45pc**-run*-burst*.link   path and size(KB) of data file (finished taping)
#
#   queue/na45pc**-Y*****-no****.waiting    daemon file (before submitted)
#   queue/na45pc**-Y*****-no****.submitted  daemon file (after submitted)
#   queue/na45pc**-Y*****-no****.finished   daemon file (after job finished)

# tape list
#   list/tapeSummaryList.dat    Available tape name list and new/old/full/complete status and size only after dumptape command
#   list/YXXXXX.list         file list of tapeDaemon-submitted files
#   list/YXXXXX.dump         list of files with dumptape -V tape(after completion of tape)
#   list/YXXXXX.final        compressed from dump 
$|=1;

$restartFlag = shift if defined @ARGV;

$restartFlag = 1 unless (defined $restartFlag);
#die "$0> usage controlTapeDaemon.pl (restart) (1=default:restart\n"
#    unless defined $restartFlag;

#update on 31.07.00
$ENV{'LD_LIBRARY_PATH'}="/usr/local/Objectivity/5.1.2/linux86/lib:/cdrShare01/cerescdr/lhcxx/specific/Linux/Sept99/lib:/cdrShare01/cerescdr/cdr/lib:/usr/X11R6/lib";

chop($host = `hostname`);

# important global variable
# variables which should not be changed...

$TAPE_ROOT  = "/cdrShare01/cerescdr/cdr2k/tape";
$controlDir = "$TAPE_ROOT/control";
$daemonDir  = "$TAPE_ROOT/queue";
$toTapeDir  = "$TAPE_ROOT/toTape";
$CheckedDir = "$TAPE_ROOT/Checked";
$TapingDir  = "$TAPE_ROOT/Taping";
$toCleanDir = "$TAPE_ROOT/toClean";
$logDir     = "$TAPE_ROOT/logs";
$lockDir    = "$TAPE_ROOT/lock";
$listDir    = "$TAPE_ROOT/list";
$statusDir  = "$TAPE_ROOT/status";

# Possibly changed variables

$tapeSize   = 50*1024*1024; # 50GB, kB unit

# limit on tape size actually written on tape in KB
$limitTapeSize = $tapeSize*0.85;
# But, this is now taken from control/limitTapeSize
$maxNStreamTape = 21; # max No. of tapeDaemon streams 
$maxNStreamTapePerHost = 3; # max No. of streams per host
$maxTapeFail = 10; # max. No. of failures allowed in one tape

#NEVER CHANGE THIS $maxNCDR
$maxNCDR = 7; # maximum No. of CDR machines

@CDRHost = ("na45pc07", "na45pc08", "na45pc09", "na45pc10",
	    "na45pc11", "na45pc12", "na45pc14");
# 04.10.00 change 
# maxNfileJobs from 20 to 40
# minNfileJobs from 10 to 20
$maxNFileJobs = 40; # maximum No. of files per tapeDaemon job. VERY IMPORTANT
                    # if no. files less than this, it will not be taped...

#default
#$minNFileJobs = 20; # minimum No. of files per tapeDaemon job to be submitted
#                    # only when there is no other daemonfiles for each host
#                    # this is important in case of low file rate.

#to tape all after end of run
$minNFileJobs = 1; # minimum No. of files per tapeDaemon job to be submitted
                    # only when there is no other daemonfiles for each host
                    # this is important in case of low file rate.

$checkFileSize = 1; # 1 is default
# if there is an empty file, tpwrite error happens!
# so, it is necessary...

# don't touch sleepTimes these are fine-tuned.
$sleepTime = 40;           # sleep time [second] for updateMasterlist
$sleepTimeTapeDaemon = 20; # sleep time for each tapeDaemon.pl

$freqBackup = 300; # frequency of backup lists...   once in 150 minutes
$freqTapeSize = 2; # frequency to check consistency between YXXXX-current.dat and between tapeSummary.dat
$freqListing  = 1;  # frequency for listing files... once in 60 minutes since 11/27/99

#start. first initializes STDOUT and STDERR
backupLists();
init();

# 1 iter = 1min
# 19 sec X 100 burst (20 burst ID) = 32 min ~ 30 iter!!
# but, anyway at currentset > maxSet. the listing is immediately done...

chdir ($TAPE_ROOT) || die "$0> cannot change to $TAPE_ROOT: $! \n";

# get the first tape
#$currentTape = getNextTape();

#cleanup or restart
$now = `date +"%D %T"`;
chop($now);

if ($restartFlag == 1) {
    print "************* Restarting $0 $now\n";
    restart();
} else {
    print "************* Starting with cleaning $0 $now\n";
    cleanup();
}

if (checkConsistency() == 0) {# consistency check and cleanup of files
  print "inconsistency found in files\n"; 
  print "check files\n";
}

# Infinite loop here
$iter = 0;

# ************* MAIN LOOP *************

use FileHandle;
while (1) {

    $loopStart = time();
# backup of lists...
    if ((($iter % $freqBackup) == 0) && ($iter>0)) {
	print "backing up lists...\n";
	backupLists();
    }

    $now = `date +"%D %T"`;
    chop($now);
#    print "$now\n";

    print "** Loop $iter at $now \n";

    getLimitTapeSize();

    signal();

    updateStatistics();

    if (($iter % $freqTapeSize) == 0) {
	foreach $CDRName (@CDRHost) {
	    $currentTape = getNextTape($CDRName);
	    checkTapeSize($currentTape);
	}
    }


    if ($printLevel > 50) {print "currentTape = $currentTape\n";}

# check data size and its existense and rename from toTape to Checked
    if ($printLevel > 20) {print "Going to checkSize\n";}
    checkSize();

# update file list for all CDR machines under toCheck directory
# and make daemon files
    if ($printLevel > 20) {print "Going to collectFileList\n";}
    if (($iter % $freqListing) == 0) {
	collectFileList();
    }

# submitting command is checked in every iteration
#check nstream
    $NStream = checkNStream(); # No. of tpwrite streams
    if ($printLevel > 20) {print "NStream = $NStream\n";}

    if ($printLevel > 20) {print "Going to updateTapeStatus\n";}

    updateTapeStatus(); # Here, tapeContentList is written


    if ($NStream < $maxNStreamTape) {
	if ($printLevel > 20) {print "Going to submitTapeDaemon\n";}
	
#
# check sortlist and submit the tape job to one of CDR's
	submitTapeDaemon();
    }

    $loopEnd = time();
# try to keep always 1 loop time is ~ $sleepTime
    $loopSleep = $sleepTime - ($loopEnd-$loopStart);
    if ($loopSleep > 0) {
	sleep($loopSleep);
    }
    $iter ++;
}

finish();

exit;


# In 2000 run we don't need to combine all the streams into one set.
# every data file is independent.
# we just need to sort in CDR and run, and split the list into each CDR machine to create daemon file.
# SOR also should be taped only once, treated exactly same as normal data files

sub checkSize {
    $testRun = 1000; # test run

#first check actual size
    if ($printLevel > 40) {
	print "Checking consistency between link file and actual data file...\n";
    }

    if ($checkFileSize == 1) {

#for SOR
	@rawFileList=();
	@rawFileList = <$toTapeDir/na45pc??-run????-sor.link>;
	foreach $toTapeFile (@rawFileList) {
	    if ($toTapeFile =~ /(na45pc\d{2})-run(\d{4})-sor\.link/) {
		$CDR = $1;
		$run = $2;
# hs added on 09.09.00
# if run number = 1000 (for test, just skip it)
		if ($testTape == 0) {
		    if ($run == $testRun) {
			next;
		    }
		}
		else {
		    if ($run != $testRun) {
			next;
		    }
		}
		$checkedFile
		    = sprintf("$CheckedDir/$CDR-run%04d-sor.link", $run);

		open (TOTAPEFILE, $toTapeFile);
		$line = <TOTAPEFILE>;
		close (TOTAPEFILE);

		($path, $size) = split(' ', $line);

		if ($size =~ /(\d+\.?\d*)/) {
		    $size = $1;
		} else {
		    print "$0> ERROR: size is strange = $size\n";
		}

		$checkSize = 0;
		$! = "";
		$fileCheck = `rsh -n $CDR ls -s $path`;

		$timeOut = 0;
		while (($! =~ /Resource temporarily unavailable/)
		       && ($timeOut<10)) {
		    print "$! observed. sleeping...\n";
		    sleep(4);
		    $fileCheck = `rsh -n $CDR ls -s $path`;
		    $timeOut++;
		}

		($checkSize, $checkPath) = split(' ', $fileCheck);
		if ($checkSize eq "") {
		    $checkSize = 0;
		}

		if ($checkSize == 0) {
		    $emptyFile
			= sprintf("$CheckedDir/$CDR-run%04d-sor-empty.link",
				  $run);

		    if ($printLevel > 50) {
			print "Warning: $path doesn't exist or size=0\n";
			print "rename $toTapeFile, $emptyFile\n";
		    }
		
		    rename $toTapeFile, $emptyFile
			|| print "$0> Failed to rename $toTapeFile, $emptyFile\n";
		    next;
		} else {
		    # rename toTape/toTape-* file to toTape-checked-*....
		    if ($printLevel > 50) {
			print "rename $toTapeFile, $checkedFile\n";
		    }
		    rename $toTapeFile, $checkedFile || die "cannot rename $toTapeFile, $checkedFile\n";
		}
	    }
	}
	
	@rawFileList=();
	
	@rawFileList = <$toTapeDir/na45pc??-run????-burst????.link>;

	$iFile = 0;

	foreach $toTapeFile (@rawFileList) {

	    if ($debug == 1) {
		print "toTapeFile = $toTapeFile\n";
	    }
	    if ($toTapeFile =~ /(na45pc\d{2})-run(\d{4})-burst(\d{4})\.link/) {
		$CDR = $1;
		$run = $2;
		$burst = $3;
# 09.09.00
# if run number = 1000 (for test, just skip it)
		if ($testTape == 0) {
		    if ($run == $testRun) {
			next;
		    }
		}
		else {
		    if ($run != $testRun) {
			next;
		    }
		}

		$checkedFile
		    = sprintf("$CheckedDir/$CDR-run%04d-burst%04d.link",
			      $run, $burst);
	    } else {
		next;
	    }
	    
	    open (TOTAPEFILE, "<$toTapeFile");
	    $line = <TOTAPEFILE>;

	    if ($debug == 1) {
		print "line=$line\n";
	    }
	    ($path, $size) = split(' ', $line);
	    close (TOTAPEFILE);
	    if ($size =~ /(\d+\.?\d*)/) {
		$size = $1;
	    }
	    
	    $checkSize = 0;

	    $! = "";
	    $fileCheck = `rsh -n $CDR ls -s $path`;
	    $timeOut = 0;
	    while (($! =~ /Resource temporarily unavailable/)
		   && ($timeOut<10)) {
		print "$! observed. sleeping...\n";
		sleep(4);
		$fileCheck = `rsh -n $CDR ls -s $path`;
		$timeOut++;
	    }


	    ($checkSize, $checkPath) = split(' ', $fileCheck);
	    if ($checkSize eq "") {
		$checkSize = 0;
	    }

	    if ($debug == 1) {
		print "path: $path, checkPath: $checkPath, size: $size, checkSize: $checkSize\n";
	    }
	    if ($checkSize == 0) {
		$emptyFile
		    = sprintf("$CheckedDir/$CDR-run%04d-burst%04d-empty.link",
			      $run, $burst);
		print "Warning: $path doesn't exist or size=0\n";
		print "rename $toTapeFile, $emptyFile\n";
		
		rename $toTapeFile, $emptyFile
		    || print "$0> Failed to rename $toTapeFile, $emptyFile\n";
		next;
	    } else {
		# rename toTape/toTape-* file to toTape-checked-*....
		rename $toTapeFile, $checkedFile || die "cannot rename $toTapeFile, $checkedFile\n";
	    }
	}
    }

    return 1;
}

sub collectFileList {

#search for files newly written by receiver

    @fileList=();

# now data file and SOR file is exactly the same treatment
#    @fileList    = <$CheckedDir/na45pc??-run????-burst????.link>;
    @fileList    = <$CheckedDir/na45pc??-run????-burst????.link $CheckedDir/na45pc??-run????-sor.link>;

    $iFile = 0;

#first collect currently existing toTape files

    @run=();
    @CDR=();
    @burst=();
    @path=();
    @size=();
    @linkLine=();
    @toTapeList=();

    @isSOR = ();

    foreach $toTapeFile (@fileList) {

	if ($debug == 1) {
	    print "toTapeFile = $toTapeFile\n";
	}
	if ($toTapeFile =~ /(na45pc\d{2})-run(\d{4})-burst(\d{4})\.link/) {
	    $CDR[$iFile] = $1;
	    $run[$iFile] = $2;
	    $burst[$iFile] = $3;
	    $isSOR[$iFile] = 0;
	} elsif ($toTapeFile =~ /(na45pc\d{2})-run(\d{4})-sor\.link/) {
	    $CDR[$iFile] = $1;
	    $run[$iFile] = $2;
	    $burst[$iFile] = -1;
	    $isSOR[$iFile] = 1;
	} else {
	    next;
	}
	if ($debug == 1) {
	    print "CDR = $CDR[$iFile], run = $run[$iFile], burst = $burst[$iFile]\n";
	}
	open (TOTAPEFILE, $toTapeFile);
	$line = <TOTAPEFILE>;
	$linkLine[$iFile] = $line;
	$toTapeList[$iFile] = $toTapeFile;
#don't chop	chop ($line);
	if ($debug == 1) {
	    print "line=$line\n";
	}
	($path[$iFile], $size[$iFile]) = split(' ', $line);
	if ($size[$iFile] =~ /(\d+\.?\d*)/) {
	    $size[$iFile] = $1;
	}

	close (TOTAPEFILE);

	if ($debug == 1) {
	    print "path=$path[$iFile], size=$size[$iFile]\n";
	}

	$iFile++;
    }

    $nFile = $iFile;

# now, we create daemon file
# either when total tape size > limitTapeSize
# or     when No. of files > maxNFileJobs in one daemon file
# then rename Checked file to Taping file

#cleanup existing daemon files
    @daemonFiles = <$controlDir/na45pc??-Y?????-no????.temp>;
    foreach $daemonFile (@daemonFiles) {
	unlink ($daemonFile); 
    }

# loop in CDRHosts
    foreach $CDRName (@CDRHost) {
	$currentTape = getNextTape($CDRName);
	$currentTapeSize = getTapeSize($currentTape);
	if ($currentTapeSize >= $tapeSize) {
#	    die "$0> tape size of $currenTape = $currentTapeSize\n";
	    print "$0> *** WARNING *** tape size of $currentTape = $currentTapeSize >= tapeSize = $tapeSize\n";
	}

	$tempSize = 0;
	$nFileJobs = 0;
# loop in files
#open daemon file first
	$numbering = 0;
	$nDaemon = 0; # no. of daemon files created
	$daemonFile = sprintf("$TAPE_ROOT/control/$CDRName-$currentTape-no%04d.temp", $numbering);

	@tempRun = ();
	@tempBurst = ();
	@tempIsSOR = ();
	
	for ($iFile=0; $iFile < $nFile; $iFile++) {
	    if ($CDR[$iFile] eq $CDRName) {
		if ($debug == 1) {
		    print "linkLine[$iFile] = $linkLine[$iFile]";
		}

		$tempSize += $size[$iFile];
		if ($debug == 1) {
		    print "currentTapeSize = $currentTapeSize\n";
		    print "tempSize = $tempSize\n";
		    print "limitTapeSize = $limitTapeSize\n";
		}

		open(DAEMONFILE,">>$daemonFile");
		print DAEMONFILE "$linkLine[$iFile]";
		$tempRun[$nFileJobs] = $run[$iFile];
		$tempBurst[$nFileJobs] = $burst[$iFile];

		$tempIsSOR[$nFileJobs] = $isSOR[$iFile];
		$nFileJobs++;
		if ($debug == 1) {
		    print "nFileJobs = $nFileJobs\n";
		    print "wrote to $daemonFile : $linkLine[$iFile]\n";
		}
		close(DAEMONFILE);

		if (($currentTapeSize+$tempSize < $limitTapeSize)
		    && ($nFileJobs < $maxNFileJobs)){

		} else { #completed daemon file...
		    $nDaemon++;
# this shouldn't happen but in case...
		    if ($currentTapeSize + $tempSize >= $tapeSize) {
			die "$0> Excceeded tapeSize for $currentTape in collectFileList. size = $currentTapeSize + $tempSize\n";
		    }
		    $finalTapeSize = $currentTapeSize + $tempSize;
		    putTapeSize($CDRName,$currentTape,$finalTapeSize);

# move daemon file to the queue/
		    $numberingTemp = 0;
		    do {
			$base = sprintf("$daemonDir/$CDRName-$currentTape-no%04d",
					$numberingTemp);
			$daemonNewFile = "$base.waiting";
			@files = ();
			@files = <$base.*>;
			$numberingTemp++;
		    } while (($#files+1) > 0);
#		    print "rename $daemonFile, $daemonNewFile\n";
		    rename $daemonFile, $daemonNewFile;

# need to reset tempSize, once putTapeSize is called;
		    $tempSize = 0;
		    if ($finalTapeSize >= $limitTapeSize) {
			markTapeSummary($currentTape,"full",-1);
			$currentTape = getNextTape($CDRName);
			$currentTapeSize = getTapeSize($currentTape);
			$numbering = 0;
		    } else {
			markTapeSummary($currentTape,"old",-1);
			$currentTapeSize = getTapeSize($currentTape);
			$numbering++;
		    }
# rename Checked file to Taping file for this daemon file
		    for ($iFileJobs=0; $iFileJobs < $nFileJobs; $iFileJobs++) {
			if ($tempIsSOR[$iFileJobs] == 0) {
			    $checkedFile = sprintf("$CheckedDir/$CDRName-run%04d-burst%04d.link",
						   $tempRun[$iFileJobs],$tempBurst[$iFileJobs]);
			    $TapingFile = sprintf("$TapingDir/$CDRName-run%04d-burst%04d.link",
						  $tempRun[$iFileJobs],$tempBurst[$iFileJobs]);
			} else {
			    $checkedFile = sprintf("$CheckedDir/$CDRName-run%04d-sor.link",
						   $tempRun[$iFileJobs]);
			    $TapingFile = sprintf("$TapingDir/$CDRName-run%04d-sor.link",
						  $tempRun[$iFileJobs]);
			}
			rename $checkedFile, $TapingFile
			    || die "cannot rename $checkedFile, $TapingFile\n";
		    }
		    $nFileJobs=0;
		    @tempRun=();
		    @tempBurst=();
		    @tempIsSOR=();

		    $daemonFile
			= sprintf("$controlDir/$CDRName-$currentTape-no%04d.temp", $numbering);
		    if ($debug == 1) {
			print "next daemonFile = $daemonFile\n";
		    }
		}
	    }
	}
	# shall we deal with the remaining files?, when file input rate is small
	# only when no. of daemon file is 0 and nFileJobs exceeded minNFileJobs
	if (($nDaemon == 0) && ($nFileJobs >= $minNFileJobs)) { #completed daemon file...
# this shouldn't happen but in case...
	    if ($currentTapeSize + $tempSize >= $tapeSize) {
		die "$0> Excceeded tapeSize for $currentTape in collectFileList. size = $currentTapeSize + $tempSize\n";
	    }
	    $finalTapeSize = $currentTapeSize + $tempSize;
	    putTapeSize($CDRName,$currentTape,$finalTapeSize);
# need to reset tempSize, once putTapeSize is called;
# move daemon file to the queue
	    $numberingTemp = 0;
	    do {
		$base = sprintf("$daemonDir/$CDRName-$currentTape-no%04d",
				$numberingTemp);
		$daemonNewFile = "$base.waiting";
		@files = ();
		@files = <$base.*>;
		$numberingTemp++;
	    } while (($#files+1) > 0);
#	    print "rename $daemonFile, $daemonNewFile\n";
	    rename $daemonFile, $daemonNewFile;

	    $tempSize = 0;
	    if ($finalTapeSize >= $limitTapeSize) {
		markTapeSummary($currentTape,"full",-1);
	    } else {
		markTapeSummary($currentTape,"old",-1);
	    }
# rename Checked file to Taping file for this daemon file
	    for ($iFileJobs=0; $iFileJobs < $nFileJobs; $iFileJobs++) {
		if ($tempIsSOR[$iFileJobs] == 0) {
		    $checkedFile = sprintf("$CheckedDir/$CDRName-run%04d-burst%04d.link",
					   $tempRun[$iFileJobs],$tempBurst[$iFileJobs]);
		    $TapingFile = sprintf("$TapingDir/$CDRName-run%04d-burst%04d.link",
					  $tempRun[$iFileJobs],$tempBurst[$iFileJobs]);
		} else {
		    $checkedFile = sprintf("$CheckedDir/$CDRName-run%04d-sor.link",
					   $tempRun[$iFileJobs]);
		    $TapingFile = sprintf("$TapingDir/$CDRName-run%04d-sor.link",
					  $tempRun[$iFileJobs]);
		}
		rename $checkedFile, $TapingFile
		    || die "cannot rename $checkedFile, $TapingFile\n";
	    }
	}
#finally cleanup remaining daemonTemp files
    }
    @daemonTempFiles = <$controlDir/na45pc??-Y?????-no????.temp>;
    foreach $daemonTempFile (@daemonTempFiles) {
	unlink $daemonTempFile;
    }
    return 1;
}

sub getNextTape {

# first check tapeSummaryList.dat and check the mark.
# if "old" mark is found, check sizeFile
# if no sizeFile for the CDR, look for "new" mark 
# and use it as next tape
# if found use it.

    my $CDR = shift(@_) if defined (@_);
    my $tape = shift(@_) if defined (@_);

    die "getNextTape> usage getNextTape CDR <start-tape>\n" unless (defined $CDR);

    if ($CDR !~ /na45pc\d{2}/) {
	die "getNextTape> usage getNextTape CDR <start-tape>\n";
    }

#check tapeSummaryList.dat and make tape list
    my $tapeList = "$TAPE_ROOT/list/tapeSummaryList.dat";
    open (TAPELIST, "<$tapeList")
	|| die "$0> cannot open $tapeList\n";

    my $nextTape = "";
    my $tempTape, $new, $size;
    my $line;

    my $ok = 1;
    $ok = 0 if defined $tape;

    while ($line = <TAPELIST>) {
	if ($line =~ /(Y\d{5}) (\w+) (\d+)/) {
	    $tempTape = $1;
	    $new  = $2;
	    $size = $3;

	    if ($ok == 0) {
		if ($tempTape eq $tape) {
		    $ok = 1;
		}
		next; #until currentTape appears, skip
	    }

	} else {
	    next;
	}

	my $currentSize;

	if ($new eq "old") {
	    $whichCDR  = getTapeCDR($tempTape);
	    if ($debug == 1) {
		print "whichCDR = $whichCDR\n";
	    }
	    if ($whichCDR eq "") {
#		die "tape $tempTape is marked old but size file cannot be found\n";
		print "*** WARNING *** tape $tempTape is marked old but size file cannot be found\n";
	    }
	    if ($CDR ne $whichCDR) {
		next;
	    }
	    $currentSize = getTapeSize($tempTape);
	    if ($debug == 1) {
		print "currentSize = $currentSize\n";
	    }

#	    if ($currentSize <= 0) {
#		die "getNextTape, mark old but size <=0 for $tempTape\n";
#	    }

	    if ($currentSize > $limitTapeSize) {
#		die "getNextTape, mark old but size too large for $tempTape = $currentSize\n";
		print "*** WARNING *** getNextTape, mark old but size too large for $tempTape = $currentSize\n";
	    }

	    $nextTape = $tempTape;
	    last;
	} elsif ($new eq "new") {
	    $nextTape = $tempTape;
	    $size = 0;

#don't mark yet	    markTapeSummary($nextTape, "old", $size);
#	    putTapeSize($CDR,$nextTape,$size);
	    last;
	}
	# ignore if $new eq "full" or "complete"
    }
    close TAPELIST;
    
    if ($nextTape eq "") {
	print "currentTape = $currentTape\n";
	print "cannot get nextTape\n";
        die "check $tapeList\n";
    }

    if ($debug == 1) {
	print "CDR = $CDR\n";
	print "nextTape = $nextTape\n";
	print "size = $size\n";
    }

    return $nextTape;
}


sub getTapeStatus {
# usage getTapeStatus($currentTape)
    my $currentTape = shift(@_);

    die "usage: getTapeStatus(tapeLabel)"
	unless defined $currentTape;

    my $tapeList = "$TAPE_ROOT/list/tapeSummaryList.dat";
    open (TAPELIST, "<$tapeList")
	|| die "$0> cannot open $tapeList\n";

    my $line;
    my $tempTape, $new, $size;
    $new = "";
    while ($line = <TAPELIST>) {
	if ($line =~ /(Y\d{5}) (\w+) (\d+)/) {
	    $tempTape = $1;
	    $new  = $2;
	    $size = $3;
	    if ($tempTape eq $currentTape) {
		last;
	    }
	}
    }
    close TAPELIST;
    
    return $new;
}


sub getTapeSize {
# usage getTapeSize($currentTape)
    my $currentTape = shift(@_);

    die "usage: getTapeSize(tapeLabel)"
	unless defined $currentTape;

    @sizeFiles = ();
    @sizeFiles = <$controlDir/size-na45pc??-$currentTape.daemon>;

    my $line;
    if ($#sizeFiles+1 == 1) {
	$sizeFile = $sizeFiles[0];
	open(SIZEFILE,"<$sizeFile");
	$line = <SIZEFILE>;
	if ($line =~ /(\d+)/) {
	    $size = $1;
	    close(SIZEFILE);
	    return $size;
	}
    } elsif ($#sizeFiles+1 > 1) {
	die "getTapeSize> too many sizeFiles for $currentTape\n";
    }

    my $tapeList = "$TAPE_ROOT/list/tapeSummaryList.dat";
    open (TAPELISTTEMP, "<$tapeList")
	|| die "$0> cannot open $tapeList\n";

    my $tempTape, $new, $size;
    $new = "";
    while ($line = <TAPELISTTEMP>) {
	if ($line =~ /(Y\d{5}) (\w+) (\d+)/) {
	    $tempTape = $1;
	    $new  = $2;
	    $size = $3;
	    if ($tempTape eq $currentTape) {
		last;
	    }
	}
    }
    close TAPELISTTEMP;
    
    return $size;
}


sub getTapeSizeWritten {
# usage getTapeSizeWritten($currentTape)
    my $currentTape = shift(@_);

    die "usage: getTapeSizeWritten(tapeLabel)"
	unless defined $currentTape;

    @sizeFiles = ();
    @sizeFiles = <$controlDir/size-na45pc??-$currentTape.written>;

    my $line;
    if ($#sizeFiles+1 == 1) {
	$sizeFile = $sizeFiles[0];
	open(SIZEFILE,"<$sizeFile");
	$line = <SIZEFILE>;
	if ($line =~ /(\d+)/) {
	    $size = $1;
	    close(SIZEFILE);
	    return $size;
	}
    } elsif ($#sizeFiles+1 > 1) {
	die "getTapeSize> too many sizeFiles for $currentTape\n";
    }

    my $tapeList = "$TAPE_ROOT/list/tapeSummaryList.dat";
    open (TAPELISTTEMP, "<$tapeList")
	|| die "$0> cannot open $tapeList\n";

    my $tempTape, $new, $size;
    $new = "";
    while ($line = <TAPELISTTEMP>) {
	if ($line =~ /(Y\d{5}) (\w+) (\d+)/) {
	    $tempTape = $1;
	    $new  = $2;
	    $size = $3;
	    if ($tempTape eq $currentTape) {
		last;
	    }
	}
    }
    close TAPELISTTEMP;
    
    return $size;
}

sub getTapeCDR {
# usage getTapeCDR($currentTape)
    my $currentTape = shift(@_);

    die "usage: getTapeCDR(tapeLabel)"
	unless defined $currentTape;

    if ($debug == 1) {
	print "sizeFile = $controlDir/size-na45pc*-$currentTape.daemon\n";
    }
    @sizeFiles = ();
    @sizeFiles = <$controlDir/size-na45pc??-$currentTape.daemon>;
    

    my $CDR = "";
    if ($#sizeFiles+1 == 1) {
	$sizeFile = $sizeFiles[0];
	
	if ($sizeFile =~ /size-(na45pc\d{2})-$currentTape\.daemon/) {
	    $CDR = $1;
	    return $CDR;
	}
    } elsif ($#sizeFiles+1 > 1) {
	die "getTapeSize> too many sizeFiles for $currentTape\n";
    }

    return $CDR;
}

sub putTapeSize {
# usage putTapeSize($CDR, $currentTape, $size)
    my $CDR         = shift(@_);
    my $currentTape = shift(@_);
    my $size        = shift(@_);

    die "usage: putTapeSize(CDR,tapeLabel,size)\n"
	unless (defined $CDR && defined $currentTape && defined $size);

    if ($size == 0) {
	return 1; # do not open file is size=0
    }

    my $currentSize = getTapeSize($currentTape);

# if size file doesn't exist, mark it as old
    if ($currentSize == 0) {
	markTapeSummary($currentTape, "old", -1);
    }

    my $sizeFile
	= "$controlDir/size-$CDR-$currentTape.daemon"; # size to be written (daemon file made)

    open (CURRENTTAPEFILE, ">$sizeFile") || die "cannot open $currentTapeFile\n";
    print CURRENTTAPEFILE "$size\n";
    close CURRENTTAPEFILE;
    return 1;
}

sub putTapeSizeWritten {
# usage putTapeSize($CDR, $currentTape, $size)
    my $CDR         = shift(@_);
    my $currentTape = shift(@_);
    my $size        = shift(@_);

    die "usage: putTapeSizeWritten(CDR,tapeLabel,size)\n"
	unless (defined $CDR && defined $currentTape && defined $size);

    if ($size == 0) {
	return 1; # do not open file is size=0
    }
    
    my $sizeFile
	= "$controlDir/size-$CDR-$currentTape.written"; # size after tape Daemon is finished

    open (CURRENTTAPEFILE, ">$sizeFile") || die "cannot open $currentTapeFile\n";
    print CURRENTTAPEFILE "$size\n";
    close CURRENTTAPEFILE;
    return 1;
}

sub rawDataOrder {
    $a =~ /run(\d{4})-burst(\d{4})\.dat/;
    $run_a = $1;
    $burst_a = $2;

    $b =~ /run(\d{4})-burst(\d{4})\.dat/;
    $run_b = $1;
    $burst_b = $2;

    $comparison = ($run_a <=> $run_b);
    if ($comparison == 0) {
	$comparison = ($burst_a <=> $burst_b);
    }

    return $comparison;
}


sub daemonFileOrder {
# ordering of waiting-*
# priority
# 1) older time stamp
# 2) smaller tape Label
# 3) smaller numbering
# 4) smaller CDR

    $a =~ /na45pc(\d{2})-Y(\d{5})-no(\d{4})\.waiting/;
    $CDR_a = $1;
    $tapeLabel_a = $2;
    $numbering_a = $3;
    $time_a = (-M $a);

    $b =~ /na45pc(\d{2})-Y(\d{5})-no(\d{4})\.waiting/;
    $CDR_b = $1;
    $tapeLabel_b = $2;
    $numbering_b = $3;
    $time_b = (-M $b);

#older file first
    $comparison = ($time_b <=> $time_a);

    if ($comparison == 0) {
	$comparison = ($tapeLabel_a <=> $tapeLabel_b);
	if ($comparison == 0) {
	    $comparison = ($numbering_a <=> $numbering_b);
	    if ($comparison == 0) {
		$comparison = ($CDR_a <=> $CDR_b);
	    }
	}
    }
    return $comparison;
}

sub updateTapeStatus {

#
# already the job should be finished...
# handling after finishing...

# check submittedFile
    @fileList=();
    @fileList = <$daemonDir/na45pc??-Y?????-no????.submitted>;
    
    foreach $submittedFile (@fileList) {
#	chop ($submittedFile);
# the same $numbering between submitted-* and log file
	if ($submittedFile =~ /(na45pc\d{2})-(Y\d{5})-no(\d{4})\.submitted/) {
	    $remoteHost = $1;
	    $realTapeLabel = $2;
	    $numbering = $3;
	    $lockFile = "$lockDir/$realTapeLabel.lock";
	    $logFile = sprintf("$logDir/$remoteHost-$realTapeLabel-no%04d.log.submitted",$numbering);
	} else {
	    next;
	}

	$success = `grep -c "command successful" $logFile`;
	$fail = `grep -c "command failed" $logFile`;
	$failNetwork = `grep -c "Host name lookup failure" $logFile`;
	$emptyError = `grep -c "is empty" $logFile`;
	$emptyError1 = `grep -c "does not exist or is unreachable" $logFile`;
	$failCDR = `grep -c "No route to host" $logFile`;
	$forkError = `grep -c "Resource temporarily unavailable" $logFile`;
	$inOutError = `grep -c "Input/output error" $logFile`;
	$usageError = `grep -c "usage: /cdrShare01/cerescdr/cdr2k/tape/scripts/tapeDaemon.pl <lockFile> <tapeLabel> <tapeFileList> <sequence> <xList>" $logFile`;

	$otherError = 0;

# if already lockFile does not exist and nothing is on. cannot resolve 
#bad	$age = (-M $submittedFile) * 24.;  # in hour
	$age = getAge($submittedFile) * 24.;  # in hour

#	print "age of $submittedFile = $age\n";

	$maxHourWait = 2.; # 3 hours
	if ((!(-e $lockFile)) && ($age > $maxHourWait)) {
#	    if (($success == 0) && ($fail == 0) && ($failNetWork == 0)
#		&& ($emptyError == 0) && ($emptyError1 == 0) && ($failCDR == 0)
#		&& ($forkError == 0) && ($inOutError == 0)) {'

	    if ($success < 1) {
		print "$lockFile does not exist, but $submittedFile exist.\n";
		print "otherError set to 1\n";
		print "age = $age\n";
		$otherError = 1;
	    }
	}
# 04.10.00 added check age of lockFile if it is too old 
#age	$ageLock = (-M $lockFile) * 24.;  # in hour
	$ageLock = getAge($lockFile) * 24.;  # in hour
	$maxHourWait = 2.; # 3 hours
	if ((-e $lockFile) && ($ageLock > $maxHourWait)) {
	    print "$lockFile too old.\n";
	    print "removing lock file\n";
	    unlink($lockFile);
	    print "age = $age\n";
	    $otherError = 1;
	}

	if ($printLevel > 50) {
	    print "submittedFile = $submittedFile\n";
	    print "logFile = $logFile\n";
	    print "success = $success";
	    print "fail    = $fail";
	    print "failNetwork = $failNetwork";
	    print "emptyError  = $emptyError";
	    print "emptyError1  = $emptyError1";
	    print "failCDR = $failCDR\n";
	    print "forkError = $forkError\n";
	    print "inOutError = $inOutError\n";
	    print "usageError = $usageError\n";
	    print "otherError = $otherError\n";
	}

	if ($success >= 1) { 

#clear No. of failure
# 2000 only open it when nFail > 0
	    $nFail = 0;
#	    $failFile = "$controlDir/$realTapeLabel-failure.dat";
#	    open(FAIL, ">$failFile");
#	    print FAIL $nFail;
#	    close(FAIL);
	    
	    if ($printLevel > 30) {
		print "Succeeded tpwrite for tape $realTapeLabel\n";
		print "adding to file list\n";
	    }

	    $numberingTemp = $numbering; # tries the same number first
	    do {
		$finishedFile
		    = sprintf("$daemonDir/$remoteHost-$realTapeLabel-no%04d.finished", $numberingTemp);
		$numberingTemp ++;
	    } while (-e $finishedFile);

	    rename $submittedFile, $finishedFile
		|| die "couldn't rename $submittedFile, $finishedFile\n";

	    $numberingTemp = $numbering;
	    do {
		$logFileFinished
		    = sprintf("$logDir/$remoteHost-$realTapeLabel-no%04d.log.finished", $numberingTemp);
		$numberingTemp ++;
	    } while(-e $logFileFinished);


	    rename $logFile, "$logFileFinished"
		|| die "couldn't rename $logFile, $logFileFinished\n";

	    open (FINISHEDFILE, "<$finishedFile")
		|| die "cannot open $finishedFile\n";

	    @finishedFileList=();
	    @finishedFileList = <FINISHEDFILE>;
	    close FINISHEDFILE;
	    
	    $tapeContentList = "$listDir/$realTapeLabel.list";

# 11.08.00 now, size is only taken from sizeWritten file
# size will be written only for this jobs on tapecontent list
	    
#	    $totalNumFiles = 0;
#	    $totalSizeInTape = 0;
#	    if (-e $tapeContentList) {
#		$line = `tail -1 $tapeContentList`;
#
#		if ($line =~ /total (\d+) (\d+)/) {
#		    $totalNumFiles   = $1;
#		    $totalSizeInTape = $2;
#		}
#	    }
#
#	    if ($totalSizeInTape > $tapeSize) {
#		print "updateTapeStatus> exceeded tapesize\n";
#		print "WARNING: updateTapeStatus>tape $realTapeLabel, totalSizeInTape = $totalSizeInTape\n";
#	    }

	    $totalSizeInTape = getTapeSizeWritten($realTapeLabel);

	    open(TAPECONTENTLIST, ">>$tapeContentList");

	    $totalSizeReset = 0;

	    foreach $finishedDiskFile (@finishedFileList) {
		($path, $size) = split(' ', $finishedDiskFile);
		$isSOR = 0;
		if ($path =~ /run(\d{4})-burst(\d{4})\.dat/) {
		    $run = $1; $burst = $2;
		    
		    $tapeFileName = sprintf("run%04d-b%04d.dat",
					    $run, $burst);
		    $isSOR = 0;
		} elsif ($path =~ /run(\d{4})\.sor/) {
		    $run = $1;
		    $burst = -1;
		    $tapeFileName
			= sprintf("run%04d.sor", $run);
		    $isSOR = 1;
		} else {
		    next;
		}

		print TAPECONTENTLIST "$tapeFileName $path $size\n";
#		$totalNumFiles ++;
		$totalSizeInTape += $size;
		$totalSizeReset  += $size;

		if ($totalSizeInTape > $tapeSize) {
		    print "updateTapeStatus> WARNING:tape $realTapeLabel, totalSizeInTape = $totalSizeInTape\n";
#		    die "updateTapeStatus> tape $realTapeLabel, totalSizeInTape = $totalSizeInTape\n";
		}

# rename Taping/Taping-* to toClean/toClean-*
		if ($isSOR == 0) {
		    $oldPath = sprintf("$TapingDir/$remoteHost-run%04d-burst%04d.link",
				       $run, $burst);
		    $newPath = sprintf("$toCleanDir/$remoteHost-run%04d-burst%04d.link",
				       $run, $burst);
		} else {
		    $oldPath = sprintf("$TapingDir/$remoteHost-run%04d-sor.link",
				       $run);
		    $newPath = sprintf("$toCleanDir/$remoteHost-run%04d-sor.link",
				       $run);
		}

		rename $oldPath, $newPath || die "$0> faild to rename $oldPath, $newPath\n";

	    }
	    $now = `date +"%D %T"`;
	    chop($now);
	    $histFile = "$statusDir/tapeDaemon.hist";
	    open(HIST, ">>$histFile");
	    
	    if ($finishedFile =~ /($remoteHost-$realTapeLabel-no\d+)/) {
		$line = sprintf("end   $1 : $now %8d", $totalSizeReset);
		print HIST "$line\n";
	    }
	    close (HIST);

	    for ($iCDR=0;$iCDR<$maxNCDR;$iCDR++) {
		if ($CDRHost[$iCDR] eq $remoteHost) {
		    last;
		}
	    }
	    $dataCDR[$iCDR] += ($totalSizeReset/1024./1024.);
	    $dataWritten += ($totalSizeReset/1024./1024.);
	    $totalDataWritten += ($totalSizeReset/1024./1024.);
	    close (TAPECONTENTLIST);
	    
	    # update tapeSummaryList.dat

#  size in tapeSummaryList.dat should be updated only when the files are successfully written on the tape
#  HS revived 8 Nov 99
	    markTapeSummary($realTapeLabel, "", $totalSizeInTape);
	    putTapeSizeWritten($remoteHost,$realTapeLabel,$totalSizeInTape);

# check consistency in tape size. If YXXXX-current.dat has smaller size set it to that in tapeSummary.dat
	    checkTapeSize($realTapeLabel);

	    @waitingList=();
	    @submittedList=();
	    @problemCheckList=();

	    @waitingList = <$daemonDir/na45pc??-$realTapeLabel-no????.waiting>;
	    @submittedList = <$daemonDir/na45pc??-$realTapeLabel-no????.submitted>;
	    @problemCheckList = <$daemonDir/na45pc??-$realTapeLabel-no????.problemCheck>;

	    if ($printLevel > 50) {
		print "checking if starting finishTape\n";
		print "waitingList = $#waitingList\n";
		print "submittedList = $#submittedList\n";
	    }
# 11/28/99 to complete tape if no daemon file is left
#	    if (($totalSizeInTape >= $limitTapeSize)
#		&& ($#waitingList == -1)
#		&& ($#submittedList == -1)) {
# 08.08.00 added requirement of "full"
	    $tempTapeMode = getTapeStatus($realTapeLabel);
	    if (($tempTapeMode eq "full") 
		&& ($#problemCheckList == -1)
		&& ($#waitingList == -1)
		&& ($#submittedList == -1)) {
#write protect tape and tape dump
		print "Going to finish Tape\n";
		finishTape($realTapeLabel, $totalSizeInTape, $remoteHost);
	    }
	    
	} elsif (($fail >= 1)
		 || ($failNetwork >= 1)
		 || ($failCDR >= 1)
		 || ($forkError >= 1)
		 || ($inOutError >= 1)
		 || ($usageError >= 1)
		 || ($otherError >= 1)){ #command failed error handling...

	    print "************ error in tapeDaemon command observed **********\n";
	    print "submittedFile = $submittedFile\n";
	    print "logFile = $logFile\n";
	    print "success = $success\n";
	    print "fail    = $fail\n";
	    print "failNetwork = $failNetwork\n";
	    print "emptyError  = $emptyError\n";
	    print "emptyError1  = $emptyError1\n";
	    print "inOutError = $inOutError\n";
	    print "usageError = $usageError\n";
	    print "failCDR = $failCDR\n";


# if there is a error showing empty file exist, just rename and
# let human to check
	    if (($emptyError >= 1) || ($emptyError1 >= 1)) {
		$numberingTemp = $numbering;
		do {
		    $problemCheckFile
			= sprintf("$daemonDir/$remoteHost-$realTapeLabel-no%04d.problemCheck",
				  $numberingTemp);
		    $numberingTemp++;
		} while (-e $problemCheckFile);
		print "rename $submittedFile, $problemCheckFile\n";
		rename $submittedFile, $problemCheckFile;
		$logFileCheckProblem = sprintf("$logsDir/$remoteHost-$realTapeLabel-no%04d.log.problemCheck",
					       $numberingTemp);
		print "rename $logFile, $LogFileCheckProblem\n";
		rename $logFile, $LogFileCheckProblem;


		print "submittedFile = $submittedFile\n";
		print "logFile = $logFile\n";
		print "success = $success";
		print "fail    = $fail";
		print "failNetwork = $failNetwork";
		print "emptyError  = $emptyError";
		print "emptyError1  = $emptyError1";
		print "failCDR = $failCDR\n";
		print "forkError = $forkError\n";
		print "inOutError = $inOutError\n";
		print "usageError = $usageError\n";
		print "otherError = $otherError\n";

		$failMsg = `tail -10 $logFile`;
		mailMsg("updateTapeStatus> failed in tapeDaemon.pl\n...  check $logFile\n Errors: $failMsg\n fail = $fail\n failNetwork = $failNetwork\n $emptyErro = $emptyError\n emptyError1 = $emptyErro1\n failCDR = $failCDR\n forkError = $forkError\n inOutError = $inOutError\n usageError = $usageError\n otherError = $otherError\n");
		
		next;
	    }

	    $failFile = "$controlDir/$realTapeLabel-failure.dat";
	    $nFail = 0;
	    if (open(FAIL, "<$failFile")) {
		$line = <FAIL>;
		if ($line =~ /(\d+)/) {
		    $nFail = $1;
		} 
		close(FAIL);
	    }

	    $nFail++;

	    if ($printLevel > 50) {
		print "*********** nFail = $nFail *********\n";
	    }

	    open(FAIL, ">$failFile");
	    print FAIL "$nFail\n";
	    close(FAIL);

	    if ($nFail > $maxTapeFail) {
# rename daemon file with the new tape....
# mark tape as complete and get the next tape.

		$realTapeLabelNew = getNextTape($remoteHost,$realTapeLabel);
		finishTape($realTapeLabel, -1, $remoteHost);
	    }
	    
	    $failMsg = `tail -10 $logFile`;
	    mailMsg("updateTapeStatus> failed in tapeDaemon.pl\n...  check $logFile\n Errors: $failMsg \n fail = $fail\n failNetwork = $failNetwork\n $emptyErro = $emptyError\n emptyError1 = $emptyErro1\n failCDR = $failCDR\n forkError = $forkError\n inOutError = $inOutError\n otherError = $otherError\n");


	    ############### tapeDaemon Failed!!!################
	    # How to deal with failed files
	    # check kind of error 
	    # save successfully written files
	    # (by checking 'BYTES COPIED')
            #####################################################
	    
	    # check how many files are written successfully
	    $command = "grep -c 'BYTES COPIED' $logFile";
#	    $nFilesWritten = `$command`;
	    $nFilesWritten = 0; # I just want to force to rewrite all files.
	    # kind of error
#	    $error = `"grep -i 'error' $logFile"`;
	    
	    print "********* tpwrite command Failed... tape = $realTapeLabel, logFile = $logFile \n";
	    if ($debug == 1)
	    {print "nFilesWritten = $nFilesWritten\n";}
	    
	    $numberingTemp = $numbering;
	    do {
		$problemFile
		    = sprintf("$daemonDir/$remoteHost-$realTapeLabel-no%04d.problem", $numberingTemp);
		$finishedFile
		    = sprintf("$daemonDir/$remoteHost-$realTapeLabel-no%04d.finished", $numberingTemp);
		$numberingTemp ++;
	    } while (-e $finishedFile);

	    open (SUBMITTEDFILE, "<$submittedFile")
		|| die "cannot open $submittedFile\n";
	    @submittedFileList=();
	    @submittedFileList = <SUBMITTEDFILE>;
	    close (SUBMITTEDFILE);

# don't do this.
#	    if ($nFilesWritten > 0) {
#		`head "-$nFilesWritten" $submittedFile > $finishedFile`;
# shrink submittedFileList
#		splice(@submittedFileList, 0, $nFilesWritten);
#	    }

# this is not necessary, I think. 
# commented out on 05.10.00
#	    open (PROBLEMFILE, ">$problemFile") 
#		|| die "couldn't open $problemFile\n";
#
# copy problem file where files are not written successfully.
#	    foreach $line (@submittedFileList) {
#		print PROBLEMFILE "$line";
#	    }
#	    close (PROBLEMFILE);

# resubmit this job
# by renaming back submitted-* file to waiting-* file...
# tries to keep the same number
	    if ($nFail > $maxTapeFail) {
		$numberingTemp = 0;
	    } else {
		$numberingTemp = $numbering;
	    }

	    do {
		if ($nFail > $maxTapeFail) {
		    # use new tape name....
		    $waitingFile
			= sprintf("$daemonDir/$remoteHost-$realTapeLabelNew-no%04d.waiting", $numberingTemp);
		} else {
		    $waitingFile
			= sprintf("$daemonDir/$remoteHost-$realTapeLabel-no%04d.waiting", $numberingTemp);
		}
		$numberingTemp ++;
	    } while (-e $waitingFile);

	    $histFile = "$statusDir/tapeDaemon.hist";
	    open(HIST, ">>$histFile");
	    if ($submittedFile =~ /($remoteHost-$realTapeLabel-no\d+)/) {
		$now = `date +"%D %T"`;
		chop($now);
		$line = sprintf("fail  $1 : $now");
		print HIST "$line\n";
	    }
	    close (HIST);

	    print "renaming daemon file for resubmission due to failure\n";
	    print "rename $submittedFile, $waitingFile\n";

	    rename $submittedFile, $waitingFile
		|| die "couldn't rename $submittedFile, $waitingFile\n";
	    `touch $waitingFile`;


# updating tape size (only if fail >= 1)
# assume a half of the files are actually written on the tape and update sizes
	    if ($fail >= 1) {
		$waitingSize = checkSizeInMasterList($waitingFile) / 2.;
		$tempCurrentSizeWritten = getTapeSizeWritten($realTapeLabel);
		$tempCurrentSizeWritten += $waitingSize;
		
		print "updating currentTapeSizeWritten assuming half are written on the tape during failure\n";
		putTapeSizeWritten($remoteHost,$realTapeLabel, $tempCurrentSizeWritten);
		markTapeSummary($realTapeLabel, "", $tempCurrentSizeWritten);
		
#need to update also size-*.daemon
		$tempCurrentSize = getTapeSize($realTapeLabel);
		$tempCurrentSize += $waitingSize;
		
		print "updating currentTapeSize assuming half are written on the tape during failure\n";
		putTapeSize($remoteHost,$realTapeLabel, $tempCurrentSize);
	    
		if ($tempCurrentSize >= $limitTapeSize) {
		    markTapeSummary($realTapeLabel, "full", -1);
		}
	    }

#
# In case, we changed the tape, we need to update curernt tape size...
# for net tape
	    if ($nFail > $maxTapeFail) {
		print "nFail = $nFail excceeded maxTapeFail = $maxTapeFail\n";
		print "changing tape to from $realTapeLabel to $realTapeLabelNew\n";
		$waitingSize = checkSizeInMasterList($waitingFile);
		$tempCurrentSize = getTapeSize($realTapeLabelNew);
		$tempCurrentSize += $waitingSize;

		putTapeSize($remoteHost,$realTapeLabelNew, $tempCurrentSize);
		if ($tempCurrentSize >= $limitTapeSize) {
		    markTapeSummary($realTapeLabelNew, "full", -1);
		}
#		$tempCurrentSize = getTapeSize($realTapeLabelNew);
#		print "tempCurrentSize(after putTapeSize) = $tempCurrentSize\n";
#HS removed 8 Nov 99		markTapeSummary($realTapeLabelNew, "", $tempCurrentSize);
		# update the currentTapeSize
	    }

	    
	    $numberingTemp = $numbering;
	    do {
		$logFileProblem = sprintf("$logDir/$remoteHost-$realTapeLabel-no%04d.log.problem", $numberingTemp);
		$numberingTemp ++;
	    } while (-e $logFileProblem);

	    rename $logFile, "$logFileProblem"
		|| die "couldn't rename $logFile, $logFile.problem\n";

	    if ($nFilesWritten > 0) {
# update tape content list if there are written on files
		
		$tapeContentList = "$listDir/$realTapeLabel.list";
	    
#		$totalNumFiles = 0;
		$totalSizeInTape = 0;
		$totalSizeReset = 0;
#		if (-e $tapeContentList) {
#		    $line = `tail -1 $tapeContentList`;
#		    
#		    if ($line =~ /total (\d+) (\d+)/) {
#			$totalNumFiles   = $1;
#			$totalSizeInTape = $2;
#			
#			if ($totalSizeInTape > $tapeSize) {
#			    
#			    print "updateTapeStatus> exceeded tapesize\n";
#			}
#		    }
#		}

		$totalSizeInTape = getTapeSizeWritten($realTapeLabel);

		open(TAPECONTENTLIST, ">>$tapeContentList");

		open (FINISHEDFILE, "<$finishedFile") 
		    || die "couldn't open $finishedFile\n";
		@finishedFileList = ();
		@finishedFileList = <FINISHEDFILE>;
		close (FINISHEDFILE);
		
		for ($i=0; $i<$nFilesWritten;$i++) {
		    $isSOR = 0;
		    ($path, $size) = split(' ', $finishedFileList[$i]);
		    if ($path =~ /run(\d{4})-burst(\d{4})\.dat/) {
			$run = $1; $burst = $2;
			$tapeFileName = sprintf("run%04d-b%04d.dat",
						$run, $burst);
			$isSOR = 0;
		    } elsif ($path =~ /run(\d{4})\.sor/) {
			$run = $1;
			$burst = -1;
			$tapeFileName
			    = sprintf("run%04d\.sor",
				      $run);   
			$isSOR = 1;
		    } else {
			next;
		    }
		    
		    print TAPECONTENTLIST "$tapeFileName $path $size\n";
#		    $totalNumFiles ++;
		    $totalSizeInTape += $size;
		    $totalSizeReset += $size;
		    
		    if ($totalSizeInTape > $tapeSize) {
			die "updateTapeStatus> tape $realTapeLabel, totalSizeInTape = $totalSizeInTape\n";
		    }
		    
# rename Taping/* to toClean/*

		    if ($isSOR == 0) {
			$oldPath = sprintf("$TapingDir/$remoteHost-run%04d-burst%04d.link",
					   $run, $burst);
			$newPath = sprintf("$toCleanDir/$remoteHost-run%04d-burst%04d.link",
					   $run, $burst);
		    } else {
			$oldPath = sprintf("$TapingDir/$remoteHost-run%04d-sor.link",
					   $run);
			$newPath = sprintf("$toCleanDir/$remoteHost-run%04d-sor.link",
					   $run);
		    }
		    
		    rename $oldPath, $newPath || die "$0> failed to rename $oldPath, $newPath\n";
		    
                }
		$now = `date +"%D %T"`;
		chop($now);
# this info should be written on a separate history file
#		print TAPECONTENTLIST "date = $now";
		$histFile = "$statusDir/tapeDaemon.hist";
		open(HIST, ">>$histFile");
		if ($finishedFile =~ /($remoteHost-$realTapeLabel-no\d+)/) {
		    $line = sprintf("end   $1 : $now %8d", $totalSizeReset);
		    print HIST "$line\n";
		}
		close (HIST);
		for ($iCDR=0;$iCDR<$maxNCDR;$iCDR++) {
		    if ($CDRHost[$iCDR] eq $remoteHost) {
			last;
		    }
		}
		$dataCDR[$iCDR] += ($totalSizeReset/1024./1024.);

		$dataWritten += ($totalSizeReset/1024./1024.);
		$totalDataWritten += ($totalSizeReset/1024./1024.);
#		print TAPECONTENTLIST "total $totalNumFiles $totalSizeInTape\n";
		close (TAPECONTENTLIST);

		# update tapeSummaryList.dat
#  this is not correct: size in tapeSummaryList.dat = Yxxxxx-current.dat
# HS revived
		markTapeSummary($realTapeLabel, "", $totalSizeInTape);
		putTapeSizeWritten($remoteHost,$realTapeLabel,$totalSizeInTape);
		
	    }
	} # end elsif fail==1
	
    } #foreach $submittedFile
# lock tape (after completion)
# take dumptape log
#         sysreq tms lock disable write vid $tapename &
}

sub checkNStream {
#contents  host tape-name totalsize(kB)

    @nStreamInCDR = (); # make it global to be used in updateStatistics 15.08.00
    my $nStream = 0;
    for ($iCDR=0;$iCDR<$maxNCDR;$iCDR++) {
	$nStreamInCDR[$iCDR] = 0;
	$remoteHost = $CDRHost[$iCDR];
	@processList=();
	@processList = `ps -ax --cols=300|grep "rsh -n $remoteHost $TAPE_ROOT/scripts/tapeDaemon.pl"`;

        if ($debug == 1) {
	    print "ps -ax --cols=300|grep rsh -n $remoteHost $TAPE_ROOT/scripts/tapeDaemon.pl\n";
	}

	foreach $process (@processList) {

	    if ($debug == 1) {
		print "checkNStream> process = $process\n";
	    }
	    if (($process !~ /grep/)&&($process !~ /defunct/)){
		$nStreamInCDR[$iCDR] ++;
	    }
	}
	$nStream += $nStreamInCDR[$iCDR];
    }
    return $nStream;
}

sub checkNStreamOnCDR {

    my $remoteHost = shift(@_);
    die "checkNStreamOnCDR(host)\n" unless defined $remoteHost;
    my $nStream = 0;

    @processList=();
    @processList = `ps -ax --cols=300|grep "rsh -n $remoteHost $TAPE_ROOT/scripts/tapeDaemon.pl"`;

    if ($debug == 1) {
	print "ps -ax --cols=300|grep rsh -n $remoteHost $TAPE_ROOT/scripts/tapeDaemon.pl\n";
    }

    foreach $process (@processList) {
	
	if ($debug == 1) {
	    print "checkNStream> process = $process\n";
	}
	if ($process !~ /grep/) {
	    $nStream ++;
	}
    }

    return $nStream;
}



sub packWaitingFile {
    @waitingFileList = ();
    @waitingFileList = <$daemonDir/na45pc??-Y?????-no????.waiting>;

    $nWaitingFile = $#waitingFileList;
    print "nWaitingFile = $#waitingFileList\n";

    for ($i=0; $i<$nWaitingFile-1; $i++) {
	$waitingFile1 = $waitingFileList[$i];
	for ($j=i+1;$j<$nWaitingFile; $j++) {
	    $waitingFile2 = $waitingFileList[$j];
	    if ((!(-e $waitingFile1)) || (!(-e $waitingFile2))){
		next;
	    }
	    if ($waitingFile1
		=~ /na45pc(\d{2})-Y(\d{5})-no(\d{4})\.waiting/) {
		$CDR1 = $1;
		$tape1 = $2;
		$numbering1 = $3;
	    } else {
		next;
	    }
	    if ($waitingFile2
		=~ /na45pc(\d{2})-Y(\d{5})-no(\d{4})\.waiting/) {
		$CDR2 = $1;
		$tape2 = $2;
		$numbering2 = $3;
	    } else {
		next;
	    }

	    if (($CDR1 == $CDR2) && ($tape1 == $tape2)
		&& ($numbering1 != $numbering2)) {
		($line1, $temp) = split(' ', `wc -l $waitingFile1`);
		($line2, $temp) = split(' ', `wc -l $waitingFile2`);
		
#pack and pack... to increase speed
		if (($line1 < $maxSet*3) && ($line2 < $maxSet*3)) {

		    $numberingTemp = 0;

		    do {
			$waitingFileNew
			    = sprintf("$daemonDir/na45pc%02d-Y%5d-no%04d.waiting", 
				      $CDR1, $tape1, $numberingTemp);
			$numberingTemp++;
		    } while (-e $waitingFileNew);

		    `cat $waitingFile1 $waitingFile2 > $waitingFileNew`;
		    $waitingFile1Sav=sprintf("$daemonDir/na45pc%02d-Y%5d-no%04d.original", $CDR1, $tape1, $numbering1);
		    $waitingFile2Sav=sprintf("$daemonDir/na45pc%02d-Y%5d-no%04d.original", $CDR2, $tape2, $numbering2);
		    rename $waitingFile1, $waitingFile1Sav;
		    rename $waitingFile2, $waitingFile2Sav;
		}
	    }
	}
    }
    return;
}


sub submitTapeDaemon {

# next submit a tapeDaemon job using daemon files

    @fileList=();
    @sortFileList=();
    @fileList = <$daemonDir/na45pc??-Y?????-no????.waiting>;
    
    @sortFileList = sort daemonFileOrder @fileList;
# sort in order of tape label, and host name

    $iStream = $NStream; # No. of stream to submit = $maxNStreamTape-$NStream
    if ($debug == 1) {
	print "submitTapeDaemon>NStream = $NStream\n";
    }
    $noSortFileList = $#sortFileList+1;
    if ($debug == 1) {
	print "submitTapeDaemon>No. of sortFileList (daemonFiles) = $noSortFileList\n";
    }

    
    foreach $daemonFile (@sortFileList) {

	if ($debug == 1) {
	    print "daemonFile = $daemonFile\n";
	}
	if ($daemonFile =~ /(na45pc\d{2})-(Y\d{5})-no(\d{4})\.waiting/) {
	    $remoteHost = $1;
	    $realTapeLabel = $2;
	    $numbering = $3;
	    $lockFile = "$lockDir/$realTapeLabel.lock";
	} else {
	    next;
	}

# if already nfail > maxFail, use the next tape...
    

	$failFile = "$controlDir/$realTapeLabel-failure.dat";
	$nFail = 0;
       
	if (open(FAIL, "<$failFile")) {
	    $line = <FAIL>;
	    if ($line =~ /(\d+)/) {
		$nFail = $1;
	    } 
	    close(FAIL);
	}

	checkTapeSize($realTapeLabel);
	$tempSizeWritten = getTapeSizeWritten($realTapeLabel);
# size in tapeSummaryList.dat should be actual size...
#recheck tapesize...       

# Fail recovery... This is very important
#09.08.00 HS changed the condition	if (($nFail > $maxTapeFail) || ($tempSizeWritten > $limitTapeSize)) {
#change the tape
	if (($nFail > $maxTapeFail) || ($tempSizeWritten >= $tapeSize)) {
	    print "$0> finishing tape due to failure\n";
	    print "nFail = $nFail\n";
	    print "tempSizeWritten = $tempSizeWritten\n";
	    	
# warning. this is not so good. what can we do?
# if there are many waiting queues which failed at one (in one loop)
	    $realTapeLabelNew = getNextTape($remoteHost,$realTapeLabel);
	    finishTape($realTapeLabel, -1, $remoteHost);
	    $numberingTemp = 0; # new tape -> numbering = 0
	    do {
		# use new tape name....
		$daemonFileNew
		    = sprintf("$daemonDir/$remoteHost-$realTapeLabelNew-no%04d.waiting",$numberingTemp);
		$numberingTemp ++;
	    } while (-e $daemonFileNew);
	    
	    $histFile = "$statusDir/tapeDaemon.hist";
	    open(HIST, ">>$histFile");
	    if ($deamonFile =~ /($remoteHost-$realTapeLabel-no\d+)/) {
		$now = `date +"%D %T"`;
		chop($now);
		$line = sprintf("fail  $1 : $now");
		print HIST "$line\n";
	    }
	    close (HIST);

	    print "renaming daemon file for resubmission due to failure\n";
	    print "tape = $realTapeLabel, nFail = $nFail, size in tape = $tempSize\n";
	    print "rename $daemonFile, $daemonFileNew\n";
	    
	    rename $daemonFile, $daemonFileNew
		|| die "couldn't rename $daemonFile, $daemonFileNew\n";
	    
# In case we changed the tape, we need to update curernt tape size...
# for next tape


	    $daemonSize = checkSizeInMasterList($daemonFileNew);

	    $tempCurrentSize = getTapeSize($realTapeLabelNew);
	    print "realTapeLabelNew = $realTapeLabelNew\n";
	    print "tempCurrentSize (from getTapeSize) = $tempCurrentSize\n";
	    print "daemonSize = $daemonSize\n";
	    $tempCurrentSize += $daemonSize;
	    print "tempCurrentSize = $tempCurrentSize\n";
	    putTapeSize($remoteHost,$realTapeLabelNew, $tempCurrentSize);
	    if ($tempCurrentSize >= $limitTapeSize) {
		markTapeSummary($realTapeLabelNew, "full", -1);
	    }

#HS removed.	    markTapeSummary($realTapeLabelNew, "", $tempCurrentSize);
	    # update the currentTapeSize
	    $daemonFile = $daemonFileNew; # use the new daemonFile name
	    $realTapeLabel = $realTapeLabelNew;
	    $lockFile = "$lockDir/$realTapeLabel.lock";
	}

	if ($debug == 1) {
	    print "remoteHost = $remoteHost\n";
	}

# if there is no lock file, submit jobs
	if (-e $lockFile) {

# already job is running for the current tape
# check if the process is really still running.
	    
	}
	else {
# no lock file. Submit the job
#
	    @processList=();
	    @processList = `ps -ax --cols=200|grep "rsh -n $remoteHost $TAPE_ROOT/scripts/tapeD"`;
	    
	    $remoteProcess = 0;
	    foreach $process (@processList) {
		if ($process !~ /grep/) {
		    $remoteProcess ++;
		}
	    }

# limit maximum No. of processes on a same host
	    if ($remoteProcess >= $maxNStreamTapePerHost) {
		next;
	    }

# get number of files in $daemonFile
	    $numFiles = `wc -l $daemonFile`;
	    $numFiles =~ /(\d+)/;
	    $numFiles = $1;

	    $tapeList = "$listDir/tapeSummaryList.dat";
 	    $tapeContentList = "$listDir/$realTapeLabel.list";

#this is bad!!! since already tape mode may be full
# tapeDaemon does not exist now.
# let's make sure then the tapeContentList is written after the tapeDaemon.
# This should have been done already in updateTapeStatus
# if ($currentTapeMode eq "new") {

	    if (!(-e $tapeContentList)) { # if tapeConentList does not exist
		$sequenceType = "1-"; # fresh tape
# replace new to old

		$size = 0; # size is still zero
                           # only after checking finished the job. add size

		$tempTapeMode = getTapeStatus($realTapeLabel);
		if ($tempTapeMode eq "new") {
		    markTapeSummary($realTapeLabel, "old", -1);
		}

	    } else {
		$sequenceType = "n";  # add to the end of tape
	    }

	    open(DAEMONFILE, "<$daemonFile") ||
		die "$0> cannot open $daemonFile\n";

	    @fileList = ();
	    $index = 0;

	    @tapeFile = ();
	    $sizeInTape = 0;
	    @fileSize = ();
	    @run=();
	    @stream=();
	    @cpci=();
	    @burst=();
	    
	    while ($line = <DAEMONFILE>) {
                $SOR = 0;
		($tempPath, $tempSize) = split(' ', $line);
		$fileList[$index] = $tempPath;
		if ($tempPath =~ /run(\d{4})-burst(\d{4})\.dat/) {
		    $run[$index] = $1;
		    $burst[$index] = $2;
#add for SOR files
		} elsif ($tempPath =~ /run(\d{4})\.sor/) {
		    $run[$index] = $1;
		    $SOR = 1;             
		} else {
		    next;
		}

		if ($SOR == 1) {
		    $tapeFile[$index]
			= sprintf("run%04d\.sor",
				  $run[$index]);   
		} else {
		    $tapeFile[$index]
			= sprintf("run%04d-b%04d.dat",
				  $run[$index], $burst[$index]);
		}

		if ($printLevel > 50) {
		    print "Submitting tpwrite for $fileList[$index]\n";
		}
		
		$fileSize[$index] = $tempSize;
		$sizeInTape += $tempSize;
		$index ++;
	    }

	    if ($debug == 1) {
		print "sizeInTape = $sizeInTape\n";
	    }

	    $nFile = $index;
	    $xList = join(':', @fileList);
	    $tapeFileList = join(':', @tapeFile);

	    if ($debug == 1) {
		print "xList = $xList\n";
		print "tapeFileList = $tapeFileList\n";
	    }

	    $numberingTemp = $numbering-1;
	    do {
		$numberingTemp++;
		$submittedFile = sprintf("$daemonDir/$remoteHost-$realTapeLabel-no%04d.submitted",
					 $numberingTemp);
	    } while (-e $submittedFile);

	    rename $daemonFile, $submittedFile
		|| die "couldn't remame $daemonFile, $submittedFile\n";
	    `touch $submittedFile`; # update time

# same numbering as submittedFile, otherwise no way to check the success of the job
	    $logFile = sprintf("$logDir/$remoteHost-$realTapeLabel-no%04d.log.submitted",
			       $numberingTemp);

	    if ($printLevel > 50) {
		print "submittedFile = $submittedFile\n";
		print "logFile = $logFile\n";
	    }

	    if (-e $logFile) {
		$numberingTempTemp = $numberingTemp;
		do {
		    $logFileOld = sprintf("$logDir/$remoteHost-$realTapeLabel-no%04d.log.old",
					  $numberingTempTemp);
		    $numberingTempTemp++;
		} while (-e $logFileOld);
		rename $logFile, $logFileOld;
	    }

	    $baseCommand = "rsh -n $remoteHost $TAPE_ROOT/scripts/tapeDaemon.pl $lockFile $realTapeLabel $tapeFileList $sequenceType$numFiles $xList > $logFile 2>&1";

	    $command = "$baseCommand &";

	    system("$command");

	    sleep($sleepTimeTapeDaemon); # sleep for a while until tapeDaemon actually runs.

	    print "\n submitted command $command\n";

	    $now = `date +"%D %T"`;
	    chop($now);
	    $histFile = "$statusDir/tapeDaemon.hist";
	    open(HIST, ">>$histFile");
	    if ($submittedFile =~ /($remoteHost-$realTapeLabel-no\d+)/) {	    
		$line = sprintf("start $1 : $now");
	    }
	    print HIST "$line\n";
	    close (HIST);
	    
	    $iStream++;
	}

	if ($iStream >= $maxNStreamTape) {last;}
    }

    return 0;
}



sub checkConsistency {

#consistency check 0
#if another taper is running in one of the CDR's, die.

    $lockFile = "$lockDir/taper.lock";
    if (-e $lockFile) {
	open(LOCKFILE, "<$lockFile") || die "$0> cannot open $lockFile\n";
	$line = <LOCKFILE>;
	($anotherPid, $anotherHost) = split(' ', $line);
	@processList = ();
	@processList = `rsh -n $anotherHost ps -ax --cols=200|grep taper|grep $anotherPid`;
	$anotherProcess = 0;
	foreach $process (@processList) {
	    if ($process !~ /grep/) {
		$anotherProcess ++;
	    }
	}
	if ($anotherProcess > 0) {
	    die "$0> lock file $lockFile exist, and process $anotherProcess running on $anotherHost. please check\n";
	} else {
	    print "$lockFile exists but taper.pl process does not exist any longer.. removing $lockFile\n";
	    unlink $lockFile || die "$0> couldn't remove $lockFile\n";
	}
    }

    open(PID, ">$lockFile") || die "cannot open lock file $lockFile\n";
    print PID "$$ $host";
    close PID;
    
#consistency check 1
# between lock file and tapeDaemon jobs on CDR's

    @lockFileList=();

    @lockFileList = <$lockDir/Y?????.lock>;

    foreach $lockFile (@lockFileList) {
	$lockFile =~ /(Y\d{5})\.lock/;
	open(LOCK, "<$lockFile");
	$line = <LOCK>;
	close(LOCK);
	($processId, $remoteHost) = split(' ', $line);

	$nProcess = 0;
	@processList=();
	@processList = `rsh -n $remoteHost ps -ax --cols=200|grep $TAPE_ROOT/scripts/tapeDaemon|grep $processId`;

	foreach $process (@processList) {
	    if ($process !~ /grep/) {
		$nProcess ++;
	    }
	}
	$nProcess2 = 0;
	print "rsh -n $remoteHost ps -ax --cols=200|grep dumpTape|grep $processId\n";
	@processList2 = `rsh -n $remoteHost ps -ax --cols=200|grep dumpTape|grep $processId`;
	foreach $process2 (@processList2) {
	    if ($process2 !~ /grep/) {
		$nProcess2 ++;
	    }
	}

	if (($nProcess == 0)&&($nProcess2 == 0)) {
	    print "$0> WARNING: $lockFile exist but tpwrite and dumpTape.pl not running on $remoteHost\n";
	}
    }

#consistency check 2
#between submitted_daemon file and tpwrite jobs
    @daemonFileList=();
    @daemonFileList = <$daemonDir/na45pc??-Y?????-no????.submitted>;

    foreach $daemonFile (@daemonFileList) {
	$daemonFile =~ /(na45pc\d{2})-(Y\d{5})-no(\d{4})\.submitted/;
	$remoteHost = $1;
	$tapeLabel = $2;
	$numbering = $3;
	$nProcess = 0;
	@processList=();
	@processList = `ps -ax --cols=200|grep "rsh -n $remoteHost $TAPE_ROOT/scripts/tapeDaemon"`;

	foreach $process (@processList) {
	    print "process = $process\n";
	    if ($process !~ /grep/) {
		$nProcess ++;
	    }
	    if ($nProcess == 0) {
		print "$0> WARNING: $daemonFile exist but tapeDaemon.pl not running on $remoteHost\n";
	    }
	}
    }


#consistency check 3
# scan Taping directory and check the status of each file
#    print "scanning $TapingDir...\n";
#    @tapingFileList=();
#    @tapingFileList = <$TapingDir/na45pc*-run*-burst*.link>;
#    foreach $tapingFile (@tapingFileList) {
#	($tempPath, $tempSize) = split (' ', $tapingFile);
#    }
    return 1;
}

sub checkTapeSize {
    my $myTape = shift(@_);

    die "$0> myTape undefined\n" unless (defined $myTape);
#consistency check 4
# size in control/size-na45pc**-Y*****.daemon and control/size-na45pc**-Y*****.written

    my $tapeSizeDaemon  = getTapeSize($myTape);
    my $tapeSizeWritten = getTapeSizeWritten($myTape);

    if (($tapeSizeWritten > $tapeSizeDaemon) || ($tapeSizeWritten > $tapeSize) 
	|| ($tapeSizeDaemon > $tapeSize)){
	print "$0::checkTapeSize WARNING>\n";
	print "tape = $myTape\n";
	print "tapeSizeWritten = $tapeSizeWritten\n";
	print "tapeSizeDaemon  = $tapeSizeDaemon\n";
#	die "Aborting...\n";
    }
    return 1;
}

sub cleanup {
#do nothing so far (2000)
# it is important to remove setFileList and setInfo
# we recycle those files...
# leave list/*
#    system("rm $controlDir/size-Y*-current.dat");
#    system("rm $daemonDir/na45pc*-Y*.waiting");
#    system("rm $daemonDir/na45pc*-Y*.submitted");

    return 1;
}

# when restarting
sub restart {

    @submittedFileList=();
    @submittedFileList = <$daemonDir/na45pc??-Y?????-no????.submitted>;
    $totalSubmittedSize = 0.;
    foreach $submittedFile (@submittedFileList) {
	`touch $submittedFile`; # update time
	if ($submittedFile =~ /(na45pc\d{2})-(Y\d{5})-no(\d{4})\.submitted/) {

	    $remoteHost = $1;
	    $realTapeLabel = $2;
	    $numbering = $3;
	    $lockFile = "$lockDir/$realTapeLabel.lock";
	    $logFile = sprintf("$logDir/$remoteHost-$realTapeLabel-no%04d.log.submitted", $numbering);

	    checkTapeSize($realTapeLabel);

# before unlink, check process
	    if (-e $lockFile) {
		@processList = ();
		open(LOCK, "<$lockFile");
		$line = <LOCK>;
		close(LOCK);
		($processId, $remoteHost) = split(' ', $line);

		@processList = `ps -ax --cols=200|grep "rsh -n $remoteHost $TAPE_ROOT/scripts/tapeDaemon|grep $processId"`;

		$nProcess = 0;

		foreach $process (@processList) {
		    if ($process !~ /grep/) {
			$nProcess ++;
		    }
		}

		@processList1 = `rsh -n $remoteHost ps -ax --cols=200|grep dumptape|grep $processId`;

		$nProcess1 = 0;

		foreach $process1 (@processList1) {
		    if ($process1 !~ /grep/) {
			$nProcess1 ++;
		    }
		}

		if (($nProcess == 0) && ($nProcess1 == 0)) {
		    print "$0> restart > no process for $lockFile running\n";
		}

	    }
	    
#	    mailMsg("restart> $submittedFile exist\n. Check $logFile\n");
	}
    }    
    return;
}

# mark tapeSummaryList with "old" or "full" and
# update its size
# if tag = "" no update for tag
# if size <0, no update for size
sub markTapeSummary {
# usage makeTapeSummary(tape, tag, size);
    my $tape = shift(@_);
    my $tag  = shift(@_);
    my $size = shift(@_);
    die "markTapeSummary> usage: markTapeSummary tape tag size\n"
	unless (defined $tape && defined $tag && defined $size);

    my $tapeList = "$listDir/tapeSummaryList.dat";
    my $oldTapeList = "$listDir/tapeSummaryList.dat.sav";
    
    `cp $tapeList $oldTapeList`;
    my $newTapeList = "$listDir/tapeSummaryList.dat.new";
    
    open (TAPELIST, "<$tapeList") ||
	die "$0> cannot open $tapeList\n";
    
    open (NEWTAPELIST, ">$newTapeList") ||
	die "$0> cannot open $newTapeList\n";
    
    my $line, $newline;
    while ($line = <TAPELIST>) {
	$line =~ /(Y\d+) (\w+) (\d+)/;
	my $tempTape = $1;
	my $tempTag  = $2;
	my $tempSize = $3;

	if ($tempTape eq $tape) {
	    if ($tag eq "") {
		$tag = $tempTag;
	    }
	    if ($size < 0) {
		$newline = "$tape $tag $tempSize\n";
	    } else {
		$newline = "$tape $tag $size\n";
	    }
	} else {
	    $newline = $line;
	}
	
	print NEWTAPELIST $newline;
    }
    close TAPELIST;
    close NEWTAPELIST;
    `cp $newTapeList $tapeList`;
    
    return 1;
}

sub finish {
    $lockFile = "$lockDir/taper.lock";
    unlink($lockFile);
    return;
}


sub finishTape {

    my $tape = shift(@_);
    my $size = shift(@_);
    my $CDRName = shift(@_);

    die "finishTape> usage: finishTape tape CDRName\n" unless (defined $tape && defined $size && defined $CDRName);


    my $tag = getTapeStatus($tape);

    if ($tag eq "complete") {
# if it is already done do nothing.
	return 1;
    }

    if ($size < 0) {
	markTapeSummary($tape, "complete", -1);
    } else {
	markTapeSummary($tape, "complete", $size);
    }

# lock tape (after completion)
# take dumptape log
#         sysreq tms lock disable write vid $tapename &

    print "submitting dumptape -V $tape\n";
    $command = "$TAPE_ROOT/scripts/dumpTape.pl $tape $CDRName &";
    system($command);
    
    return 1;
}

sub mailMsg {
    ($sec,$min,$hr,$mday,$mon,$yr,$wday,$yday,$isdst) = localtime(time);
    @months=("Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec");
    $now=sprintf("%2.2d-%s-%2.2d %2.2d:%2.2d:%2.2d",$mday,$months[$mon],$yr,$hr,$min,$sec);
    open (MAIL, "|/bin/mail H.Sako\@gsi.de");
    print MAIL "\nThis is an automatic mail from the CDR on host $host.\n";
    print MAIL "Message from $0 at $now : \n";
    print MAIL "\n @_ ";
    close MAIL;
    return;
}

# calculate data recording rate
sub updateStatistics {

    $statusFile = "$statusDir/taper.sta";
    open(STAT, ">>$statusFile");
    $sec = time();

    if ($iter == 0) {
	$totalDataWritten = 0.; # in GB
	$secInit = $sec;
	$dataWritten = 0.;
	for ($iCDR=0;$iCDR<$maxNCDR;$iCDR++) {
	    $dataCDR[$iCDR] = 0.;
	}
	$secOld = $sec;
    }

    if (($iter>0) && ($iter%5 == 0)) {
	$totalSecond = $sec-$secInit;
	$totalSpeed = ($totalDataWritten*1024.)/$totalSecond;
	$temp = $totalDataWritten*1024.;
	print "Speed [total] : $temp / $totalSecond = $totalSpeed [MB/s]\n";

	$second = $sec-$secOld;
	$speed = ($dataWritten*1024.)/$second;
	$temp = $dataWritten*1024.;
	print "Speed [5 loops] : $temp / $second = $speed [MB/s]\n";
	$secOld = $sec;

	$now = `date +"%D %T"`;
	chop($now);
	$line = sprintf("$now %6d %9.3f %8d %6.3f %6.3f %3d %6.3f %2d %1d %1d %1d %1d %1d %1d %1d", $iter,
			$totalDataWritten, $totalSecond, $totalSpeed,
			$dataWritten, $second, $speed, $NStream,
			$nStreamInCDR[0], $nStreamInCDR[1], $nStreamInCDR[2],
			$nStreamInCDR[3], $nStreamInCDR[4], $nStreamInCDR[5],
			$nStreamInCDR[6]);
#	$line = sprintf("$now %6d %9.3f %8d %6.3f %6.3f %3d %6.3f %2d", $iter,
#			$totalDataWritten, $totalSecond, $totalSpeed,
#			$dataWritten, $second, $speed, $NStream);
	print STAT "$line\n";

	print "dataCDR : ";
	for ($iCDR=0; $iCDR<$maxNCDR; $iCDR++) {
	    $line = sprintf("%6.3f", $dataCDR[$iCDR]);
	    printf " $line";
	}
	print "\n";
	print "nStreamCDR : ";
	for ($iCDR=0; $iCDR<$maxNCDR; $iCDR++) {
	    print " $nStreamInCDR[$iCDR]";
	}
	print "\n";

	$dataWritten = 0.;
	for ($iCDR=0;$iCDR<maxNCDR;$iCDR++) {
	    $dataCDR[$iCDR] = 0.;
	}
    }    
    close(STAT);
    return;
}

sub init() {
    $mainLogFile = "$logDir/taper.log";
    open(SAVEOUT , ">&STDOUT");
    open(SAVEERR , ">&STDERR");

    open(STDOUT, ">$mainLogFile");
    open(STDERR, ">&STDOUT");
    return;
}

sub backupLists() {

    $now = `date`;
    @dateList = split(' ', $now);
    @timeList = split(':', $dateList[3]);
# only use date for time stamp No. of backup lists will be reduced...
    $dating = "$dateList[2]$dateList[1]$dateList[5]";

    $tapeSummary = "$listDir/tapeSummaryList.dat";
    $tapeSummaryBackup = "$listDir/backup/tapeSummaryList.dat.$dating";

    $mainLogFile = "$logDir/taper.log";
    $mainLogFileSav = "$logDir/backup/taper.log.$dating";

    if (-e $mainLogFile) {
	$command = "cat $mainLogFile >> $mainLogFileSav";
	system($command);
#	close(STDOUT);?
	open(STDOUT, ">$mainLogFile"); # reopen
	open(STDERR, ">&STDOUT");
    }

    $statusFile = "$statusDir/taper.sta";
    $statusFileSav = "$statusDir/backup/taper.sta.$dating";

    if (-e $statusFile) {
	$command = "cat $statusFile >> $statusFileSav";
	system($command);
	unlink $statusFile;
    }

    if ($printLevel > 50) {
	print "cp $tapeSummary $tapeSummaryBackup\n";
    }

    $histFile = "$statusDir/tapeDaemon.hist";
    $histFileSav = "$statusDir/backup/tapeDaemon.hist.$dating";

    if (-e $histFile) {
	$command = "cat $histFile >> $histFileSav";
	system($command);
	unlink $histFile;
    }

    if ($printLevel > 50) {
	print "cp $tapeSummary $tapeSummaryBackup\n";
    }

    `cp $tapeSummary $tapeSummaryBackup`;

    @tapeLists = <$listDir/Y?????.list>;
    foreach $tapeList (@tapeLists) {
	if ($tapeList =~ /(Y\d{5}\.list)/) {
	    $tapeListBackup = "$listDir/backup/$1";
	    if ($printLevel > 50) {
		print "tapeListBackup = $tapeListBackup\n";
		print "cp $tapeList $tapeListBackup\n";
	    }
	    `cp $tapeList $tapeListBackup`;
	}
    }

    @tapeLists = <$listDir/Y?????.final>;
    foreach $tapeList (@tapeLists) {
	if ($tapeList =~ /(Y\d{5}\.final)/) {
	    $tapeListBackup = "$listDir/backup/$1";
	    if ($printLevel > 50) {
		print "tapeListBackup = $tapeListBackup\n";
		print "cp $tapeList $tapeListBackup\n";
	    }
	    `cp $tapeList $tapeListBackup`;
	}
    }
    return;
}

sub getLimitTapeSize {

    $limitTapeSizeFile = "$controlDir/limitTapeSize";

    if (-e $limitTapeSizeFile) {
	open (LIMIT,"<$limitTapeSizeFile");
	$limitTapeSize = <LIMIT>;
	$limitTapeSize *= $tapeSize;
	print "*** limitTapeSize = $limitTapeSize\n";
	close (LIMIT);
    }
    return;
}



sub checkSizeInMasterList {
    my $masterList = shift(@_);

    die "checkSizeInMasterList(): Failed to get $masterList \n"
	unless defined $masterList;

    if (!(-e $masterList)) {
	print "$masterList does not exist\n";
	return 0;
    }
    $tempTotalSize = 0;
    open(MASTERLIST, "<$masterList") ||
	die "$0> cannot open $masterList\n";
    while ($line = <MASTERLIST>) {
	($tempPath, $tempSize) = split(' ', $line);
	$tempTotalSize += $tempSize;
    }
    close (MASTERLIST);
    return $tempTotalSize;
}

sub signal {
    $sigFile = "$controlDir/signal";
    open(SIG, "<$sigFile") || return;
    $line = <SIG>;

    if (($line =~ /stop/) ||($line =~ /STOP/)) {
	print "signal> stop signal detected... stoppping\n";
	close(SIG);
	finish();
	exit;
    }

    if (($line =~ /pause/) || ($line =~ /PAUSE/)) {
	close(SIG);
	do {
	    print "signal> pause detected... sleeping\n";
	    sleep(10);
	    open(SIG, "<$sigFile") || next;
	    $line = <SIG>;
	    close(SIG);
	} while (($line !~ /continue/) && ($line !~ /CONTINUE/));
	return;
    }
    return;
}



sub getAge() {
    my $tfile = shift(@_) if defined (@_);

    die "Usage getAge(file)\n" unless (defined $tfile);

    ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,
     $atime,$mtime,$ctime,$blksize,$blocks) = stat($tfile);
    ($fsec,$fmin,$fhour,$mday,$mon,$year,$wday,$yday,$isdt)
	= localtime($mtime);
    $timeStamp = sprintf("%02d:%02d", $fhour, $fmin);
    $dateStamp = sprintf("%02d.%02d.%02d", $mday,$mon+1,$year);

    $t = time();
#    ($sec,$min,$hour,$mday,$mon,$yr,$wday,$yday,$isdst) = localtime(time);

    my $age = ($t-$mtime)/60./60./24.;

    $temp = $t-$mtime;
#    print "t-mtime [sec] = $temp\n";

    return $age;
}