#!/usr/local/bin/perl5
$|=1;
# revision 
# 24 Nov 99 19:50
# added $(matches at the end of the string on regular expression /$dataFile$/
# and (20:08) changed add additional space charactor in grep command
# need recheck tape and seq number later
# up to ~RUN 750 with burst numbers >=100
# 03 Oct 00
# changed to use disks only up to data06


use File::Basename;
use FileHandle;

$clean = 1;
$debug = 0;
$TAPE_ROOT  = "/cdrShare01/cerescdr/cdr2k/tape";

$freqFree = 20; # frequency to update free space (No. of files to be deleted at the same time)
# increased from 10 to 20 on 25 sep 00

@CDRHost = ("na45pc07", "na45pc08", "na45pc09", "na45pc10",
	    "na45pc11", "na45pc12", "na45pc14");

$sleepTime = 60.; # [sec] 1 min corresponds to 3 bursts = 450MB*3/7=200MB...

#$minFree = 0.5; # minimum free space 50 % (changed from 30% on 21 Nov 99)
#$minFree = 0.80; # minimum free space 50 % (after run 2000)
$minFree = 0.80; # minimum free space 50 % (after run 2000)

# minimum required free space on each CDR
                # 12 hours mergin
$freqBackup = 100;

chdir ($TAPE_ROOT) || die "$0> cannot chdir $TAPE_ROOT\n";

chop($host = `hostname`);

$logFile = "$TAPE_ROOT/logs/cleaner-$host.log";

$lockFile = "$TAPE_ROOT/lock/cleaner-$host.lock";

if ($clean == 1) {
if (-e $lockFile) {
    die "$0> $lockFile exist. check process.\n";
}
}

if ($clean == 1) {
open(PID, ">$lockFile") || die "cannot open lock file $lockFile\n";
print PID "$$ $host";
close PID;
}

$statusFile = "$TAPE_ROOT/status/cleaner-$host.sta";
$iter = 0;
if ($clean == 1) {
    init();
}
while(1) {

    $cleanedFile = 0;
    $cleanedSize = 0;

    $now = `date +"%D %T"`;
    chop($now);
    print "Start loop $iter $now\n";
    if ($clean == 1) {
	@out = `$TAPE_ROOT/scripts/cleanTestRun.pl`;
	foreach $line (@out) {
	    if ($line =~ /deleted/) {
		chop ($line);
		print "$line in cleanTestRun.pl\n";
	    }
	}
    }

    if ($iter % $freqBackup ==0) {
	if ($clean == 1) {
	    backupLog();
	}
    }
  
    @linkFileList = <$TAPE_ROOT/toClean/$host-run????-burst????.link>;

# if there is not linkFile..    
    
    $counter = 0;
#    $free = $minFree - 0.1; # initialize free to be smaller than minFree
#    $free = getFreeSpace(); # update free space once before loop
    $used = getFreeSpace(); # update free space once before loop
    foreach $linkFile (@linkFileList) {
	
	if (!open(LINKFILE, "<$linkFile")) {
	    $counter++;
	    next;
	}
	$linkContent = <LINKFILE>;
	($path, $dataSize) = split(" ", $linkContent);
	close (LINKFILE);
	if ($path =~ /run(\d{4})-burst(\d{4})\.dat/) {
	    $run   = $1;
	    $burst = $2;
	    if ($debug == 1) {
		print "run = $run, burst = $burst\n" 
	    }
	} else {
	    $counter++;
	    next;
	}

# HS Use free space instead of age
	if ($counter % $freqFree == 0) {
#hs	    $free = getFreeSpace();
	    $used = getFreeSpace();
#	    print "counter $counter :free space = $free\n";
	}

#hs	if ($free < $minFree) {
	if ($used > 32*1024*1024) { # 32GB
	    
#	    @dumpList = ();
#	    @dumpList = <$TAPE_ROOT/list/Y?????.final>;
	    $dataFile = sprintf("RUN%04d-B%04d.DAT", $run, $burst);
	    $nFileInTape = 0;

	    if ($debug == 1) {
		print "dataFile = $dataFile\n" 
	    }

# list/Y30394.final:RUN1500-B0154.DAT Y30394 0198 258.41 26.10.00 22:24
#	    foreach $dumpTape (@dumpList) {
#		if ($dumpTape =~ /(Y\d{5})\.final/) {
	    @lines = ();
	    @lines = `grep \'$dataFile\' $TAPE_ROOT/list/Y*.final`;

		    if ($#lines + 1 <= 0) {
#			$counter++;
			next;
		    }

		    foreach $line (@lines) {
			chop($line);
			if ($line =~ /$TAPE_ROOT\/list\/(Y\d{5})\.final\:RUN(\d{4})-B(\d{4})\.DAT (Y\d{5}) (\d{4})\s+(\d+\.?\d*|\.\d+) (\d{2}\.\d{2}\.\d{2}) (\d{2}\:\d{2})/) {

			    $tape = $1;
			    $runc = $2;
			    $burstc = $3;
			    $tapec = $4;
			    $seq = $5;
			    $size = $6;
			    $date = $7;
			    $time = $8;

			    ($final, $content, $min) = split(':', $line);
			    $content = "$content:$min";
			    if ($debug == 1) {
				print "tape = $tape\n";
				print "tapec = $tapec\n";
				print "run = $run\n";
				print "runc = $runc\n";
				print "burst = $burst\n";
				print "burstc = $burstc\n";
#				print "final = $final\n";
				print "content = $content\n";
			    }
			    
			    if ($debug == 1) {
				print "runc = $runc, burst = $burstc\n";
			    }

			    if (($run != $runc) || ($burst != $burstc)) {
#				$counter++;
				next;
			    }
# how about for each tape instead of a common file

			    $finalDataList = "$TAPE_ROOT/list/$tape.cleaned";
		
			    
			    $exist = `grep -c \'$content\' $finalDataList`;
			    $exist = int($exist);

			    if ($debug == 1) {
#				print "grep -c $content $finalDataList\n";
#				print "exist = $exist\n";
			    }


# changed on 25.09.00
#			    print "$content\n";
			    if ($exist == 0) {
				if ($clean == 1) {
				    open(CHECKTAPED, ">>$finalDataList");
				    printf CHECKTAPED "$content\n";
				    close CHECKTAPED;
				}
			    }

			    $cleanedLinkFile
				= sprintf("$TAPE_ROOT/Cleaned/$host-run%04d-burst%04d.link", $run, $burst);
			    
			    $nFileInTape++;
			    if ($debug == 1) {
				print "going to removeData\n";
			    }

			    if ($clean == 1) {
				removeData($linkFile, $path, $cleanedLinkFile);
			    }
			    $cleanedFile++;
			    $cleanedSize += $size; # in MB
			}
		    }
#		} # if ($dumpTape =~ /(Y\d{5})\.final/) 
#	    } # loop foreach $dumpTape
	    
	    if ($nFileInTape == 0) {
		$lost = 1; # is the data file lost in tape?
		
		# the data file is identified to be lost in tape
		# if
		# 1) it is NOT included in Y?????.list
		# 2) or, it is in the Y?????.list but is not in Y?????.final
		# (if Y?????.final exits)
		
		@output = `grep -i $dataFile $TAPE_ROOT/list/Y?????.list`;

		#if Y?????.list exists, check if corresponding Y?????.final
		# has been already made or not yet
		if ($#output+1>0) {
		    foreach $out (@output) {
			if ($out =~ /(Y\d{5})\.list/) {
			    $tape = $1;
			    $tapeFinalList = "$TAPE_ROOT/list/Y$tape.final";
			    if (!(-e $tapeFinalList)) {
				#not yet Y$tape.final is made
				$lost = 0;
			    }
			}
		    }
		}

		if ($lost == 1) {
		    $missingList = "$TAPE_ROOT/list/notTaped-$host.dat";

		    @templines = `grep \'$linkFile\' $missingList`;
		    if ($#templines + 1 <= 0) {
			print "missing file: $linkFile\n";
			if ($clean == 1) {
			    open(MISSING, ">> $missingList");
			    print MISSING "$linkFile\n";
			    close MISSING;
			}
		    }
		}
	    }
	} else { # if free space is enough, sleep...
	    last; # update linkFileList (important)
	}
	$counter ++; # no. of checked link files
    }

    sleep($sleepTime);

    if ($clean == 1) {
	open(STAT, ">>$statusFile");
	$now = `date +"%D %T"`;
	chop($now);
#    print STAT "$iter : $now $cleanedFile $cleanedSize\n";
	$line = sprintf("%10d : $now %4d %8.2f %4.2f",
			$iter, $cleanedFile, $cleanedSize, $free);
	print STAT "$line\n";
	close (STAT);
    }
    
    $iter ++;
}

exit;


sub removeData {
    my $link = shift(@_) if defined (@_);
    my $data = shift(@_) if defined (@_);
    my $cleanedLink = shift(@_) if defined (@_);
    

    if (!(defined $link && defined $data && defined $cleanedLink)) {
	print "$0> usage removeData link data cleanedLink\n";
	return;
    } 


    rename $link, $cleanedLink;
    $command = "rm $data\n";   # Commented out temporarily not to remove any data
    $unlinkData = `$command`;

    return;
}


sub getFreeSpace {


    if (($host eq "na45pc08")
	||($host eq "na45pc10")) {
	
	@disks = (
		  "/shift/$host/data01",
		  "/shift/$host/data02",
		  "/shift/$host/data03",
		  "/shift/$host/data04",
		  "/shift/$host/data05"
		  );
    }
    elsif ($host eq "na45pc11") {
	@disks = (
		  "/shift/$host/data01",
		  "/shift/$host/data02",
		  "/shift/$host/data03",
		  "/shift/$host/data04",
		  "/shift/$host/data05",
		  "/shift/$host/data06"
		  );
    }
    elsif (($host eq "na45pc12") || ($host eq "na45pc14")) {
	@disks = (
		  "/shift/$host/data01",
		  "/shift/$host/data02",
		  "/shift/$host/data03",
		  "/shift/$host/data04",
		  "/shift/$host/data05",
		  "/shift/$host/data06"
		  );

    }
    else {
	@disks = (
		  "/shift/$host/data01",
		  "/shift/$host/data02",
		  "/shift/$host/data03",
		  "/shift/$host/data04"
		  );
    }
    
    $totalFree = 0;
    $totalSpace = 0;
    foreach $disk (@disks) {
	@diskSizeList = `df -k $disk`;
	foreach $diskSize (@diskSizeList) {

	    if ($diskSize =~ /$disk/) {
		@space = ();
		@space = split(' ', $diskSize);
		$totalFree += $space[3];
		$totalSpace += $space[1];
	    }
	}
    }

    $usedSpace = $totalSpace-$totalFree;
    $occupancy = $totalFree/$totalSpace;
#    printf "$host : free %9d/%9d (%4.1f %)\n", $totalFree, $totalSpace, $occupancy*100.;
    printf "$host : free %9d/%9d (%4.1f %) used = $usedSpace\n", $totalFree, $totalSpace, $occupancy*100.;

# default    return $occupancy;
# return used space
    return $usedSpace;
}


sub init() {
    open(SAVEOUT , ">&STDOUT");
    open(SAVEERR , ">&STDERR");

    open(STDOUT, ">$logFile");
    open(STDERR, ">&STDOUT");
    return;
}

sub backupLog {
    $now = `date`;
    chop($now);
    @dateList = split(' ', $now);
    $dating = "$dateList[2]$dateList[1]$dateList[5]";
    $logFileSav = "$TAPE_ROOT/logs/backup/cleaner-$host.log.$dating";
    $statusFileSav = "$TAPE_ROOT/status/backup/cleaner-$host.sta.$dating";

    if (-e $logFile) {
	$command = "cat $logFile >> $logFileSav";
	system($command);
	open(STDOUT, ">$logFile"); # reopen
	open(STDERR, ">&STDOUT");
    }

    if (-e $statusFile) {
	$command = "cat $statusFile >> $statusFileSav";
	system($command);
	$command = "rm $statusFile";
	system($command);
    }

    return;
}