#!/usr/bin/perl -w
###############################################################################
# h.Sako 17.08.00
###############################################################################
use Tk;
use Tk::NoteBook;
use FileHandle;

Tk::CmdLine::SetResources('*font: -*-courier-*-r-*-*-12-*-*-*-*-*-*-*');
# Tk::CmdLine::SetResources('*font: -etl-fixed-*-*-*-*-14-*-*-*-c-*-*-*');
# Tk::CmdLine::SetResources('*font: -elot-fixed-*-*-*-*-13-*-*-*-*-*-*-*');
#Tk::CmdLine::SetResources('*background: #ffa050');
#Tk::CmdLine::SetResources('*background: wheat2');
Tk::CmdLine::SetResources('*background: skyblue');
#Tk::CmdLine::SetResources('*background: white');


&initScript;
&initTk;
&slowUpdate;
&middleUpdate;
&fastUpdate;
MainLoop;

exit;

###############################################################################
###############################################################################
sub initTk {
    $mw = new MainWindow;
    $mw->repeat($fastPeriod, sub{&fastUpdate;});
    $mw->repeat($middlePeriod, sub{&middleUpdate;});
    $mw->repeat($slowPeriod, sub{&slowUpdate;});

# common header

    $header = $mw->Frame()->pack(-fill => "x");
    $header->Button(-text => "exit", -command => sub {exit;})->pack(-side => "right");
    $header->Button(-text => "update", -command => \&update)->pack(-side => "right");
    $header->Label(-textvariable => \$now)->pack(-side => "left");

# NoteBook: pages of information:

    $nb = $mw->NoteBook();


    $mach  = $nb->add("mach",  -label => " CDR Status  ");
    $evb   = $nb->add("evb",   -label => "Event Builder");
    $tape  = $nb->add("tape",  -label => "    Taper    ");
    $clean = $nb->add("clean", -label => "   Cleaner   ");

# individual pages: evb
#    $evb_but = $evb->Frame()->pack(-padx => 20, -pady => 20);
    $evb_rin = $evb->Frame()->pack(-fill => 'x', -padx => 50, -pady => 40);
    $evb_bst = $evb->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10, -side => 'top');
#    $evb_rst = $evb->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10, -side => 'top');

    $l_gsta_evb = tk_add_slice(\$evb_rin,
			     "Event Builder Overall Status : ",
			      \$gStatEvb, "blue");

    @l_pro_evb = ();
    tk_add_title(\$evb_rin, "\nNo. of evb Processes");
    for ($index=0;$index<($#ebpc+1);$index++) {
	$l_pro_evb[$index] = tk_add_slice(\$evb_rin,
			     "$ebpc[$index] ",
					  \$nEvb[$index], "blue");
    }

    @l_pro_receiver = ();
    tk_add_title(\$evb_rin, "\nNo. of daqmem_receiver Processes");
    for ($index=0;$index<($#ebpc+1);$index++) {
	$l_pro_receiver[$index] = tk_add_slice(\$evb_rin,
					       "$ebpc[$index] ",
					       \$nReceiver[$index], "blue");
    }


    tk_add_title(\$evb_rin, "\nevb Status Logs");
    tk_add_title(\$evb_rin, "           rcv-pc   run brst   ev     MB    start-time     end-time  evm egr ecp  status");
    for ($index=0;$index<($#ebpc+1);$index++) {
	$evb_stat[$index] = tk_add_slice(\$evb_rin, "$ebpc[$index] ",
				   \$statEvb[$index]);
    }


    tk_add_title(\$evb_bst, "Evb History");
    $evb_hist = $evb_bst->Scrolled("Text", -width => 10, -height => 10)->pack(-side => 'top',
						  -fill => 'both',
						  -expand => 1);
    $evb_hist->tagConfigure("blue", -foreground => "blue");
    $evb_hist->tagConfigure("red", -foreground => "red");
    $evb_hist->tagConfigure("magenta", -foreground => "magenta");



# individual pages: tape

    $tape_but = $tape->Frame()->pack(-padx => 20, -pady => 20);
    $tape_rin = $tape->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10);
    $tape_bst = $tape->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10, -side => 'top');
#    $tape_rst = $tape->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10, -side => 'top');

    $l_gsta_taper = tk_add_slice(\$tape_rin,
				 "Taper Overall Status : ",
				 \$gStatTaper, "blue");

    $b_sub_taper = $tape_but->Button(-text => "submit taper",
				     -command => \&submitTaper,
				     -state => 'disabled',
				     -width => 15)->pack(-side => "left");

    $b_stp_taper = $tape_but->Button(-text => "stop taper",
				     -command => \&stopTaper,
				     -state => 'disabled',
				     -width => 15)->pack(-side => "left");


    tk_add_title(\$tape_bst, "Taper Log");
    $tape_text = $tape_bst->Scrolled("Text", -width => 20, -height => 10)->pack(-side => 'top',
						  -fill => 'both',
						  -expand => 1);
    $tape_text->tagConfigure("blue", -foreground => "blue");
    $tape_text->tagConfigure("red", -foreground => "red");


    tk_add_title(\$tape_bst, "Tape Summary List");
    $tape_sum_list = $tape_bst->Scrolled("Text", -width => 6, -height => 10)->pack(-side => 'left',
						       -fill => 'both',
						       -expand => 1);
    $tape_sum_list->tagConfigure("blue", -foreground => "blue");
    $tape_sum_list->tagConfigure("red", -foreground => "red");

    tk_add_title(\$tape_bst, "Tapedaemon History");
    $tape_hist = $tape_bst->Scrolled("Text", -width => 20, -height => 10)->pack(-side => 'right',
						       -fill => 'both',
						       -expand => 1);
    $tape_hist->tagConfigure("blue", -foreground => "blue");
    $tape_hist->tagConfigure("red", -foreground => "red");



    $l_pro_taper = tk_add_slice(\$tape_rin, "Tape Host : ", \$tppc);
    $l_pro_taper = tk_add_slice(\$tape_rin, "No. of taper.pl processes ", \$tapeProcess,
				"blue");

    tk_add_title(\$tape_rin, "\nNo. of queues");
    $l_nwaiting = tk_add_slice(\$tape_rin, "Waiting      ", \$nWaiting);
    tk_add_slice(\$tape_rin, "Submitted    ", \$nSubmitted);
#    tk_add_slice(\$tape_rin, "Finished     ", \$nFinished);
    tk_add_slice(\$tape_rin, "Problem      ", \$nProblem);
    tk_add_slice(\$tape_rin, "ProblemCheck ", \$nProblemCheck);

    tk_add_title(\$tape_rin, "\nTaper Status Log");
    tk_add_title(\$tape_rin, "    date     time     loop     S[GB]   dT[s] dS/dT[MB/s] s  dt   ds/dt  nTD");
    tk_add_slice(\$tape_rin, " ", \$statTape);
#    tk_add_title(\$tape_rin, "\nNo. of tapeDaemon.pl processes");
#
#    for ($index=0;$index<($#ebpc+1);$index++) {
#	tk_add_slice(\$tape_rin, "$ebpc[$index] ", \$nTapeDaemon[$index]);
#    }

# individual pages: clean

    $clean_but = $clean->Frame()->pack(-padx => 20, -pady => 20);
    $clean_rin = $clean->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10);
#    $clean_bst = $clean->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10, -side => 'top');
#    $clean_rst = $clean->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10, -side => 'top');

    $l_gsta_cleaner = tk_add_slice(\$clean_rin,
				   "Cleaner Overall Status : ",
				   \$gStatCleaner, "blue");


    $b_sub_cleaner = $clean_but->Button(-text => "submit cleaners",
					-command => \&submitCleaner,
					-state => 'disabled',
					-width => 15)->pack(-side => "left");


    $b_kil_cleaner = $clean_but->Button(-text => "kill cleaners",
					-command => \&killCleaner,
					-state => 'disabled',
					-width => 15)->pack(-side => "left");


#    $t_err_cleaner = $clean_rst->Scrolled("Text")->pack(-side => 'top',
#							  -fill => 'both',
#							  -expand => 1);
#    $t_err_cleaner->tagConfigure("blue", -foreground => "blue");
#    $t_err_cleaner->tagConfigure("red", -foreground => "red");

#    tk_add_slice(\$clean_rin, "Cleaner Status Message", \$cleanerError);

    tk_add_title(\$clean_rin, "\nNo. of cleaner.pl processes");
    @l_pro_cleaner = ();
    for ($index=0;$index<($#ebpc+1);$index++) {
	$l_pro_cleaner[$index] = tk_add_slice(\$clean_rin, "$ebpc[$index] ", \$nCleaner[$index], "blue");
    }

    tk_add_title(\$clean_rin, "\nCleaner Status Log");
    tk_add_title(\$clean_rin,     " evb-pc         loop :   date     time   #file   S[MB] free disk space");
    for ($index=0;$index<($#ebpc+1);$index++) {
	tk_add_slice(\$clean_rin, "$ebpc[$index] ", \$statClean[$index]);
    }

    tk_add_title(\$clean_rin, "\nNo. of Data Files");
    tk_add_slice(\$clean_rin, "Before taping  ", \$nChecked);
    tk_add_slice(\$clean_rin, "Taping         ", \$nTaping);
    tk_add_title(\$clean_rin, "To be cleaned");
    for ($index=0;$index<($#ebpc+1);$index++) {
	tk_add_slice(\$clean_rin, "  $ebpc[$index]     ", \$nToClean[$index]);
    }

# individual pages: machine status


    $mach_rin = $mach->Frame()->pack(-fill => 'x', -padx => 50, -pady => 10);
    $l_gsta_mach = tk_add_slice(\$mach_rin,
				   "CDR Overall Status : ",
				   \$gStatCDR, "blue");

    tk_add_title(\$mach_rin, "\nStatus of NFS PC (with uptime)");

    $nfspcStatLabel = tk_add_slice(\$mach_rin, "$nfspc: ", \$nfspcStat);

    tk_add_title(\$mach_rin, "\nStatus of receiver PC's (with uptime)");

    for ($index=0;$index<($#ebpc+1);$index++) {
	$ebpcStatLabel[$index] = tk_add_slice(\$mach_rin, "$ebpc[$index]: ", \$ebpcStat[$index]);
    }


    tk_add_title(\$mach_rin, "\nStatus of taper PC's (with uptime)");

    for ($index=0;$index<($#tppca+1);$index++) {
	$tppcStatLabel[$index] = tk_add_slice(\$mach_rin, "$tppca[$index]: ", \$tppcStat[$index]);
    }


    tk_add_title(\$mach_rin, "\nFree Disk Space on receiver PC's");
#    tk_add_title(\$mach_rin, "            data01     data02     data03     data04     data05     data06     data07     data08     data09");
    tk_add_title(\$mach_rin, "            data01     data02     data03     data04     data05     data06");

    for ($index=0;$index<($#ebpc+1);$index++) {
	getDisks($ebpc[$index]);

	$slice = $mach_rin->Frame()->pack(-fill => 'x', -side => 'top');
	$slice->Label(-text => "$ebpc[$index]: ")->pack(-side => 'left');
	$color = "black";

	$nDisks = $#disks+1;
	if ($debug) {
	    print "nDisks = $nDisks\n";
	}

	for ($idisk=0;$idisk<$nDisks;$idisk++) {
	    $diskLabel[$index][$idisk] = $slice->Label(-textvariable => \$diskSpace[$index][$idisk],
						       -foreground => $color)->pack(-side => 'left');
	}

	$totalDiskLabel[$index] = tk_add_slice(\$mach_rin, "             ", \$totalDiskSpace[$index], "blue");
    }

    tk_add_title(\$mach_rin, "\nFree Disk Space of Shared Disk");
    $shareDiskLabel = tk_add_slice(\$mach_rin, "/cdrShare01 ", \$shareDiskSpace, "blue");



    
# pack it all:

    $nb->pack();

# open log files for reading

}
###############################################################################
sub tk_add_butr {        # add a button on a grid, $com is subroutine reference
    (scalar(@_) >= 3) or die "but1 needs 3 arguments\n";
    my $parent = $_[0];  # pointer to parent
    my $label  = $_[1];  # label string
    my $com    = $_[2];  # command
    my $height = $_[3];  # height, 1 or 2
    my $row    = $_[4];
    my $column = $_[5];
    my $but = $$parent->Button(-text => $label, 
			       -command => sub {$com->()});
    $but->grid(-rowspan=>$height, -row=>$row, -column=>$column, -sticky=>'ns');
}
###############################################################################
sub tk_add_buts {              # add a button on a grid, command is system $com
    (scalar(@_) >= 3) or die "but1 needs 3 arguments\n";
    my $parent = $_[0];  # pointer to parent
    my $label  = $_[1];  # label string
    my $com    = $_[2];  # command
    my $height = $_[3];  # height, 1 or 2
    my $row    = $_[4];
    my $column = $_[5];
    my $but = $$parent->Button(-text => $label, 
			       -command => sub {printf "$com\n";system($com)});
    $but->grid(-rowspan=>$height, -row=>$row, -column=>$column, -sticky=>'ns');
}
###############################################################################
sub tk_add_slice {  # make a horiz. slice in parent, containing label and value
    (scalar(@_) >= 3) or die "tk_add_slice needs 3 arguments\n";
    my $parent = $_[0];  # pointer to parent
    my $label  = $_[1];  # label string
    my $value  = $_[2];  # pointer to variable
    my $color  = $_[3];
    my $font   = $_[4];
    $color = "black" unless (defined $color);
    $slice = $$parent->Frame()->pack(-fill => 'x', -side => 'top');
    $slice->Label(-text => $label)        ->pack(-side => 'left');
    if (defined $font) {
	$labelPtr = $slice->Label(-textvariable => $value, -foreground => $color, -font => $font)->pack(-side => 'left');
    } else {
	$labelPtr = $slice->Label(-textvariable => $value, -foreground => $color)->pack(-side => 'left');
    }

    return $labelPtr;
}

sub tk_add_title {  # make a horiz. slice in parent, containing label and value
    (scalar(@_) >= 2) or die "tk_add_title needs 2 arguments\n";
    my $parent = $_[0];  # pointer to parent
    my $label  = $_[1];  # label string
    my $color  = $_[2];
    my $font   = $_[3];
    $color = "black" unless (defined $color);
    $slice = $$parent->Frame()->pack(-fill => 'x', -side => 'top');
    
    if (defined $font) {
	$labelPtr = $slice->Label(-text => $label,-foreground => $color,-font => $font) ->pack(-side => 'left');
    } else {
	$labelPtr = $slice->Label(-text => $label,-foreground => $color) ->pack(-side => 'left');
    }
    return $labelPtr;
}

sub update {
    slowUpdate();
    middleUpdate();
    fastUpdate();
    return;
}
###############################################################################
sub fastUpdate {
#    printf "that's fast part of updating\n";

    $now = `date +\"%D %T\"`;
    chop($now);

    fastEvb();

    return;
}

###############################################################################
sub middleUpdate {
#    printf "that's middle part of updating\n";

    $now = `date +\"%D %T\"`;
    chop($now);

    midEvb();
    midTape();

    return;
}
###############################################################################
sub slowUpdate {
#    printf "that's slow part of updating\n";
    slowCDR();
    slowClean();
    return;
}


###############################################################################
###############################################################################
sub initScript  {
    $rshpc = "";


    $minFree        = 0.4;  # minimum free space fraction / CDR
    $minFreePerDisk = 0.3;  # minimum free space fraction / disk
    $minFreeShare   = 0.1; # minimum free space fraction on /cdrShare01

    $debug = 0;

    $evbError     = 1; # normal 
    $taperError   = 1; # normal
    $cleanerError = 1; # normal 
    $cdrError     = 1; # normal 

    $gStatEvb     = "NORMAL";
    $gStatCleaner = "NORMAL";
    $gStatTaper   = "NORMAL";
    $gStatCDR     = "NORMAL";

    $monRoot  = "/home/daq/cdrmon";
    $tapeRoot = "/cdrShare01/cerescdr/cdr2k/tape";
    $evbRoot  = "/cdrShare01/cerescdr/cdr2k/evb";
    $statDir  = "$tapeRoot/status";
    $logDir   = "$tapeRoot/logs";
    $listDir  = "$tapeRoot/list";

    $toTapeDir = "$tapeRoot/toTape";
    $CheckedDir = "$tapeRoot/Checked";
    $TapingDir = "$tapeRoot/Taping";
    $toCleanDir = "$tapeRoot/toClean";
#    $CleanedDir = "$tapeRoot/Cleaned";
    $queueDir = "$tapeRoot/queue";

    $statTapeFile = "$statDir/taper.sta";

    $hostList = "$monRoot/hosts.config";
    eval `cat $hostList`;

    $tppc = $tppca[0];


    $nfspcAlive = 1;
    for ($index=0;$index<($#ebpc+1);$index++) {
	$ebpcAlive[$index] = 1;
    }

# define names of the logfiles

# msec
    $fastPeriod   = 10*1000;  # 10000 = 10 second
    $middlePeriod = 120*1000;  # 120000 = 120 second
    $slowPeriod   = 240*1000; # 240000 = 240 second

# global variables


    @statClean = ();
    @statEvb = ();
    @nEvb = ();
    @nReceiver = ();
    @nCleaner = ();

#    system("$monRoot/scripts/cdrview.pl &");
}

sub submitTaper {
    $script = "$monRoot/scripts/submitTaper.pl";
    system($script);
    return;
}

sub stopTaper {
    $script = "$monRoot/scripts/stopTaper.pl";
    system($script);
    return;
}

sub submitCleaner {
    $script = "$monRoot/scripts/submitCleaner.pl";
    system($script);
    return;
}

sub killCleaner {
    $script = "$monRoot/scripts/killCleaner.pl";
    system($script);
    return;
}

sub getDisks {
# 03.10.00 use only upto data06
    @disks = ();

    my $CDR = shift(@_);

    if (($CDR eq "na45pc08")
	||($CDR eq "na45pc10")) {
	
	@disks = (
		  "/shift/$CDR/data01",
		  "/shift/$CDR/data02",
		  "/shift/$CDR/data03",
		  "/shift/$CDR/data04",
		  "/shift/$CDR/data05"
		  );
    } elsif ($CDR eq "na45pc11") {
	@disks = (
		  "/shift/$CDR/data01",
		  "/shift/$CDR/data02",
		  "/shift/$CDR/data03",
		  "/shift/$CDR/data04",
		  "/shift/$CDR/data05",
		  "/shift/$CDR/data06"
		  );
	
    } elsif (($CDR eq "na45pc12") || ($CDR eq "na45pc14")) {
	@disks = (
		  "/shift/$CDR/data01",
		  "/shift/$CDR/data02",
		  "/shift/$CDR/data03",
		  "/shift/$CDR/data04",
		  "/shift/$CDR/data05",
		  "/shift/$CDR/data06"
		  );

    }
    else {
	@disks = (
		  "/shift/$CDR/data01",
		  "/shift/$CDR/data02",
		  "/shift/$CDR/data03",
		  "/shift/$CDR/data04"
		  );
    }

    return;
}


sub midEvb {
   $evb_hist->delete("1.0", "end");

   $histFile = "$evbRoot/hist/evb.hist";
   $tmpFile = "/tmp/evb.hist";
   unlink $tmpFile;
   `rfcp $rshpc:$histFile $tmpFile 2> /dev/null`;
   
   if (!open(FH, "tail -200 $tmpFile 2>&1 |")) {
       $evb_hist->insert("end", "ERROR: Could not open $histFile\n", "red");
   } else {
       while ($line=<FH>) {
#	   print "line = $line";
#	   if ($line eq "") { last;}
	   if ($line =~ /succeed/) {

	   if ($line =~/(\d+)\s+(\d+)\s+(\d+) evb/) {
	       if (($1>0)||($2>0)||($3>0)) {
		   $evb_hist->insert("end", $line, "magenta");
	       } else {
		   $evb_hist->insert("end", $line, "blue");
	       }
	   } else {
	       $evb_hist->insert("end", $line, "blue");
	   }

	   } elsif ($line =~ /fail/) {
	       $evb_hist->insert("end", $line, "red");
	   } else {
	       $evb_hist->insert("end", $line);
	   }

       }
       close(FH);
   }

}

sub midTape {

#    $statTape = `rsh $rshpc -n -l cerescdr tail -1 $statTapeFile 2>&1 &`;
    $tmpFile = "/tmp/taper.sta";
    unlink $tmpFile;
    `rfcp $rshpc:$statTapeFile $tmpFile 2> /dev/null`;
    $statTape = `tail -1 $tmpFile 2>&1`;

    $tape_text->delete("1.0", "end");

    $logFile = "$logDir/taper.log";
    $tmpFile = "/tmp/taper.log";
    unlink $tmpFile;
    `rfcp $rshpc:$logFile $tmpFile 2> /dev/null`;

    if (!open(FH, "tail -40 $tmpFile 2>&1|")) {
	$tape_text->insert("end", "ERROR: Could not open $logFile\n", "red");
    } else {
	while (<FH>) {$tape_text->insert("end", $_);}
	close(FH);
    }

    $tape_sum_list->delete("1.0", "end");

    $tapeList = "$listDir/tapeSummaryList.dat";
    $tmpFile  = "/tmp/tapeSummaryList.dat";
    unlink $tmpFile;
    `rfcp $rshpc:$tapeList $tmpFile 2> /dev/null`;

    if (!open(FH, "< $tmpFile")) {
	$tape_sum_list->insert("end", "ERROR: Could not open $tapeList\n", "red");
    } else {
	while (<FH>) {$tape_sum_list->insert("end", $_);}
	close(FH);
    }

    $tape_hist->delete("1.0", "end");

    $tapeHist = "$statDir/tapeDaemon.hist";
    $tmpFile = "/tmp/tapeDaemon.hist";
    unlink $tmpFile;
    `rfcp $rshpc:$tapeHist $tmpFile 2> /dev/null`;

    if (!open(FH, "tail -40 $tmpFile 2>&1 |")) {
	$tape_sum_list->insert("end", "ERROR: Could not open $tapeHist\n", "red");
    } else {
	while (<FH>) {$tape_hist->insert("end", $_);}
	close(FH);
    }

#check taper process

    @nTapeProcess = ();
    for ($index=0;$index<($#tppca+1);$index++) {

	if ($tppcAlive[$index] == 1) {
	    @processList = ();
	    @processList = `rsh $tppca[$index] -n -l cerescdr ps -ax --cols=200|grep taper.pl |grep -v grep`; 

	    $nTapeProcess[$index] = 0;
	    foreach $process (@processList) {
		if (($process =~ /perl/) && ($process !~ /grep/) && ($process !~ /emacs/)) {
		    $nTapeProcess[$index] ++;
		}
	    }
	} else {
	    $nTapeProcess[$index] = 0;
	}
    }

    if (($nTapeProcess[0] == 1) && ($nTapeProcess[1] == 0)) {
	$tppc = $tppca[0];
	$tapeProcess = 1;
	$l_pro_taper->configure(-foreground => 'blue');
	$taperError = 1;
    } 
    elsif (($nTapeProcess[0] == 0) && ($nTapeProcess[1] == 1)) {
	$tppc = $tppca[1];
	$tapeProcess = 1;
	$l_pro_taper->configure(-foreground => 'blue');
	$taperError = 1;
    } 
    elsif (($nTapeProcess[0] == 0) && ($nTapeProcess[1] == 0)) {
	$tppc = "";
	$tapeProcess = "0, No taper running!, submit taper!";
	$taperError = -1; # no process running!
	$l_pro_taper->configure(-foreground => 'red');
    } else {
	$tppc = $tppca[0];
	$tapeProcess = "$nTapeProcess[0] on $tppca[0],  $nTapeProcess[1] on $tppca[1], too many taper processes";
	$l_pro_taper->configure(-foreground => 'red');
	print "processList = @processList\n";
	$taperError = -1;
    }


    $nWaiting      = `rsh $rshpc -n -l cerescdr ls $queueDir/*.waiting 2> /dev/null | wc -l &`;
    $nWaiting      = int($nWaiting);

    if ($nWaiting > 30) {
	$l_nwaiting->configure(-foreground => 'red');
    } else {
	$l_nwaiting->configure(-foreground => 'black');
    }
    $nSubmitted    = `rsh $rshpc -n -l cerescdr ls $queueDir/*.submitted 2> /dev/null | wc -l &`;
    $nSubmitted    = int($nSubmitted);
#    $nFinished     = `rsh $rshpc -n -l cerescdr ls $queueDir/*.finished 2> /dev/null | wc -l &`;
#    $nFinished     = int($nFinished);
    $nProblem      = `rsh $rshpc -n -l cerescdr ls $queueDir/*.problem 2> /dev/null | wc -l &`;
    $nProblem      = int($nProblem);
    $nProblemCheck = `rsh $rshpc -n -l cerescdr ls $queueDir/*.problemCheck 2> /dev/null | wc -l &`;
    $nProblemCheck = int($nProblemCheck);



    if ($taperError >0) {
	$b_sub_taper->configure(-state => 'disabled');
	$b_stp_taper->configure(-state => 'disabled');
	$gStatTaper = "NORMAL";
	$l_gsta_taper->configure(-foreground => 'blue');
    } else {
	$b_sub_taper->configure(-state => 'normal');
	$b_stp_taper->configure(-state => 'disabled');
	$gStatTaper = "ERROR";
	$l_gsta_taper->configure(-foreground => 'red');
    }

    return;
}

sub fastEvb {
# scan ebpc for all processes
    
    for ($index=0;$index<($#ebpc+1);$index++) {
	$ebpcProc[$index] = `rsh $ebpc[$index] -n -l cerescdr ps -ax --cols=200 | grep doEvb.pl | grep -v grep`;
    }


    $nEvbError = 0;
    for ($index=0;$index<($#ebpc+1);$index++) {
	if ($ebpcAlive[$index] == 1) {
	    $staFile = "$evbRoot/sta/$ebpc[$index]";
	    $tmpFile = "/tmp/$ebpc[$index]";
	    unlink $tmpFile;
	    `rfcp $rshpc:$staFile $tmpFile 2> /dev/null`;
	    $statEvb[$index] = `cat $tmpFile 2>&1`;
	    chop($statEvb[$index]);
	    if ($statEvb[$index] =~ "succeed") {
		if ($statEvb[$index] =~/(\d+)\s+(\d+)\s+(\d+) evb/) {
#		    print "errors: $1 $2 $3\n";
		    if (($1>0)||($2>0)||($3>0)) {
			$evb_stat[$index]->configure(foreground => 'magenta');
		    } else {
			$evb_stat[$index]->configure(foreground => 'blue');
		    }
		} else {
		    $evb_stat[$index]->configure(foreground => 'blue');
		}


	    } elsif ($statEvb[$index] =~ /fail/)  {
		$evb_stat[$index]->configure(foreground => 'red');
		$nEvbError++;
	    } else {
		$evb_stat[$index]->configure(foreground => 'black');
	    }
	} else {
	    $statEvb[$index] = "NA";
	}
    }
    if ($nEvbError > 3) {
	$evbError = -1;
    }

#check evb process 

    $evbError = 1;

    for ($index=0;$index<($#ebpc+1);$index++) {
	if ($ebpcAlive[$index] == 1) {
	    @processList = split('\n', $ebpcProc[$index]);

	    $nEvb[$index] = 0;
	    $nReceiver[$index] = 0;
	    foreach $process (@processList) {
		if (($process =~ /perl/) && ($process =~ /doEvb\.pl/)
		    &&($process !~ /grep/)) {
		    $nEvb[$index] ++;
		}
		if ($process =~ /daqmem\_receiver/) {
		    $nReceiver[$index] ++;
		}
	    }

	    if ($nEvb[$index] < 1) {
		$nEvb[$index] = "$nEvb[$index]";
		$l_pro_evb[$index]->configure(-foreground => 'blue');
	    } elsif ($nEvb[$index] > 1) {
		$nEvb[$index] = "$nEvb[$index], too many evb running!";
		print "processList = @processList\n";
		$evbError = -1;
		$l_pro_evb[$index]->configure(-foreground => 'red');
	    } else {
		$l_pro_evb[$index]->configure(-foreground => 'blue');
	    }

	    if ($nReceiver[$index] <= 15) {
		$nReceiver[$index] = "$nReceiver[$index]";
		$l_pro_receiver[$index]->configure(-foreground => 'blue');
	    } else {
		$nReceiver[$index] = "$nReceiver[$index] too many daqmem_receiver running!";
		$l_pro_receiver[$index]->configure(-foreground => 'red');
		$evbError = -1;
	    }
	} else {
	    $nEvb[$index] = 0;
	    $nReceiver[$index] = 0;
	}
    }

# update status of button

    if ($evbError >0) {
	$gStatEvb = "NORMAL";
	$l_gsta_evb->configure(-foreground => 'blue');
    } else {
	$gStatEvb = "ERROR";
	$l_gsta_evb->configure(-foreground => 'red');
    }

    return;
}

sub slowClean {

    $index=0;
    foreach $eb (@ebpc) {
	$path = "$statDir/cleaner-$eb.sta";
	$tmpFile = "/tmp/cleaner-$eb.sta";
	unlink $tmpFile;
	`rfcp $rshpc:$path $tmpFile 2> /dev/null`;
	
	$statClean[$index] = `tail -1 $tmpFile  2>&1`;
	chop($statClean[$index]);
	$index++;
    }

#check cleaner process 

    $cleanerError = 1;

    for ($index=0;$index<($#ebpc+1);$index++) {
	if ($ebpcAlive[$index] == 1) {
	@processList = `rsh $ebpc[$index] -n -l cerescdr ps -ax --cols=200|grep cleaner.pl|grep -v grep`; 
	    
	    $nCleaner[$index] = 0;
	    foreach $process (@processList) {
		if (($process =~ /perl/) && ($process =~ /cleaner\.pl/) && ($process !~ /grep/)) {
		    $nCleaner[$index] ++;
		}
	    }
	} else {
	    $nCleaner[$index]=0;
	}

	if ($nCleaner[$index] < 1) {

	    $nCleaner[$index] = "$nCleaner[$index], no cleaner running. submit cleaner!";

#	    $t_err_cleaner->insert("end","No. of cleaner processes = $nCleaner[$index] on $ebpc[$index]\n", "red");

	    $cleanerError = -1;
	    $l_pro_cleaner[$index]->configure(-foreground => 'red');
	} elsif ($nCleaner[$index] > 1) {
	    $nCleaner[$index] = "$nCleaner[$index], too many cleaner running.";
	    print "processList = @processList\n";
	    $cleanerError = -1;
	    $l_pro_cleaner[$index]->configure(-foreground => 'red');
	} else {
	    $l_pro_cleaner[$index]->configure(-foreground => 'blue');
	}
    }

# update status of button


    if ($cleanerError >0) {
	$b_sub_cleaner->configure(-state => 'disabled');
	$b_kil_cleaner->configure(-state => 'disabled');
	$gStatCleaner = "NORMAL";
	$l_gsta_cleaner->configure(-foreground => 'blue');
    } else {
	$b_sub_cleaner->configure(-state => 'normal');
	$b_kil_cleaner->configure(-state => 'disabled');
	$gStatCleaner = "ERROR";
	$l_gsta_cleaner->configure(-foreground => 'red');
    }

#    $nToTape = `rsh $rshpc -n -l cerescdr ls $toTapeDir |wc -w`; 
#    $nToTape = int($nToTape);
    $nChecked = `rsh $rshpc -n -l cerescdr ls $toTapeDir $CheckedDir |wc -w`; 
    $nChecked = int($nChecked);
    $nTaping = `rsh $rshpc -n -l cerescdr ls $TapingDir |wc -w`; 
    $nTaping = int($nTaping);
    for ($index=0;$index<($#ebpc+1);$index++) {
	$nToClean[$index] = `rsh $rshpc -n -l cerescdr ls $toCleanDir/$ebpc[$index]-* |wc -w`; 
	$nToClean[$index] = int($nToClean[$index]);
    }

    return;
}


sub slowCDR {

    $cdrError = 1;

    $nfspcAlive = 1;
    @ping = `ping -c 1 $nfspc`;
    foreach $line (@ping) {
	if ($line =~/(\d+) packets received/) {
	    $nPacket = $1;
	    if ($nPacket == 0) {
		$nfspcAlive = 0;
	    }
	}
    }
    if ($nfspcAlive == 1) {

	$output = `rsh $nfspc -n -l cerescdr uptime`;
	    
	if ($output eq "") {
	    $nfspcStat = "Serious Problem. Cannot rsh uptime. Check the machine status";
	    $nfspcStatLabel->configure(-foreground => 'red');
	    $cdrError = -1;
	} else {
	    chop($output);
	    $nfspcStat = $output;
	    $nfspcStatLabel->configure(-foreground => 'blue');
	}
    } else {
	$nfspcStat = "Serious Problem. Cannot ping. Check the machine status";
	$nfspcStatLabel->configure(-foreground => 'red');
	$cdrError = -1;
    }


#CDR is alive?

    $rshpc = "";
    for ($index=0;$index<($#ebpc+1);$index++) {

	$ebpcAlive[$index] = 1;
	@ping = `ping -c 1 $ebpc[$index]`;
#10 packets transmitted, 0 packets received, +9 errors, 100% packet loss
	foreach $line (@ping) {
	    if ($line =~/(\d+) packets received/) {
		$nPacket = $1;
		if ($nPacket == 0) {
		    $ebpcAlive[$index] = 0;
		}
	    }
	}

	if ($ebpcAlive[$index] == 1) {
	    if ($rshpc eq "") {
		$rshpc = $ebpc[$index];
	    }
	    $output = `rsh $ebpc[$index] -n -l cerescdr uptime`;
	    
	    if ($output eq "") {
		$ebpcStat[$index] = "Serious Problem. Cannot rsh uptime. Check the machine status";
		$ebpcStatLabel[$index]->configure(-foreground => 'red');
		$cdrError = -1;
	    } else {
		chop($output);
		$ebpcStat[$index] = $output;
		$ebpcStatLabel[$index]->configure(-foreground => 'blue');
	    }
	} else {
		$ebpcStat[$index] = "Serious Problem. Cannot ping. Check the machine status";
		$ebpcStatLabel[$index]->configure(-foreground => 'red');
		$cdrError = -1;
	}
    }

#tape pc is alive? use ping
    for ($index=0;$index<($#tppca+1);$index++) {

	@ping = `ping -c 1 $tppca[$index]`;
#10 packets transmitted, 0 packets received, +9 errors, 100% packet loss
	foreach $line (@ping) {
	    if ($line =~/(\d+) packets received/) {
		$nPacket = $1;
		if ($nPacket == 0) {
		    $tppcAlive[$index] = 0;
		} else {
		    $tppcAlive[$index] = 1;
		}
	    }
	}

	if ($tppcAlive[$index] == 1) {
	    $output = `rsh $tppca[$index] -n -l cerescdr uptime`;
	    
	    if ($output eq "") {
		$tppcStat[$index] = "Serious Problem. Cannot rsh uptime. Check the machine status";
		$tppcStatLabel[$index]->configure(-foreground => 'red');
		$cdrError = -1;
	    } else {
		chop($output);
		$tppcStat[$index] = $output;
		$tppcStatLabel[$index]->configure(-foreground => 'blue');
	    }
	} else {
	    $tppcStat[$index] = "Serious Problem. Cannot ping. Check the machine status";
	    $tppcStatLabel[$index]->configure(-foreground => 'red');
	    $cdrError = -1;
	}
    }

#check disk spaces

    for ($index=0;$index<($#ebpc+1);$index++) {
#	print "ebpcAlive[$index] = $ebpcAlive[$index]\n";
	if ($ebpcAlive[$index] == 1) {
	    $script = "$monRoot/scripts/freeDiskSpace.pl $ebpc[$index]";
	    @output = `$script`;
	    foreach $line (@output) {
		chop($line);
		if ($line =~ /(na45pc\d\d)/) {
#lin = /shift/na45pc14/data01 32732084 / 34451248
		    if ($line =~ /\/shift\/na45pc(\d\d)\/data(\d\d)\s+(\d+) \/\s+(\d+)/) {
			$idisk = $2-1;
			$free  = $3/1024./1024.;
			$space = $4/1024./1024.;

			$freeSpace[$index][$idisk] =$free/$space;
			$diskSpace[$index][$idisk] = sprintf("%4.1f/%4.1f ",
							     $free, $space);
			if ($freeSpace[$index][$idisk] < $minFreePerDisk) {
			    $diskLabel[$index][$idisk]->configure(-foreground => "magenta");
			} else {
			    $diskLabel[$index][$idisk]->configure(-foreground => "black");
			}
			if ($debug) {
			    print "diskSpace[$index][$idisk] = $diskSpace[$index][$idisk]\n";
			}
		    }
#		    na45pc12 : free 299237944 / 361479888 (82.8 percent)
		    if ($line =~ /(na45pc\d\d) \: free\s+(\d+) \/\s+(\d+)/) {
			$free = $2/1024./1024.;
			$space = $3/1024./1024.;
			if ($space > 0) {
			    $totalFreeSpace[$index] = $free/$space;
			} else {
			    $totalFreeSpace[$index] = 0.;
			}
			if ($totalFreeSpace[$index] < $minFree) {
			    $cdrError = -1;
			$totalDiskSpace[$index] = sprintf("ERROR!! Total %5.1f / %5.1f [GB] = %4.2f, check cleaner and taper processes!", 
							  $free, $space, $totalFreeSpace[$index]);
			    $totalDiskLabel[$index]->configure(-foreground => "red");
			} else {
			    $string = "%";
			    $totalDiskSpace[$index] = sprintf("Total %5.1f / %5.1f [GB] = %4.2f %s", 
							  $free, $space, $totalFreeSpace[$index]*100.,$string);
			    $totalDiskLabel[$index]->configure(-foreground => "blue");
			}

			if ($debug) {
			    print "totalDiskSpace[$index] = $totalDiskSpace[$index]\n";
			}
		    }

		}
	    }

	}
    }

    $output = `rsh -n -l cerescdr $rshpc "df /cdrShare01 |tail -1"`;

    @df = split(' ', $output);

    if ($debug) {
	print "free  = $df[2]\n";
	print "total = $df[0]\n";
    }
    $shareFree = $df[2]/$df[0];
    
    if ($debug) {
	print "free/total = $shareFree\n";
    }
    
    $string = "%";
    $shareDiskSpace = sprintf("%4.1f / %4.1f [GB] = %4.2f %s",
			      $df[2]/1024./1024., $df[0]/1024./1024., $shareFree*100., $string);

    if ($shareFree < $minFreeShare) {
	$shareDiskLabel->configure(-foreground => "red");
	$cdrError = -1;
    } else {
	$shareDiskLabel->configure(-foreground => "blue");
    }


    if ($cdrError>0) {
	$gStatCDR = "NORMAL";
	$l_gsta_mach->configure(-foreground => 'blue');
    } else {
	$gStatCDR = "ERROR";
	$l_gsta_mach->configure(-foreground => 'red');
    }

    return;
}