#!/usr/bin/perl
use Getopt::Long;
use Getopt::Long;
use strict;

#important files and fields to capture
#######/proc/pid/stat#################
# field 10 number of minor faults
# field 12 number of major faults
# filed 22 virtual memory size    (better to take this field  form  status file)
# field 23 residen set memory size ( better to take  this field  form status file)
######/proc/pid/status################
#VmPeak   peak virtual memory size
#VmSize   total program size
#VmRSS    size of memory portions
#VmSwap   size of swap usage (the number of referred swapents)
#####/proc/pid/statm ##########
#Field    Content
#1-size     total program size (pages)
#2-resident size of memory portions (pages)	(same as VmRSS in status)
#3-shared   number of pages that are shared	(i.e. backed by a file)
#####/proc/pid/io #######
#rchar  - bytes read
#wchar  - byres written
#syscr  - number of read syscalls
#syscw  - number of write syscalls
#read_bytes - number of bytes caused by this process to read from underlying storage
#write_bytes - number of bytes caused by this process to written from underlying storage

#mesauring the %CPU Utilization
#ps -p PID -o "pcpu" | sed  '/%CPU/d' gives us the CPU %  per pid

#####NETWORK################
#####file  /proc/PID/net/dev
##Format################
#Interface, Receive : bytes,  packets,
#           Trasmit : bytes,  packets,

############NFS FILE SYSTEM #############
#cat /proc/$pid/mountstats | grep WRITE | awk '{print $5}'  bytes writes to a NFS system
#cat /proc/$pid/mountstats | sed -n '/READ$/p' | awk '{print $5}'
#######################################################################################################

###File Format  ##############
#timestamp
#OAR_JOB_ID
#PID
#command line
#number of minor faults
#number of major faults
#VmPeak   peak virtual memory size
#VmSize   total program size
#VmRSS    size of memory portions
#VmSwap   size of swap usage (the number of referred swapents)
#size     total program size (pages)
#resident size of memory portions (pages)	(same as VmRSS in status)
#shared   number of pages that are shared	(i.e. backed by a file)
#rchar  - bytes read
#wchar  - byres written
#syscr  - number of read syscalls
#syscw  - number of write syscalls
#read_bytes - number of bytes caused by this process to read from underlying storage
#write_bytes - number of bytes caused by this process to written from           underlying storage
#%CPU Utilization
#Receive : bytes
#Trasmit : bytes

###################################################################################################
###### Parameters ###############################
#Trace directory
#CPUSET directory

my @task;
my $dir       = "";
my $conf_file = "clustermon.conf";           ##### default config file
my $log_file  = "/var/log/clustermon.log";
my $cmdline;
my %params = ();
my $pid;
my $numtask;
my $jobid;
my $hostname;
my $vmswap;
my $vmsize;
my $vmpeak;
my $vmrss;
my $syscr;
my $charread;
my $charwrite;
my $interface;
my $cpu;
my $syscw;
my @dirs;
my $timestamp;
my $read_bytes;
my $write_bytes;
my $isolated;
my $recvtotal = 0;
my $sendtotal = 0;

sub usage() {
    print <<EOS;
Usage: $0 -f confile
Options are:
 -f,  configuration file 
EOS
}

use sigtrap 'handler' => \&cleanAndExit, 'INT', 'ABRT', 'QUIT', 'TERM';

sub cleanAndExit() {

    print "Cleaning up and  exiting\n";
    exit(1);
}

sub readconf {
    open CONF, $conf_file or die "Cannot open configuration file \n";
    %params = ();
    while (<CONF>) {
        chomp;
        my ($key, $val) = split(/:/, $_);
        $val =~ s/^\s+//;    ##getting rid of possble blank spaces at the begining of the variable
        $params{$key} = $val;
    }
    close(CONF);
}

sub writelog($) {
    my $message = shift();
    my $timelog = time();
    open(LOG, ">>$log_file");
    print LOG "$timelog   $message \n";
    close(LOG);

}

sub readtask($) {
    my @taskread = ();
    my $dirtask  = shift();
    open TASKS, "$params{CPUSETDIR}$dirtask/tasks " or die "Cannot open file \n";

    #print "/dev/cpuset/oar/$dir/tasks \n";

    while ($numtask = <TASKS>) {
        chomp($numtask);

        #Getting ride of the OAR tasks
        #Testing

        my $jobid = (split(/_/, $dirtask))[1];
        my $oartask2;
        my $oartask3;
        if (-e "/var/lib/oar/pid_of_oarexec_for_jobId_$jobid") {
            $oartask2 = `cat /var/lib/oar/pid_of_oarexec_for_jobId_$jobid`;
        }
        if (-e "/var/lib/oar/oarsub_connections_$jobid") {
            chomp($oartask3 = `cat /var/lib/oar/oarsub_connections_$jobid`);
        }
        my $processname = `ps -p $numtask |  sed -n 2p | awk '{print \$4}'`;
        chomp(my $oartask1 = `ps ax | grep $numtask | sed '2 d' | awk '{print \$6}'`);
        if ($oartask1 !~ /.*oar\@notty.+/ &&
            $oartask2 !~ $numtask &&
            $oartask3 !~ $numtask &&
            $processname ne "") {
            push(@taskread, $numtask);
        }

    }
    close(TASKS);
    return (@taskread);
}

Getopt::Long::Configure("gnu_getopt");
GetOptions("f=s" => \$conf_file);

#Start
#Reading configuration file

readconf;

#getting hostname
chomp($hostname = `hostname `);

# attempt to create TRACEDIR if it doesn't exist
if (!-d $params{TRACEDIR}) { mkdir $params{TRACEDIR} or die "Unable to create $params{TRACEDIR}"; }
system("chown", "oar", "$params{TRACEDIR}");

for (;;) {

    # Getting directories
    my $temp;    #### to keep the directories ####
    if (-d $params{CPUSETDIR}) { $temp = `ls $params{CPUSETDIR} | grep -E '[0-9]\$'`; }
    else                       { writelog("Directory CPUSET does not exist "); }
    @dirs = ();
    foreach $dir (split(/\n/, $temp)) { push(@dirs, $dir); }

    #print " dirs: @dirs \n";
    if (@dirs) {
        $isolated = 0;

        #verifying if there are more than one job in the machine
        if (@dirs > 1) { $isolated = 1; }

        #reading PID
        foreach $dir (@dirs) {

            #print "#############################################\n";
            @task = readtask($dir);

            #print "#############################################\n";
            foreach $pid (@task) {

                if (-d "/proc/$pid") {

                    #Getting time
                    $timestamp = time();

                    #getting the OAR JOB ID
                    my $oarjobid = (split(/_/, $dir))[1];

                    #Open the trace file
                    open(TRACE, ">>$params{TRACEDIR}trace-$hostname-$oarjobid.log");

                    #Setting permission for user oar
                    system("chown", "oar", "$params{TRACEDIR}trace-$hostname-$oarjobid.log");

                    ## getting the cmdline
                    $cmdline = `ps -p $pid | sed -n 2p | awk '{print \$4}'`;
                    $cmdline =~ s/\n//;
                    $cmdline = substr($cmdline, 0, 15); ###Adjusting it to a length of 15 characters

                    ##reading /proc/pid/stat file ####
                    my $stat        = `cat /proc/$pid/stat `;
                    my $minorfaults = (split(/ /, $stat))[9];
                    my $majorfaults = (split(/ /, $stat))[11];

                    ## reading /proc/pid/status file ###
                    chomp($vmpeak = `cat /proc/$pid/status | sed -n '/VmPeak/p'`);
                    $vmpeak = (split(/  */, $vmpeak))[1];
                    chomp($vmsize = `cat /proc/$pid/status | sed -n '/VmSize/p'`);
                    $vmsize = (split(/  */, $vmsize))[1];
                    chomp($vmrss = `cat /proc/$pid/status | sed -n '/VmRSS/p'`);
                    $vmrss = (split(/  */, $vmrss))[1];
                    chomp($vmswap = `cat /proc/$pid/status | sed -n '/VmSwap/p'`);
                    my $vmswap = (split(/  */, $vmswap))[1];
                    if ($vmswap eq "") { $vmswap = 0; }

                    ###reading /proc/pid/statm file ##
                    my $statm     = `cat /proc/$pid/statm `;
                    my $msize     = (split(/ /, $statm))[0];
                    my $mresident = (split(/ /, $statm))[1];
                    my $mshared   = (split(/ /, $statm))[2];

                    ###reading /proc/pid/io file #######
                    if (-e "/proc/$pid/io") {
                        my $iostatus = `cat /proc/$pid/io `;
                        chomp($charread = (split(/\n/, $iostatus))[0]);
                        $charread = (split(/ /, $charread))[1];
                        chomp($charwrite = (split(/\n/, $iostatus))[1]);
                        $charwrite = (split(/ /, $charwrite))[1];
                        chomp($syscr = (split(/\n/, $iostatus))[2]);
                        $syscr = (split(/ /, $syscr))[1];
                        chomp($syscw = (split(/\n/, $iostatus))[3]);
                        $syscw = (split(/ /, $syscw))[1];
                        chomp($read_bytes = (split(/\n/, $iostatus))[4]);
                        $read_bytes = (split(/ /, $read_bytes))[1];
                        chomp($write_bytes = (split(/\n/, $iostatus))[5]);
                        $write_bytes = (split(/ /, $write_bytes))[1];
                    } else {
                        $charread    = 0;
                        $charwrite   = 0;
                        $syscr       = 0;
                        $syscw       = 0;
                        $read_bytes  = 0;
                        $write_bytes = 0;
                    }

                    ### getting the CPU % used by the process ####
                    chomp($cpu = `ps -p $pid -o "pcpu" | sed  '/CPU/d'`);

                    ##### Network information /proc/PID/net/dev ####
                    if (-e "/proc/$pid/net/dev") {
                        my $network = `cat /proc/$pid/net/dev`;    #####testing code
                        my $recv;
                        my $send;
                        foreach $interface (split(/\n/, $network)) {
                            if ($interface =~
                                /^\s+(\w+):\s*(\d+)\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+\d+\s+(\d+)/
                            ) {
                                $recv = $2;
                                $send = $3;
                            }
                            $recvtotal = sprintf("%d", $recv) + $recvtotal;
                            $sendtotal = sprintf("%d", $send) + $sendtotal;
                        }
                    } else {
                        $recvtotal = 0;
                        $sendtotal = 0;
                    }

                    #printing information into file
                    print TRACE
                      "$timestamp $oarjobid $pid $cmdline $minorfaults $majorfaults $vmpeak $vmsize $vmrss $vmswap $msize $mresident $mshared $charread $charwrite $syscr $syscw $read_bytes $write_bytes $cpu $recvtotal $sendtotal \n";
                    if ($isolated == 1) {
                        print TRACE "Warning: There are more than one Job in the machine \n";
                    }
                }
            }
        }
    }
    close(TRACE);
    sleep(60);
}

