#!/usr/bin/env python

from random import randint
from time   import sleep
import subprocess
import pty
import socket
import os
import sys
import errno
import signal
import resource
import pwd
import stat
import re

#get testconfig
# This assumes Makefile.in in main dir, but only Makefile in test dir.
try:
  sys.path += [os.getenv("PWD") + '/test', os.getenv('PWD')]
  from autotest_config import *

except ImportError:
  print "\n*** Error importing autotest_config.py: "
  sys.exit()

if USE_TEST_SUITE == "no":
  print "\n*** DMTCP test suite is disabled. To re-enable the test suite,\n" + \
        "***  re-configure _without_ './configure --disable-test-suite'\n"
  sys.exit()

# Disable ptrace tests for now.
PTRACE_SUPPORT="no"

signal.alarm(1800)  # half hour

if sys.version_info[0] != 2 or sys.version_info[0:2] < (2,4):
  print "test/autotest.py works only with Python 2.x for 2.x greater than 2.3"
  print "Change the beginning of test/autotest.py if you believe you can run."
  sys.exit(1)

if sys.version_info[0] == 2 and sys.version_info[1] >= 7:
  uname_m = subprocess.check_output(['uname', '-m'])
  uname_p = subprocess.check_output(['uname', '-p'])
else:
  uname_m = subprocess.Popen(['uname', '-m'],
                             stdout=subprocess.PIPE).communicate()[0]
  uname_p = subprocess.Popen(['uname', '-p'],
                             stdout=subprocess.PIPE).communicate()[0]

# This assumes Makefile.in in main dir, but only Makefile in test dir.
os.system("test -f Makefile || ./configure")
if USE_TEST_SUITE == "no":
  print "\n*** DMTCP test suite is disabled. To re-enable the test suite,\n" + \
          "***  re-configure _without_ './configure --disable-test-suite'\n"
  sys.exit()

#number of checkpoint/restart cycles
CYCLES=2

#Number of times to try dmtcp_restart
RETRIES=2

#Sleep after each program startup (sec)
DEFAULT_S=0.3
if uname_p[0:3] == 'arm':
  DEFAULT_S *= 2
#Allow extra time for slower CPUs
if uname_m in ["i386", "i486", "i586", "i686", "armv7", "armv71"]:
  DEFAULT_S *= 4

S=DEFAULT_S
#Appears as S*SLOW in code.  If --slow, then SLOW=5
SLOW=1

#Max time to wait for ckpt/restart to finish (sec)
TIMEOUT=10
# Raise this value when /usr/lib/locale/locale-archive is 100MB.
# This can happen on Red Hat-derived distros.
if os.path.exists("/usr/lib/locale/locale-archive") and \
   os.path.getsize("/usr/lib/locale/locale-archive") > 10e6:
  TIMEOUT *= int( os.path.getsize("/usr/lib/locale/locale-archive") / 10e6 )

#Interval between checks for ckpt/restart complete
INTERVAL=0.1

#Buffers for process i/o
BUFFER_SIZE=4096*8

#False redirects process stderr
VERBOSE=False

#Should we retry on a failure?
RETRY_ONCE=False

#Run (most) tests with user default (usually with gzip enable)
GZIP=os.getenv('DMTCP_GZIP') or "1"

#Warn if can't create a file of size:
REQUIRE_MB=50

#Binaries
BIN="./bin/"

#Checkpoint command to send to coordinator
CKPT_CMD='c'

#parse program args
args={}
for i in sys.argv:
  args[i]=True
  if i=="-v" or i=="--verbose":
    VERBOSE=True
  if i=="--stress":
    CYCLES=100000
  if i=="--slow":
    SLOW=5
  if i=="--retry-once":
    RETRY_ONCE = True
  #TODO:  Install SIGSEGV handler with infinite loop, and add to LD_PRELOAD
  #In test/Makefile, build libcatchsigsegv.so
  #Add --catchsigsegv  to usage string.
  # if i=="--catchsigsegv":
  #   if os.getenv('LD_PRELOAD'):
  #     os.environ['LD_PRELOAD'] += ':libcatchsigsegv.so'
  #   else:
  #     os.environ['LD_PRELOAD'] = 'libcatchsigsegv.so'
  if i=="-h" or i=="--help":
    print ("USAGE "+sys.argv[0]+
      " [-v] [--stress] [--slow] [testname] [testname ...]")
    sys.exit(1)

stats = [0, 0]

def xor(bool1, bool2):
  return (bool1 or bool2) and (not bool1 or not bool2)

def replaceChar(string, index, char):
  return string[0:index] + char + string[index+1:len(string)]

def splitWithQuotes(string):
  inSingleQuotes = False
  inDoubleQuotes = False
  isOuter = False
  escapeChar = False
  for i in range(len(string)):
    if escapeChar:
      escapeChar = False
      continue
    if string[i] == "\\":
      escapeChar = True
      # Remove one level of escaping if same quoting char as isOuter
      string = replaceChar(string, i, '#')
      continue
    if string[i] == "'":
      inSingleQuotes = not inSingleQuotes
    if string[i] == '"':
      inDoubleQuotes = not inDoubleQuotes
    # Remove outermost quotes: 'bash -c "sleep 30"' => ['bash','-c','sleep 30']
    if string[i] == "'" or string[i] == '"':
      # This triggers twice in:  '"..."'  (on first ' and second ")
      if xor(inSingleQuotes, inDoubleQuotes) and not isOuter: # if beg. of quote
        isOuter = string[i]
        string = replaceChar(string, i, '#')
      elif isOuter == string[i]:  # if end of quote
        isOuter = False
        string = replaceChar(string, i, '#')
    if not inSingleQuotes and not inDoubleQuotes and string[i] == ' ':
      # FIXME (Is there any destructive way to do this?)
      string = replaceChar(string, i, '%')
  string = string.replace('#', '')
  return string.split('%')

def shouldRunTest(name):
  # FIXME:  This is a hack.  We should have created var, testNaems and use here
  if len(sys.argv) <= 1+(VERBOSE==True)+(SLOW!=1)+(CYCLES!=2)+(RETRY_ONCE==True):
    return True
  return name in sys.argv

#make sure we are in svn root
if os.system("test -d bin") != 0:
  os.chdir("..")
if USE_M32:
  assert os.system("test -d bin") == 0, \
  "  bin/dmtcp_launch not found.  Please configure and build the\n" + \
  "  default 64-bit mode before configuring with --enable-m32 and re-building"
else:
  assert os.system("test -d bin") == 0

#make sure dmtcp is built
if os.system("make -s --no-print-directory tests") != 0:
  print "`make all tests` FAILED"
  sys.exit(1)

#pad a string and print/flush it
def printFixed(str, w=1):
  # The comma at end of print prevents a "newline", but still adds space.
  if sys.version_info[0] == 2:
    # Replace "print str.ljust(w),"  by version compatible w/ python3:
    os.write(sys.stdout.fileno(), str.ljust(w).encode("ascii"))
  sys.stdout.flush()

#exception on failed check
class CheckFailed(Exception):
  def __init__(self, value=""):
    self.value = value

class MySubprocess:
  "dummy class: same fields as from subprocess module"
  def __init__(self, pid):
    self.pid = pid
    self.stdin = os.open(os.devnull, os.O_RDONLY)
    self.stdout = os.open(os.devnull, os.O_WRONLY)
    self.stderr = os.open(os.devnull, os.O_WRONLY)

def master_read(fd):
  os.read(fd, 4096)
  return ''

#launch a child process
# NOTE:  Can eventually migrate to Python 2.7:  subprocess.check_output
devnullFd = os.open(os.devnull, os.O_WRONLY)
def launch(cmd):
  global devnullFd
  global master_read
  if VERBOSE:
    print "Launching... ", cmd
  cmd = splitWithQuotes(cmd);
  # Example cmd:  dmtcp_launch screen ...
  ptyMode = False
  for str in cmd:
    # Checkpoint image can be emacs23_x, or whatever emacs is a link to.
    # vim can be vim.gnome, etc.
    if re.search("(_|/|^)(screen|script|vim.*|emacs.*|pty)(_|$)", str):
      ptyMode = True
  try:
    os.stat(cmd[0])
  except:
    raise CheckFailed(cmd[0] + " not found")
  if ptyMode:
    # FOR DEBUGGING:  This can mysteriously fail, causing pty.fork() to fail
    try:
      (fd1, fd2) = os.openpty()
    except OSError, e:
      print "\n\n/dev/ptmx:"; os.system("ls -l /dev/ptmx /dev/pts")
      raise e
    else:
      os.close(fd1); os.close(fd2)
    (pid, fd) = pty.fork()
    if pid == 0:
      # Close all fds except stdin/stdout/stderr
      os.closerange(3,1024)
      signal.alarm(300) # pending alarm inherited across exec, but not a fork
      # Problem:  pty.spawn invokes fork.  alarm() will have no effect.
      pty.spawn(cmd, master_read)
      sys.exit(0)
    else:
      return MySubprocess(pid)
  else:
    if cmd[0] == BIN+"dmtcp_coordinator":
      childStdout = subprocess.PIPE
      # Don't mix stderr in with childStdout; need to read stdout
      if VERBOSE:
        childStderr = None
      else:
        childStderr = devnullFd
    elif VERBOSE:
      childStdout=None  # Inherit child stdout from parent
      childStderr=None  # Inherit child stderr from parent
    else:
      childStdout = devnullFd
      childStderr = subprocess.STDOUT # Mix stderr into stdout file object
    # NOTE:  This might be replaced by shell=True in call to subprocess.Popen
    proc = subprocess.Popen(cmd, bufsize=BUFFER_SIZE,
                 stdin=subprocess.PIPE, stdout=childStdout,
                 stderr=childStderr, close_fds=True)
  return proc

#randomize port and dir, so multiple processes works
ckptDir="dmtcp-autotest-%d" % randint(100000000,999999999)
os.mkdir(ckptDir);
os.environ['DMTCP_HOST'] = "localhost"
os.environ['DMTCP_PORT'] = str(randint(2000,10000))
os.environ['DMTCP_CHECKPOINT_DIR'] = os.path.abspath(ckptDir)
#Use default SIGCKPT for test suite.
os.unsetenv('DMTCP_SIGCKPT')
os.unsetenv('MTCP_SIGCKPT')
#No gzip by default.  (Isolate gzip failures from other test failures.)
#But note that dmtcp3, frisbee and gzip tests below still use gzip.
if not VERBOSE:
  os.environ['JALIB_STDERR_PATH'] = os.devnull
if VERBOSE:
  print "coordinator port:  " + os.environ['DMTCP_PORT']

# We'll copy ckptdir to DMTCP_TMPDIR in case of error.
def dmtcp_tmpdir():
  tmpdir = os.getenv('DMTCP_TMPDIR') or os.getenv('TMPDIR') or '/tmp'
  return tmpdir + '/dmtcp-' + os.environ['USER'] + '@' + socket.gethostname()

def free_diskspace(dir):
  s = os.statvfs('.')
  return s.f_bavail * s.f_frsize

# We'll save core dumps in our default directory (usually dmtcp-autotest-*)
# We can use the lesser of half the free disk space of filesystem or 100 MB.
if free_diskspace(ckptDir) > 20*1024*1024:
  oldLimit = resource.getrlimit(resource.RLIMIT_CORE)
  newLimit = [min(free_diskspace(ckptDir)/2, 100*1024*1024), oldLimit[1]]
  if oldLimit[1] != resource.RLIM_INFINITY:  # Keep soft limit below hard limit
    newLimit[0] = min(newLimit[0], oldLimit[1])
  resource.setrlimit(resource.RLIMIT_CORE, newLimit)

# This can be slow.
print("Verifying there is enough disk space ...")
tmpfile=ckptDir + "/freeSpaceTest.tmp"
if os.system("dd if=/dev/zero of=" + tmpfile + " bs=1MB count=" +
             str(REQUIRE_MB) + " 2>/dev/null") != 0:
  GZIP="1"
  print '''

!!!WARNING!!!
Fewer than '''+str(REQUIRE_MB)+'''MB are available on the current volume.
Many of the tests below may fail due to insufficient space.
!!!WARNING!!!

'''
os.system("rm -f "+tmpfile)

os.environ['DMTCP_GZIP'] = GZIP
if os.getenv('LD_LIBRARY_PATH'):
    os.environ['LD_LIBRARY_PATH'] += ':' + os.getenv("PWD")+"/lib"
else:
    os.environ['LD_LIBRARY_PATH'] = os.getenv("PWD")+"/lib"

#launch the coordinator
coordinator = launch(BIN+"dmtcp_coordinator")

#send a command to the coordinator process
def coordinatorCmd(cmd):
  try:
    if VERBOSE and cmd != "s":
      print "COORDINATORCMD(",cmd,")"
    coordinator.stdin.write(cmd+"\n")
    coordinator.stdin.flush()
  except:
    raise CheckFailed("failed to write '%s' to coordinator (pid: %d)" % \
                      (cmd, coordinator.pid))

#clean up after ourselves
def SHUTDOWN():
  try:
    coordinatorCmd('q')
    sleep(S*SLOW)
  except:
    print "SHUTDOWN() failed"
  os.system("kill -9 %d" % coordinator.pid)
  os.system("rm -rf  %s" % ckptDir)
  os.close(devnullFd)

#make sure val is true
def CHECK(val, msg):
  if not val:
    raise CheckFailed(msg)

#wait TIMEOUT for test() to be true, or throw error
def WAITFOR(test, msg):
  left=TIMEOUT*(S/DEFAULT_S)/INTERVAL
  while not test():
    if left <= 0:
      CHECK(False, msg())
    left-=1
    sleep(INTERVAL)

#extract (NUM_PEERS, RUNNING) from coordinator
def getStatus():
  coordinatorCmd('s')

  returncode = coordinator.poll()
  if returncode:
    if returncode < 0:
      print "Coordinator terminated by signal ", str(-returncode)
    CHECK(False, "coordinator died unexpectedly")
    return (-1, False)

  while True:
    try:
      line=coordinator.stdout.readline()
      if not line:  # Immediate empty string on stdout means EOF
        CHECK(False, "coordinator died unexpectedly")
        return (-1, False)
      line=line.strip()
      if line=="Status...":
        break;
      if VERBOSE:
        print "Ignoring line from coordinator: ", line
        sleep(1)
    except IOError, (errno, strerror):
      if coordinator.poll():
        if coordinator.poll() < 0:
          print "Coordinator terminated by signal ", str(-returncode)
        CHECK(False, "coordinator died unexpectedly")
        return (-1, False)
      if errno==4: #Interrupted system call
        continue
      raise CheckFailed("I/O error(%s): %s" % (errno, strerror))

  x,peers=coordinator.stdout.readline().strip().split("=")
  CHECK(x=="NUM_PEERS", "reading coordinator status")
  x,running=coordinator.stdout.readline().strip().split("=")
  CHECK(x=="RUNNING", "reading coordinator status")

  if VERBOSE:
    print "STATUS: peers=%s, running=%s" % (peers,running)
  return (int(peers), (running=="yes"))

#delete all files in ckptDir
def clearCkptDir():
  for TRIES in range(2):  # Try twice in case ckpt_*_dmtcp.temp is renamed.
    #clear checkpoint dir
    for root, dirs, files in os.walk(ckptDir, topdown=False):
      for name in files:
        try:
          # if name.endswith(".dmtcp") :
          #   import shutil
          #   shutil.copy(os.path.join(root, name), "/home/kapil/dmtcp/ramfs")
          # else:
          #   os.remove(os.path.join(root, name))
          os.remove(os.path.join(root, name))
        except OSError, e:
          if e.errno != errno.ENOENT:  # Maybe ckpt_*_dmtcp.temp was renamed.
            raise e
      for name in dirs:
        os.rmdir(os.path.join(root, name))

def getNumCkptFiles(dir):
  return len(filter(lambda f: f.startswith("ckpt_") and f.endswith(".dmtcp"),\
                              os.listdir(dir)))


# Test a given list of commands to see if they checkpoint
# runTest() sets up a keyboard interrupt handler, and then calls this function.
def runTestRaw(name, numProcs, cmds):
  #the expected/correct running status
#  if USE_M32:
#    def forall(fnc, lst):
#      return reduce(lambda x, y: x and y, map(fnc, lst))
#    if not forall(lambda x: x.startswith("./test/"), cmds):
#      return
  status=(numProcs, True)
  procs=[]

  def doesStatusSatisfy(newStatus,requiredStatus):
    if isinstance(requiredStatus[0], int):
      statRange = [requiredStatus[0]]
    elif isinstance(requiredStatus[0], list):
      statRange = requiredStatus[0]
    else:
      raise NotImplementedError
    return newStatus[0] in statRange and newStatus[1] == requiredStatus[1]

  def wfMsg(msg):
    #return function to generate error message
    return lambda: msg+", "+str(status[0])+ \
                   " expected, %d found, running=%d" % getStatus()

  def testKill():
    #kill all processes
    coordinatorCmd('k')
    try:
      WAITFOR(lambda: getStatus()==(0, False),
              lambda:"coordinator kill command failed")
    except CheckFailed:
      global coordinator
      coordinatorCmd('q')
      os.system("kill -9 %d" % coordinator.pid)
      print "Trying to kill old coordinator, and launch new one on same port"
      coordinator = launch(BIN+"dmtcp_coordinator")
    for x in procs:
      #cleanup proc
      try:
        if isinstance(x.stdin,int):
          os.close(x.stdin)
        elif x.stdin:
          x.stdin.close()
        if isinstance(x.stdout,int):
          os.close(x.stdout)
        elif x.stdout:
          x.stdout.close()
        if isinstance(x.stderr,int):
          os.close(x.stderr)
        elif x.stderr:
          x.stderr.close()
      except:
        None
      try:
        os.waitpid(x.pid, os.WNOHANG)
      except OSError, e:
        if e.errno != errno.ECHILD:
          raise e
      procs.remove(x)

  def testCheckpoint():
    #start checkpoint
    coordinatorCmd(CKPT_CMD)

    #wait for files to appear and status to return to original
    WAITFOR(lambda: getNumCkptFiles(ckptDir)>0 and \
                    (CKPT_CMD == 'xc' or doesStatusSatisfy(getStatus(), status)),
            wfMsg("checkpoint error"))

    #make sure the right files are there
    numFiles=getNumCkptFiles(ckptDir) # len(os.listdir(ckptDir))
    CHECK(doesStatusSatisfy((numFiles,True),status),
          "unexpected number of checkpoint files, %s procs, %d files"
          % (str(status[0]), numFiles))

  def testRestart():
    #build restart command
    cmd=BIN+"dmtcp_restart --quiet"
    for i in os.listdir(ckptDir):
      if i.endswith(".dmtcp"):
        cmd+= " "+ckptDir+"/"+i
    #run restart and test if it worked
    procs.append(launch(cmd))
    WAITFOR(lambda: doesStatusSatisfy(getStatus(), status),
            wfMsg("restart error"))
    if HBICT_DELTACOMP == "no":
      clearCkptDir()

  try:
    printFixed(name,15)

    if not shouldRunTest(name):
      print "SKIPPED"
      return

    stats[1]+=1
    CHECK(getStatus()==(0, False), "coordinator initial state")

    #start user programs
    for cmd in cmds:
      procs.append(launch(BIN+"dmtcp_launch "+cmd))
      sleep(S*SLOW)

    WAITFOR(lambda: doesStatusSatisfy(getStatus(), status),
            wfMsg("user program startup error"))

    for i in range(CYCLES):
      if i!=0 and i%2==0:
        print #newline
        printFixed("",15)
      printFixed("ckpt:")
      # NOTE:  If this faile, it will throw an exception to CheckFailed
      #  of this function:  testRestart
      testCheckpoint()
      printFixed("PASSED ")
      testKill()

      printFixed("rstr:")
      for j in range(RETRIES):
        try:
          testRestart()
          printFixed("PASSED")
          break
        except CheckFailed, e:
          if j == RETRIES-1:
            # Save checkpoint images for later diagnosis.
            if os.path.isdir(dmtcp_tmpdir()) and os.path.isdir(ckptDir):
              if subprocess.call( ("cp -pr " + ckptDir + ' '
                                   + dmtcp_tmpdir()).split() ) == 0:
                print "\n***** Copied checkpoint images to " + dmtcp_tmpdir() \
                      + "/" + ckptDir
            raise e
          else:
            printFixed("FAILED ")
            (oldpid, oldstatus) = os.waitpid(procs[-1].pid, os.WNOHANG)
            if oldpid == procs[-1].pid:
              if os.WIFEXITED(oldstatus):
                printFixed("(first process exited: oldstatus "
                           + str(os.WEXITSTATUS(oldstatus)) + ")")
              if os.WIFSIGNALED(oldstatus):
                printFixed("(first process rec'd signal "
                           + str(os.WTERMSIG(oldstatus)) + ")")
              if os.WCOREDUMP(oldstatus):
                coredump = "core." + str(oldpid)
                if os.path.isdir(dmtcp_tmpdir()) and os.path.isfile(coredump):
                  if subprocess.call( ("cp -pr " + coredump + ' '
                                   + dmtcp_tmpdir()).split() ) == 0:
                    printFixed(" (" + coredump + " copied to DMTCP_TMPDIR:" +
                               dmtcp_tmpdir() + "/)")
            else:
              printFixed("(first process didn't die)")
            printFixed(" retry:")
            testKill()
      if i != CYCLES - 1:
        printFixed("; ")

    testKill()
    print #newline
    stats[0]+=1

  except CheckFailed, e:
    print "FAILED"
    printFixed("",15)
    print "root-pids:", map(lambda x: x.pid, procs), "msg:", e.value
    try:
      testKill()
    except CheckFailed, e:
      print "CLEANUP ERROR:", e.value
      SHUTDOWN()
      saveResultsNMI()
      sys.exit(1)
    if RETRY_ONCE:
      clearCkptDir()
      raise e

  clearCkptDir()

def getProcessChildren(pid):
    p = subprocess.Popen("ps --no-headers -o pid --ppid %d" % pid, shell = True,
                         stdout = subprocess.PIPE, stderr = subprocess.PIPE)
    stdout, stderr = p.communicate()
    return [int(pid) for pid in stdout.split()]

# If the user types ^C, then kill all child processes.
def runTest(name, numProcs, cmds):
  for i in range(2):
    try:
      runTestRaw(name, numProcs, cmds)
      break;
    except KeyboardInterrupt:
      for pid in getProcessChildren(os.getpid()):
        try:
          os.kill(pid, signal.SIGKILL)
        except OSError: # This happens if pid already died.
          pass
    except CheckFailed, e:
      if not RETRY_ONCE:
        break
      if i == 0:
        stats[1]-=1
        print "Trying once again"

def saveResultsNMI():
  if DEBUG == "yes":
    # WARNING:  This can cause a several second delay on some systems.
    host = socket.getfqdn()
    if re.search("^nmi-.*.cs.wisc.edu$", host) or \
       re.search("^nmi-.*.cs.wisconsin.edu$", host):
      tmpdir = os.getenv("TMPDIR", "/tmp") # if "TMPDIR" not set, return "/tmp"
      target = "./dmtcp-" + pwd.getpwuid(os.getuid()).pw_name + \
               "@" + socket.gethostname()
      cmd = "mkdir results; cp -pr " + tmpdir + "/" + target + \
               " ./dmtcp/src/libdmtcp.so" + \
               " ./dmtcp/src/dmtcp_coordinator" + \
               " ./mtcp/libmtcp.so" + \
               " results/"
      os.system(cmd)
      cmd = "tar zcf ../results.tar.gz ./results; rm -rf results"
      os.system(cmd)
      print "\n*** results.tar.gz ("+tmpdir+"/"+target+ \
                                              ") written to DMTCP_ROOT/.. ***"

print "== Tests =="

#tmp port
p0=str(randint(2000,10000))
p1=str(randint(2000,10000))
p2=str(randint(2000,10000))
p3=str(randint(2000,10000))

# Use uniform user shell.  Else apps like script have different subprocesses.
os.environ["SHELL"]="/bin/bash"

runTest("dmtcp1",        1, ["./test/dmtcp1"])

runTest("dmtcp2",        1, ["./test/dmtcp2"])

runTest("dmtcp3",        1, ["./test/dmtcp3"])

runTest("dmtcp4",        1, ["./test/dmtcp4"])

# In 32-bit Ubuntu 9.10, the default small stacksize (8 MB) forces
# legacy_va_layout, which places vdso in low memory.  This collides with text
# in low memory (0x110000) in the statically linked mtcp_restart executable.
oldLimit = resource.getrlimit(resource.RLIMIT_STACK)
# oldLimit[1] is old hard limit
if oldLimit[1] == resource.RLIM_INFINITY:
  newCurrLimit = 8L*1024*1024
else:
  newCurrLimit = min(8L*1024*1024, oldLimit[1])
resource.setrlimit(resource.RLIMIT_STACK, [newCurrLimit, oldLimit[1]])
runTest("dmtcp5",        2, ["./test/dmtcp5"])
resource.setrlimit(resource.RLIMIT_STACK, oldLimit)

# Test for a bunch of system calls. We want to use the 'xc' mode for
# checkpointing so that the process is killed right after checkpoint. Otherwise
# the syscall-tester could fail in the following case:
#   1. create and open temp file
#   2. close temp file
#   3. ckpt
#   4. unlink temp file
# If the last step is executed before the process is killed after ckpt-resume,
# the file would have been deleted from the disk. However, on restart, the test
# program will try to unlink the file once again, but the unlink operation will
# fail, causing the test to fail.
old_ckpt_cmd = CKPT_CMD
CKPT_CMD = 'xc'
runTest("syscall-tester",  1, ["./test/syscall-tester"])
CKPT_CMD = old_ckpt_cmd

# Test for files opened with WRONLY mode and later unlinked.
runTest("file1",         1, ["./test/file1"])

runTest("dmtcpaware1",   1, ["./test/dmtcpaware1"])

PWD=os.getcwd()
runTest("plugin-sleep2", 1, ["--with-plugin "+
                             PWD+"/test/plugin/sleep1/dmtcp_sleep1hijack.so:"+
                             PWD+"/test/plugin/sleep2/dmtcp_sleep2hijack.so "+
                             "./test/dmtcp1"])

runTest("plugin-example-db", 2, ["--with-plugin "+
                            PWD+"/test/plugin/example-db/dmtcp_example-dbhijack.so "+
                             "env EXAMPLE_DB_KEY=1 EXAMPLE_DB_KEY_OTHER=2 "+
                             "./test/dmtcp1",
                                 "--with-plugin "+
                            PWD+"/test/plugin/example-db/dmtcp_example-dbhijack.so "+
                             "env EXAMPLE_DB_KEY=2 EXAMPLE_DB_KEY_OTHER=1 "+
                             "./test/dmtcp1"])

# Test special case:  gettimeofday can be handled within VDSO segment.
runTest("gettimeofday",  1, ["./test/gettimeofday"])

runTest("sigchild",      1, ["./test/sigchild"])

runTest("shared-fd",     2, ["./test/shared-fd"])

runTest("stale-fd",      2, ["./test/stale-fd"])

# Disable procfd1 until we fix readlink
#runTest("procfd1",       2, ["./test/procfd1"])

# popen1 can have more than one processes
#runTest("popen1",          1, ["./test/popen1"])

runTest("poll",          1, ["./test/poll"])

runTest("forkexec",      2, ["./test/forkexec"])

if HAS_SSH == "yes":
  S=5*DEFAULT_S
  runTest("sshtest",     4, ["./test/sshtest"])
  S=DEFAULT_S

S=2*DEFAULT_S
runTest("waitpid",      2, ["./test/waitpid"])
S=DEFAULT_S

runTest("client-server", 2, ["./test/client-server"])

# frisbee creates three processes, each with 14 MB, if no gzip is used
os.environ['DMTCP_GZIP'] = "1"
runTest("frisbee",       3, ["./test/frisbee "+p1+" localhost "+p2,
                             "./test/frisbee "+p2+" localhost "+p3,
                             "./test/frisbee "+p3+" localhost "+p1+" starter"])
os.environ['DMTCP_GZIP'] = "0"

runTest("shared-memory", 2, ["./test/shared-memory"])

runTest("sysv-shm1",     2, ["./test/sysv-shm1"])
runTest("sysv-shm2",     2, ["./test/sysv-shm2"])
runTest("sysv-sem",      2, ["./test/sysv-sem"])
runTest("sysv-msg",      2, ["./test/sysv-msg"])

# ARM glibc 2.16 with Linux kernel 3.0 doesn't support mq_send, etc.
if uname_p[0:3] == 'arm':
  print "Skipping posix-mq1/mq2 tests; ARM/glibc/Linux does not support mq_send"
elif TEST_POSIX_MQ == "yes":
  runTest("posix-mq1",     2, ["./test/posix-mq1"])
  runTest("posix-mq2",     2, ["./test/posix-mq2"])

#Invoke this test when we drain/restore data in pty at checkpoint time.
# runTest("pty1",   2, ["./test/pty1"])
runTest("pty2",   2, ["./test/pty2"])

#Invoke this test when support for timers is added to DMTCP.
runTest("timer",   1, ["./test/timer"])
runTest("clock",   1, ["./test/clock"])

old_ld_library_path = os.getenv("LD_LIBRARY_PATH")
if old_ld_library_path:
  os.environ['LD_LIBRARY_PATH'] += ':' + os.getenv("PWD") + \
                                   "/test:" + os.getenv("PWD")
else:
  os.environ['LD_LIBRARY_PATH'] = os.getenv("PWD") + "/test:" + os.getenv("PWD")
runTest("dlopen1",        1, ["./test/dlopen1"])
# Disable the dlopen2 test until we can figure out a way to handle calls to
# fork/exec/wait during library intialization with dlopen().
#if not USE_M32:
#  runTest("dlopen2",        1, ["./test/dlopen2"])
if old_ld_library_path:
  os.environ['LD_LIBRARY_PATH'] = old_ld_library_path
else:
  del os.environ['LD_LIBRARY_PATH']

runTest("pthread1",      1, ["./test/pthread1"])
runTest("pthread2",      1, ["./test/pthread2"])
S=10*DEFAULT_S
runTest("pthread3",      1, ["./test/pthread2 80"])
S=DEFAULT_S
runTest("pthread4",      1, ["./test/pthread4"])
runTest("pthread5",      1, ["./test/pthread5"])

os.environ['DMTCP_GZIP'] = "1"
runTest("gzip",          1, ["./test/dmtcp1"])
os.environ['DMTCP_GZIP'] = GZIP

if HAS_READLINE == "yes":
  runTest("readline",    1,  ["./test/readline"])

runTest("perl",          1, ["/usr/bin/perl"])

if HAS_PYTHON == "yes":
  runTest("python",      1, ["/usr/bin/python"])

os.environ['DMTCP_GZIP'] = "0"
runTest("bash",        2, ["/bin/bash --norc -c 'ls; sleep 30; ls'"])
os.environ['DMTCP_GZIP'] = GZIP

if HAS_DASH == "yes":
  os.environ['DMTCP_GZIP'] = "0"
  os.unsetenv('ENV')  # Delete reference to dash initialization file
  runTest("dash",        2, ["/bin/dash -c 'ls; sleep 30; ls'"])
  os.environ['DMTCP_GZIP'] = GZIP

if HAS_TCSH == "yes":
  os.environ['DMTCP_GZIP'] = "0"
  runTest("tcsh",        2, ["/bin/tcsh -f -c 'ls; sleep 30; ls'"])
  os.environ['DMTCP_GZIP'] = GZIP

if HAS_ZSH == "yes":
  os.environ['DMTCP_GZIP'] = "0"
  S=3*DEFAULT_S
  runTest("zsh",         2, ["/bin/zsh -f -c 'ls; sleep 30; ls'"])
  S=DEFAULT_S
  os.environ['DMTCP_GZIP'] = GZIP

if HAS_VIM == "yes":
  # Wait to checkpoint until vim finishes reading its initialization files
  S=10*DEFAULT_S
  if sys.version_info[0:2] >= (2,6):
    # Delete previous vim processes.  Vim behaves poorly with stale processes.
    vimCommand = VIM + " /etc/passwd +3" # +3 makes cmd line unique
    def killCommand(cmdToKill):
      if os.getenv('USER') == None or HAS_PS == 'no':
        return
      ps = subprocess.Popen(['ps', '-u', os.environ['USER'], '-o',
                             'pid,command'],
                            stdout=subprocess.PIPE).communicate()[0]
      for row in ps.split('\n')[1:]:
        cmd = row.split(None, 1) # maxsplit=1
        if len(cmd) > 1 and cmd[1] == cmdToKill:
          os.kill(int(cmd[0]), signal.SIGKILL)
    killCommand(vimCommand)
    runTest("vim",       1,  ["env TERM=vt100 " + vimCommand])
    killCommand(vimCommand)
  S=DEFAULT_S

if sys.version_info[0:2] >= (2,6):
  #On some systems, "emacs -nw" launches dbus-daemon processes in
  #background throwing off the number of processes in the computation. The
  #test thus fails. The fix is to launch emacs-nox, if found. emacs-nox
  #doesn't launch any background processes.
  S=15*DEFAULT_S
  if HAS_EMACS_NOX == "yes":
    # Wait to checkpoint until emacs finishes reading its initialization files
    # Under emacs23, it opens /dev/tty directly in a new fd.
    # To avoid this, consider using emacs --batch -l EMACS-LISTP-CODE ...
    # ... or else a better pty wrapper to capture emacs output to /dev/tty.
    runTest("emacs",     1,  ["env TERM=vt100 /usr/bin/emacs-nox" +
                              " --no-init-file /etc/passwd"])
  elif HAS_EMACS == "yes":
    # Wait to checkpoint until emacs finishes reading its initialization files
    # Under emacs23, it opens /dev/tty directly in a new fd.
    # To avoid this, consider using emacs --batch -l EMACS-LISTP-CODE ...
    # ... or else a better pty wrapper to capture emacs output to /dev/tty.
    runTest("emacs",     1,  ["env TERM=vt100 /usr/bin/emacs -nw" +
                                " --no-init-file /etc/passwd"])
  S=DEFAULT_S

if HAS_SCRIPT == "yes":
  S=7*DEFAULT_S
  if sys.version_info[0:2] >= (2,6):
    # NOTE: If 'script' fails, try raising value of S, above, to larger number.
    #  Arguably, there is a bug in glibc, in that locale-archive can be 100 MB.
    #  For example, in Fedora 13 (and other recent Red Hat-derived distros?),
    #  /usr/lib/locale/locale-archive is 100 MB, and yet 'locale -a |wc' shows
    #  only 8KB of content in ASCII.  The 100 MB of locale-archive condenses
    #  to 25 MB _per process_ under gzip, but this can be slow at ckpt time.
    runTest("script",    4,  ["/usr/bin/script -f" +
                              " -c 'bash -c \"ls; sleep 30\"'" +
                              " dmtcp-test-typescript.tmp"])
  os.system("rm -f dmtcp-test-typescript.tmp")
  S=DEFAULT_S

# SHOULD HAVE screen RUN SOMETHING LIKE:  bash -c ./test/dmtcp1
# FIXME: Currently fails on dekaksi due to DMTCP not honoring
#        "Async-signal-safe functions" in signal handlers (see man 7 signal)
if HAS_SCREEN == "yes":
  S=3*DEFAULT_S
  if sys.version_info[0:2] >= (2,6):
    runTest("screen",    3,  ["env TERM=vt100 " + SCREEN +
                                " -c /dev/null -s /bin/sh"])
  S=DEFAULT_S

if PTRACE_SUPPORT == "yes" and ARM_HOST == "no" and \
   sys.version_info[0:2] >= (2,6):
  if HAS_STRACE == "yes":
    S=10*DEFAULT_S
    runTest("strace",    2,  ["--ptrace strace test/dmtcp2"])
    S=DEFAULT_S

  if HAS_GDB == "yes":
    if uname_p[0:3] == 'arm':
      print "On ARM, there is a known issue with DMTCP for gdb-* test." + \
            "  Not running it."
    else:
      os.system("echo 'run' > dmtcp-gdbinit.tmp")
      S=10*DEFAULT_S
      runTest("gdb",          2,
              ["--ptrace gdb -n -batch -x dmtcp-gdbinit.tmp test/dmtcp1"])

      runTest("gdb-pthread0", 2,
              ["--ptrace gdb -n -batch -x dmtcp-gdbinit.tmp test/dmtcp3"])

      # These tests currently fail sometimes (if the computation is checkpointed
      # while a thread is being created). Re-enable them when this issue has
      # been fixed in the ptrace plugin.
      #runTest("gdb-pthread1", 2, ["gdb -n -batch -x dmtcp-gdbinit.tmp test/pthread1"])
      #runTest("gdb-pthread2",2, ["gdb -n -batch -x dmtcp-gdbinit.tmp test/pthread2"])

      S=DEFAULT_S
      os.system("rm -f dmtcp-gdbinit.tmp")

if HAS_JAVAC == "yes" and HAS_JAVA == "yes":
  S=10*DEFAULT_S
  os.environ['CLASSPATH'] = './test'
  runTest("java1",         1,  ["java -Xmx5M java1"])
  del os.environ['CLASSPATH']
  S=DEFAULT_S

if HAS_CILK == "yes":
  runTest("cilk1",        1,  ["./test/cilk1 38"])

# SHOULD HAVE gcl RUN LARGE FACTORIAL OR SOMETHING.
if HAS_GCL == "yes":
  S=3*DEFAULT_S
  runTest("gcl",         1,  [GCL])
  S=DEFAULT_S

if HAS_OPENMP == "yes":
  runTest("openmp-1",         1,  ["./test/openmp-1"])
  runTest("openmp-2",         1,  ["./test/openmp-2"])

# SHOULD HAVE matlab RUN LARGE FACTORIAL OR SOMETHING.
if HAS_MATLAB == "yes" and sys.version_info[0:2] >= (2,6):
  S=10*DEFAULT_S
  runTest("matlab-nodisplay", 1,  [MATLAB+" -nodisplay -nojvm"])
  S=DEFAULT_S

if HAS_MPICH == "yes":
  runTest("mpd",         1, [MPICH_MPD])

  runTest("hellompich-n1", 4,
          [MPICH_MPD, MPICH_MPIEXEC + " -n 1 ./test/hellompich"])

  runTest("hellompich-n2", 6,
          [MPICH_MPD, MPICH_MPIEXEC + " -n 2 ./test/hellompich"])

  runTest("mpdboot",     1, [MPICH_MPDBOOT+" -n 1"])

  #os.system(MPICH_MPDCLEANUP)


if HAS_OPENMPI == "yes":
  # Compute:  USES_OPENMPI_ORTED
  if os.path.isfile('./test/openmpi') and \
     0 == os.system(OPENMPI_MPICC +
                    " -o ./test_openmpi test/hellompi.c 2>/dev/null 1>&2"):
    os.system("rm -f ./uses_openmpi_orted")
    # The 'sleep 1' below may not fix the race, creating a runaway test_openmpi.
    os.system('/bin/sh -c "$OPENMPI_MPIRUN -np 2 ./test_openmpi' +
              '   2>/dev/null 1>&2 &'
              ' sleep 1 &&'
              ' ps auxw | grep $USER | grep -v grep | grep -q orted &&'
              ' touch ./uses_openmpi_orted" 2>/dev/null')
    os.system("/bin/kill -9 `ps -eo pid,args | grep test_openmpi |" +
              " sed -e 's%\([0-9]\) .*$%\1%'` 2>/dev/null")
    if os.path.exists('./uses_openmpi_orted'):
      os.system('rm -f ./uses_openmpi_orted')
      USES_OPENMPI_ORTED = "yes"
    else:
      USES_OPENMPI_ORTED = "no"
  else:
    HAS_OPENMPI = "no"
  os.system('rm -f ./test_openmpi')

#Temporarily disabling Open MPI test as it fails on some distros (OpenSUSE 11.4)
if HAS_OPENMPI == "yes":
  numProcesses = 5 + int(USES_OPENMPI_ORTED == "yes")
  # FIXME: Replace "[5,6]" by numProcesses when bug in configure is fixed.
  # /usr/bin/openmpi does not work if /usr/bin is not also in user's PATH
  oldPath = ""
  if not os.environ.has_key('PATH'):
    oldPath = None
    os.environ['PATH'] = os.path.dirname(OPENMPI_MPIRUN)
  elif (not re.search(os.path.dirname(OPENMPI_MPIRUN),
                     os.environ['PATH'])):
    oldPath = os.environ['PATH']
    os.environ['PATH'] += ":" + os.path.dirname(OPENMPI_MPIRUN)
  S=3*DEFAULT_S
  runTest("openmpi", [5,6], [OPENMPI_MPIRUN + " -np 4" +
                             " ./test/openmpi"])
  S=DEFAULT_S
  if oldPath:
    os.environ['PATH'] = oldPath
  if oldPath == None:
    del os.environ['PATH']

print "== Summary =="
print "%s: %d of %d tests passed" % (socket.gethostname(), stats[0], stats[1])

saveResultsNMI()

try:
  SHUTDOWN()
except CheckFailed, e:
  print "Error in SHUTDOWN():", e.value
except:
  print "Error in SHUTDOWN()"

sys.exit( stats[1] - stats[0] )  # Return code is number of failing tests.
