util/qdo

#! /usr/bin/env python2.7

# Copyright (c) 2004-2005, 2007 The Regents of The University of Michigan
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Steve Reinhardt
#          Ali Saidi

# Important!
# This script expects a simple $ prompt, if you are using a shell other than
# sh which defaults to this you'll need to add something like the following
# to your bashrc/bash_profile script:
#if [ "$OAR_USER" = "xxxx" ]; then
#   PS1='$ '


import sys
import os
import re
import time
import optparse

import pexpect

progname = os.path.basename(sys.argv[0])

usage = "%prog [options] command [command arguments]"
optparser = optparse.OptionParser(usage=usage)
optparser.allow_interspersed_args=False
optparser.add_option('-e', dest='stderr_file',
                     help='command stderr output file')
optparser.add_option('-o', dest='stdout_file',
                     help='command stdout output file')
optparser.add_option('-l', dest='save_log', action='store_true',
                     help='save oarsub output log file')
optparser.add_option('-N', dest='job_name',
                     help='oarsub job name')
optparser.add_option('-q', dest='dest_queue',
                     help='oarsub destination queue')
optparser.add_option('--qwait', dest='oarsub_timeout', type='int',
                     help='oarsub queue wait timeout', default=30*60)
optparser.add_option('-t', dest='cmd_timeout', type='int',
                     help='command execution timeout', default=600*60)

(options, cmd) = optparser.parse_args()

if cmd == []:
    print >>sys.stderr, "%s: missing command" % progname
    sys.exit(1)

# If we want to do this, need to add check here to make sure cmd[0] is
# a valid PBS job name, else oarsub will die on us.
#
#if not options.job_name:
#    options.job_name = cmd[0]

cwd = os.getcwd()

# Deal with systems where /n is a symlink to /.automount
if cwd.startswith('/.automount/'):
    cwd = cwd.replace('/.automount/', '/n/', 1)

if not cwd.startswith('/n/poolfs/'):
    print >>sys.stderr, "Error: current directory must be under /n/poolfs."
    sys.exit(1)

# The Shell class wraps pexpect.spawn with some handy functions that
# assume the thing on the other end is a Bourne/bash shell.
class Shell(pexpect.spawn):
    # Regexp to match the shell prompt.  We change the prompt to
    # something fixed and distinctive to make it easier to match
    # reliably.
    prompt_re = re.compile('qdo\$ ')

    def __init__(self, cmd):
        # initialize base pexpect.spawn object
        try:
            pexpect.spawn.__init__(self, cmd)
        except pexpect.ExceptionPexpect, exc:
            print "%s:" % progname, exc
            sys.exit(1)
        # full_output accumulates the full output of the session
        self.full_output = ""
        self.quick_timeout = 15
        # wait for a prompt, then change it
        try:
            self.expect('\$ ', options.oarsub_timeout)
        except pexpect.TIMEOUT:
            print >>sys.stderr, "%s: oarsub timed out." % progname
            self.kill(9)
            self.safe_close()
            sys.exit(1)
        self.do_command('unset PROMPT_COMMAND; PS1="qdo$ "')

    # version of expect that updates full_output too
    def expect(self, regexp, timeout = -1):
        pexpect.spawn.expect(self, regexp, timeout)
        self.full_output += self.before + self.after

    # Just issue a command and wait for the next prompt.
    # Returns a string containing the output of the command.
    def do_bare_command(self, cmd, timeout = -1):
        global full_output
        self.sendline(cmd)
        # read back the echo of the command
        self.readline()
        # wait for the next prompt
        self.expect(self.prompt_re, timeout)
        output = self.before.rstrip()
        return output

    # Issue a command, then query its exit status.
    # Returns a (string, int) tuple with the command output and the status.
    def do_command(self, cmd, timeout = -1):
        # do the command itself
        output = self.do_bare_command(cmd, timeout)
        # collect status
        status = int(self.do_bare_command("echo $?", self.quick_timeout))
        return (output, status)

    # Check to see if the given directory exists.
    def dir_exists(self, dirname):
        (output, status) = shell.do_command('[ -d %s ]' % dirname,
                                            self.quick_timeout)
        return status == 0

    # Don't actually try to close it.. just wait until it closes by itself
    # We can't actually kill the pid which is what it's trying to do, and if
    # we call wait we could be in an unfortunate situation of it printing input
    # right as we call wait, so the input is never read and the process never ends
    def safe_close(self):
        count = 0
        while self.isalive() and count < 10:
            time.sleep(1)
        self.close(force=False)

# Spawn the interactive pool job.

# Hack to do link on poolfs... disabled for now since
# compiler/linker/library versioning problems between poolfs and
# nodes.  May never work since poolfs is x86-64 and nodes are 32-bit.
if False and len(cmd) > 50:
    shell_cmd = 'ssh -t poolfs /bin/sh -l'
    print "%s: running %s on poolfs" % (progname, cmd[0])
else:
    shell_cmd = 'oarsub -I'
    if options.job_name:
        shell_cmd += ' -n "%s"' % options.job_name
    if options.dest_queue:
        shell_cmd += ' -q ' + options.dest_queue
    shell_cmd += ' -d %s' % cwd

shell = Shell(shell_cmd)

try:
    # chdir to cwd
    (output, status) = shell.do_command('cd ' + cwd)

    if status != 0:
        raise OSError, "Can't chdir to %s" % cwd

    # wacky hack: sometimes scons will create an output directory then
    # fork a job to generate files in that directory, and the job will
    # get run before the directory creation propagates through NFS.
    # This hack looks for a '-o' option indicating an output file and
    # waits for the corresponding directory to appear if necessary.
    try:
        if 'cc' in cmd[0] or 'g++' in cmd[0]:
            output_dir = os.path.dirname(cmd[cmd.index('-o')+1])
        elif 'm5' in cmd[0]:
            output_dir = cmd[cmd.index('-d')+1]
        else:
            output_dir = None
    except (ValueError, IndexError):
        # no big deal if there's no '-o'/'-d' or if it's the final argument
        output_dir = None

    if output_dir:
        secs_waited = 0
        while not shell.dir_exists(output_dir) and secs_waited < 90:
            time.sleep(5)
            secs_waited += 5
        if secs_waited > 30:
            print "waited", secs_waited, "seconds for", output_dir

    # run command
    if options.stdout_file:
        cmd += ['>', options.stdout_file]
    if options.stderr_file:
        cmd += ['2>', options.stderr_file]
    try:
        (output, status) = shell.do_command(' '.join(cmd), options.cmd_timeout)
    except pexpect.TIMEOUT:
            print >>sys.stderr, "%s: command timed out after %d seconds." \
                  % (progname, options.cmd_timeout)
            shell.sendline('~.') # oarsub/ssh termination escape sequence
            shell.safe_close()
            status = 3
    if output:
        print output
finally:
    # end job
    if shell.isalive():
        shell.sendline('exit')
        shell.expect('Disconnected from OAR job .*')
        shell.safe_close()

    # if there was an error, log the output even if not requested
    if status != 0 or options.save_log:
        log = file('qdo-log.' + str(os.getpid()), 'w')
        log.write(shell.full_output)
        log.close()
del shell

sys.exit(status)