|
#!/usr/bin/env python |
|
# Copyright (c) 2006 The Regents of The University of Michigan |
|
# All rights reserved. |
|
# |
|
# Redistribution and use in source and binary forms, with or without |
|
# modification, are permitted provided that the following conditions are |
|
# met: redistributions of source code must retain the above copyright |
|
# notice, this list of conditions and the following disclaimer; |
|
# redistributions in binary form must reproduce the above copyright |
|
# notice, this list of conditions and the following disclaimer in the |
|
# documentation and/or other materials provided with the distribution; |
|
# neither the name of the copyright holders nor the names of its |
|
# contributors may be used to endorse or promote products derived from |
|
# this software without specific prior written permission. |
|
# |
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
|
# |
|
# Authors: Kevin Lim |
|
|
|
import os, os.path, re, socket, sys |
|
from os import environ as env, listdir |
|
from os.path import basename, isdir, isfile, islink, join as joinpath, normpath |
|
from filecmp import cmp as filecmp |
|
from shutil import copy |
|
|
|
def nfspath(dir): |
|
if dir.startswith('/.automount/'): |
|
dir = '/n/%s' % dir[12:] |
|
elif not dir.startswith('/n/'): |
|
dir = '/n/%s%s' % (socket.gethostname().split('.')[0], dir) |
|
return dir |
|
|
|
def syncdir(srcdir, destdir): |
|
srcdir = normpath(srcdir) |
|
destdir = normpath(destdir) |
|
if not isdir(destdir): |
|
sys.exit('destination directory "%s" does not exist' % destdir) |
|
|
|
for root, dirs, files in os.walk(srcdir): |
|
root = normpath(root) |
|
prefix = os.path.commonprefix([root, srcdir]) |
|
root = root[len(prefix):] |
|
if root.startswith('/'): |
|
root = root[1:] |
|
for rem in [ d for d in dirs if d.startswith('.') or d == 'SCCS']: |
|
dirs.remove(rem) |
|
|
|
for entry in dirs: |
|
newdir = joinpath(destdir, root, entry) |
|
if not isdir(newdir): |
|
os.mkdir(newdir) |
|
print 'mkdir', newdir |
|
|
|
for i,d in enumerate(dirs): |
|
if islink(joinpath(srcdir, root, d)): |
|
dirs[i] = joinpath(d, '.') |
|
|
|
for entry in files: |
|
dest = normpath(joinpath(destdir, root, entry)) |
|
src = normpath(joinpath(srcdir, root, entry)) |
|
if not isfile(dest) or not filecmp(src, dest): |
|
print 'copy %s %s' % (dest, src) |
|
copy(src, dest) |
|
|
|
progpath = nfspath(sys.path[0]) |
|
progname = basename(sys.argv[0]) |
|
usage = """\ |
|
Usage: |
|
%(progname)s [-c] [-e] [-f] [-j <jobfile>] [-q queue] [-v] <regexp> |
|
-c clean directory if job can be run |
|
-C submit the checkpointing runs |
|
-d Make jobs be dependent on the completion of the checkpoint runs |
|
-e only echo pbs command info, don't actually send the job |
|
-f force the job to run regardless of state |
|
-q <queue> submit job to the named queue |
|
-j <jobfile> specify the jobfile (default is <rootdir>/Test.py) |
|
-v be verbose |
|
|
|
%(progname)s [-j <jobfile>] -l [-v] <regexp> |
|
-j <jobfile> specify the jobfile (default is <rootdir>/Test.py) |
|
-l list job names, don't submit |
|
-v be verbose (list job parameters) |
|
|
|
%(progname)s -h |
|
-h display this help |
|
""" % locals() |
|
|
|
try: |
|
import getopt |
|
opts, args = getopt.getopt(sys.argv[1:], '-Ccdefhj:lnq:Rt:v') |
|
except getopt.GetoptError: |
|
sys.exit(usage) |
|
|
|
depend = False |
|
clean = False |
|
onlyecho = False |
|
exprs = [] |
|
force = False |
|
listonly = False |
|
queue = '' |
|
verbose = False |
|
jfile = 'Test.py' |
|
docpts = False |
|
doruns = True |
|
runflag = False |
|
node_type = 'FAST' |
|
update = True |
|
|
|
for opt,arg in opts: |
|
if opt == '-C': |
|
docpts = True |
|
if opt == '-c': |
|
clean = True |
|
if opt == '-d': |
|
depend = True |
|
if opt == '-e': |
|
onlyecho = True |
|
if opt == '-f': |
|
force = True |
|
if opt == '-h': |
|
print usage |
|
sys.exit(0) |
|
if opt == '-j': |
|
jfile = arg |
|
if opt == '-l': |
|
listonly = True |
|
if opt == '-n': |
|
update = False |
|
if opt == '-q': |
|
queue = arg |
|
if opt == '-R': |
|
runflag = True |
|
if opt == '-t': |
|
node_type = arg |
|
if opt == '-v': |
|
verbose = True |
|
|
|
if docpts: |
|
doruns = runflag |
|
|
|
for arg in args: |
|
exprs.append(re.compile(arg)) |
|
|
|
import jobfile, batch |
|
from job import JobDir, date |
|
|
|
conf = jobfile.JobFile(jfile) |
|
|
|
if update and not listonly and not onlyecho and isdir(conf.linkdir): |
|
if verbose: |
|
print 'Checking for outdated files in Link directory' |
|
if not isdir(conf.basedir): |
|
os.mkdir(conf.basedir) |
|
syncdir(conf.linkdir, conf.basedir) |
|
|
|
jobnames = {} |
|
joblist = [] |
|
|
|
if docpts and doruns: |
|
gen = conf.alljobs() |
|
elif docpts: |
|
gen = conf.checkpoints() |
|
elif doruns: |
|
gen = conf.jobs() |
|
|
|
for job in gen: |
|
if job.name in jobnames: |
|
continue |
|
|
|
if exprs: |
|
for expr in exprs: |
|
if expr.match(job.name): |
|
joblist.append(job) |
|
break |
|
else: |
|
joblist.append(job) |
|
|
|
if listonly: |
|
if verbose: |
|
for job in joblist: |
|
job.printinfo() |
|
else: |
|
for job in joblist: |
|
print job.name |
|
sys.exit(0) |
|
|
|
if not onlyecho: |
|
newlist = [] |
|
for job in joblist: |
|
jobdir = JobDir(joinpath(conf.rootdir, job.name)) |
|
if jobdir.exists(): |
|
if not force: |
|
status = jobdir.getstatus() |
|
if status == 'queued': |
|
continue |
|
|
|
if status == 'running': |
|
continue |
|
|
|
if status == 'success': |
|
continue |
|
|
|
if not clean: |
|
sys.exit('job directory %s not clean!' % jobdir) |
|
|
|
jobdir.clean() |
|
newlist.append(job) |
|
joblist = newlist |
|
|
|
class NameHack(object): |
|
def __init__(self, host='pbs.pool', port=24465): |
|
self.host = host |
|
self.port = port |
|
self.socket = None |
|
|
|
def setname(self, jobid, jobname): |
|
try: |
|
jobid = int(jobid) |
|
except ValueError: |
|
jobid = int(jobid.strip().split('.')[0]) |
|
|
|
jobname = jobname.strip() |
|
# since pbs can handle jobnames of 15 characters or less, |
|
# don't use the raj hack. |
|
if len(jobname) <= 15: |
|
return |
|
|
|
if self.socket is None: |
|
import socket |
|
self.socket = socket.socket() |
|
# Connect to pbs.pool and send the jobid/jobname pair to port |
|
# 24465 (Raj didn't realize that there are only 64k ports and |
|
# setup inetd to point to port 90001) |
|
self.socket.connect((self.host, self.port)) |
|
|
|
self.socket.send("%s %s\n" % (jobid, jobname)) |
|
|
|
namehack = NameHack() |
|
|
|
rootdir = conf.rootdir |
|
script = joinpath(rootdir, 'Base', 'job.py') |
|
|
|
for job in joblist: |
|
jobdir = JobDir(joinpath(rootdir, job.name)) |
|
if depend: |
|
cptdir = JobDir(joinpath(rootdir, job.checkpoint.name)) |
|
path = str(cptdir) |
|
if not isdir(path) or not isfile(joinpath(path, '.success')): |
|
continue |
|
|
|
cptjob = cptdir.readval('.batch_jobid') |
|
|
|
if not onlyecho: |
|
jobdir.create() |
|
os.chdir(str(jobdir)) |
|
os.environ['PWD'] = str(jobdir) |
|
|
|
print 'Job name: %s' % job.name |
|
print 'Job directory: %s' % jobdir |
|
|
|
|
|
qsub = batch.oarsub() |
|
qsub.oarhost = 'poolfs.eecs.umich.edu' |
|
#qsub.stdout = jobdir.file('jobout') |
|
qsub.name = job.name |
|
qsub.walltime = '50' |
|
#qsub.join = True |
|
#qsub.node_type = node_type |
|
#qsub.env['ROOTDIR'] = conf.rootdir |
|
#qsub.env['JOBNAME'] = job.name |
|
#if depend: |
|
# qsub.afterok = cptjob |
|
#if queue: |
|
# qsub.queue = queue |
|
qsub.properties = "64bit = 'Yes' or 64bit = 'No'" |
|
qsub.build(script) |
|
|
|
if verbose: |
|
print 'cwd: %s' % qsub.command |
|
print 'PBS Command: %s' % qsub.command |
|
|
|
if not onlyecho: |
|
ec = qsub.do() |
|
if ec == 0: |
|
jobid = qsub.result |
|
print 'OAR Jobid: %s' % jobid |
|
#namehack.setname(jobid, job.name) |
|
queued = date() |
|
jobdir.echofile('.batch_jobid', jobid) |
|
jobdir.echofile('.batch_jobname', job.name) |
|
jobdir.echofile('.queued', queued) |
|
jobdir.setstatus('queued on %s' % queued) |
|
else: |
|
print 'OAR Failed' |
|
print |
|
print |
