[ARVADOS] updated: 412c3d66297962c8b2b0385ec0fa14d470dfdb03
git at public.curoverse.com
git at public.curoverse.com
Thu Aug 14 13:17:16 EDT 2014
Summary of changes:
crunch_scripts/arvados-bcbio-nextgen.py | 2 +-
crunch_scripts/collection-merge | 2 +-
.../crunchutil}/__init__.py | 0
crunch_scripts/{ => crunchutil}/robust_put.py | 1 +
crunch_scripts/{ => crunchutil}/subst.py | 2 +-
crunch_scripts/{ => crunchutil}/vwd.py | 2 +-
crunch_scripts/decompress-all.py | 8 +++---
crunch_scripts/run-command | 31 +++++++++++++---------
crunch_scripts/split-fastq.py | 6 ++---
9 files changed, 31 insertions(+), 23 deletions(-)
copy {services/fuse/tests => crunch_scripts/crunchutil}/__init__.py (100%)
rename crunch_scripts/{ => crunchutil}/robust_put.py (99%)
rename crunch_scripts/{ => crunchutil}/subst.py (98%)
rename crunch_scripts/{ => crunchutil}/vwd.py (98%)
via 412c3d66297962c8b2b0385ec0fa14d470dfdb03 (commit)
via 54e4827b3e5918b5df439f7c273eedb3c00e372f (commit)
via eb6992db915a1fea2a8f9acfccf26d2d091e66e0 (commit)
from 02d6330526fbd94f4a1d711ee6ffc18a1553febf (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 412c3d66297962c8b2b0385ec0fa14d470dfdb03
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Aug 14 13:17:06 2014 -0400
3505: tinker with logging
diff --git a/crunch_scripts/run-command b/crunch_scripts/run-command
index 9e8c13f..c624e3c 100755
--- a/crunch_scripts/run-command
+++ b/crunch_scripts/run-command
@@ -1,5 +1,8 @@
#!/usr/bin/env python
+import logging
+logging.basicConfig(level=logging.INFO, format="run-command: %(message)s")
+
import arvados
import re
import os
@@ -15,12 +18,10 @@ import copy
import traceback
import pprint
import multiprocessing
-import logging
import crunchutil.robust_put as robust_put
import crunchutil.vwd as vwd
os.umask(0077)
-logging.basicConfig(format="run-command: %(message)s")
t = arvados.current_task().tmpdir
commit 54e4827b3e5918b5df439f7c273eedb3c00e372f
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Aug 14 11:41:45 2014 -0400
3505: Fix typo in run-command
diff --git a/crunch_scripts/run-command b/crunch_scripts/run-command
index 54f6838..9e8c13f 100755
--- a/crunch_scripts/run-command
+++ b/crunch_scripts/run-command
@@ -164,7 +164,7 @@ try:
stdoutname = subst.do_substitution(taskp, taskp["task.stdout"])
stdoutfile = open(stdoutname, "wb")
- logging.info("{}{}{}".format(' '.join(cmd), (" < " + stdinname) if stdinname is not None else ""), (" > " + stdoutname) if stdoutname is not None else ""))
+ logging.info("{}{}{}".format(' '.join(cmd), (" < " + stdinname) if stdinname is not None else "", (" > " + stdoutname) if stdoutname is not None else ""))
except Exception as e:
logging.exception("caught exception")
commit eb6992db915a1fea2a8f9acfccf26d2d091e66e0
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Aug 14 11:35:06 2014 -0400
3505: Move helper scripts into crunchutil module. In run-command, added
task.stdin redirection, and renamed save.stdout to task.stdout for consistency.
Stylistic change "x == None" and "x != None" to "x is None" and "x is not
None".
diff --git a/crunch_scripts/arvados-bcbio-nextgen.py b/crunch_scripts/arvados-bcbio-nextgen.py
index df66cf6..c296248 100755
--- a/crunch_scripts/arvados-bcbio-nextgen.py
+++ b/crunch_scripts/arvados-bcbio-nextgen.py
@@ -2,7 +2,7 @@
import arvados
import subprocess
-import subst
+import crunchutil.subst as subst
import shutil
import os
import sys
diff --git a/crunch_scripts/collection-merge b/crunch_scripts/collection-merge
index 63b63fa..ca80a82 100755
--- a/crunch_scripts/collection-merge
+++ b/crunch_scripts/collection-merge
@@ -15,7 +15,7 @@
import arvados
import md5
-import subst
+import crunchutil.subst as subst
import subprocess
import os
import hashlib
diff --git a/crunch_scripts/crunchutil/__init__.py b/crunch_scripts/crunchutil/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/crunch_scripts/robust_put.py b/crunch_scripts/crunchutil/robust_put.py
similarity index 99%
rename from crunch_scripts/robust_put.py
rename to crunch_scripts/crunchutil/robust_put.py
index ce05415..158ceb1 100644
--- a/crunch_scripts/robust_put.py
+++ b/crunch_scripts/crunchutil/robust_put.py
@@ -1,6 +1,7 @@
import arvados
import arvados.commands.put as put
import os
+import logging
def machine_progress(bytes_written, bytes_expected):
return "upload wrote {} total {}\n".format(
diff --git a/crunch_scripts/subst.py b/crunch_scripts/crunchutil/subst.py
similarity index 98%
rename from crunch_scripts/subst.py
rename to crunch_scripts/crunchutil/subst.py
index 8154d0e..b352688 100644
--- a/crunch_scripts/subst.py
+++ b/crunch_scripts/crunchutil/subst.py
@@ -59,7 +59,7 @@ def do_substitution(p, c, subs=default_subs):
while True:
#print("c is", c)
m = search(c)
- if m != None:
+ if m is not None:
v = do_substitution(p, c[m[0]+2 : m[1]])
var = True
for sub in subs:
diff --git a/crunch_scripts/vwd.py b/crunch_scripts/crunchutil/vwd.py
similarity index 98%
rename from crunch_scripts/vwd.py
rename to crunch_scripts/crunchutil/vwd.py
index beb6b2f..3d54c9c 100644
--- a/crunch_scripts/vwd.py
+++ b/crunch_scripts/crunchutil/vwd.py
@@ -14,7 +14,7 @@ import stat
# doesn't exist.
def checkout(source_collection, target_dir, keepmount=None):
# create symlinks
- if keepmount == None:
+ if keepmount is None:
keepmount = os.environ['TASK_KEEPMOUNT']
if not os.path.exists(target_dir):
diff --git a/crunch_scripts/decompress-all.py b/crunch_scripts/decompress-all.py
index 261c78e..50d11f4 100755
--- a/crunch_scripts/decompress-all.py
+++ b/crunch_scripts/decompress-all.py
@@ -18,7 +18,7 @@ import re
import subprocess
import os
import sys
-import robust_put
+import crunchutil.robust_put as robust_put
arvados.job_setup.one_task_per_input_file(if_sequence=0, and_end_task=True,
input_as_path=True)
@@ -33,7 +33,7 @@ outdir = os.path.join(task.tmpdir, "output")
os.makedirs(outdir)
os.chdir(outdir)
-if infile_parts == None:
+if infile_parts is None:
print >>sys.stderr, "Failed to parse input filename '%s' as a Keep file\n" % input_file
sys.exit(1)
@@ -41,7 +41,7 @@ cr = arvados.CollectionReader(infile_parts.group(1))
streamname = infile_parts.group(3)[1:]
filename = infile_parts.group(4)[1:]
-if streamname != None:
+if streamname is not None:
subprocess.call(["mkdir", "-p", streamname])
os.chdir(streamname)
else:
@@ -49,7 +49,7 @@ else:
m = re.match(r'.*\.(gz|Z|bz2|tgz|tbz|zip|rar|7z|cab|deb|rpm|cpio|gem)$', arvados.get_task_param_mount('input'), re.IGNORECASE)
-if m != None:
+if m is not None:
rc = subprocess.call(["dtrx", "-r", "-n", "-q", arvados.get_task_param_mount('input')])
if rc == 0:
task.set_output(robust_put.upload(outdir))
diff --git a/crunch_scripts/run-command b/crunch_scripts/run-command
index c85e74a..54f6838 100755
--- a/crunch_scripts/run-command
+++ b/crunch_scripts/run-command
@@ -6,7 +6,7 @@ import os
import subprocess
import sys
import shutil
-import subst
+import crunchutil.subst as subst
import time
import arvados.commands.put as put
import signal
@@ -16,8 +16,8 @@ import traceback
import pprint
import multiprocessing
import logging
-import robust_put
-import vwd
+import crunchutil.robust_put as robust_put
+import crunchutil.vwd as vwd
os.umask(0077)
logging.basicConfig(format="run-command: %(message)s")
@@ -102,7 +102,7 @@ def get_items(p, value):
fn = subst.do_substitution(p, value)
mode = os.stat(fn).st_mode
prefix = fn[len(os.environ['TASK_KEEPMOUNT'])+1:]
- if mode != None:
+ if mode is not None:
if stat.S_ISDIR(mode):
items = ["$(dir %s/%s/)" % (prefix, l) for l in os.listdir(fn)]
elif stat.S_ISREG(mode):
@@ -114,6 +114,8 @@ def get_items(p, value):
stdoutname = None
stdoutfile = None
+stdinname = None
+stdinfile = None
rcode = 1
try:
@@ -122,7 +124,7 @@ try:
var = jobp["task.foreach"]
items = get_items(jobp, jobp[var])
logging.info("parallelizing on %s with items %s" % (var, items))
- if items != None:
+ if items is not None:
for i in items:
params = copy.copy(jobp)
params[var] = i
@@ -154,11 +156,15 @@ try:
cmd = expand_list(taskp, taskp["command"])
- if "save.stdout" in taskp:
- stdoutname = subst.do_substitution(taskp, taskp["save.stdout"])
+ if "task.stdin" in taskp:
+ stdinname = subst.do_substitution(taskp, taskp["task.stdin"])
+ stdinfile = open(stdinname, "rb")
+
+ if "task.stdout" in taskp:
+ stdoutname = subst.do_substitution(taskp, taskp["task.stdout"])
stdoutfile = open(stdoutname, "wb")
- logging.info("{}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
+ logging.info("{}{}{}".format(' '.join(cmd), (" < " + stdinname) if stdinname is not None else ""), (" > " + stdoutname) if stdoutname is not None else ""))
except Exception as e:
logging.exception("caught exception")
@@ -167,7 +173,7 @@ except Exception as e:
sys.exit(1)
try:
- sp = subprocess.Popen(cmd, shell=False, stdout=stdoutfile)
+ sp = subprocess.Popen(cmd, shell=False, stdin=stdinfile, stdout=stdoutfile)
sig = SigHandler()
# forward signals to the process.
@@ -178,7 +184,7 @@ try:
# wait for process to complete.
rcode = sp.wait()
- if sig.sig != None:
+ if sig.sig is not None:
logging.critical("terminating on signal %s" % sig.sig)
sys.exit(2)
else:
diff --git a/crunch_scripts/split-fastq.py b/crunch_scripts/split-fastq.py
index be37f04..253dd22 100755
--- a/crunch_scripts/split-fastq.py
+++ b/crunch_scripts/split-fastq.py
@@ -94,7 +94,7 @@ prog = re.compile(r'(.*?)(_[12])?\.fastq(\.gz)?$')
for s in inp.all_streams():
for f in s.all_files():
name_pieces = prog.match(f.name())
- if name_pieces != None:
+ if name_pieces is not None:
if s.name() != ".":
# The downstream tool (run-command) only iterates over the top
# level of directories so if there are fastq files in
@@ -105,7 +105,7 @@ for s in inp.all_streams():
sys.exit(1)
p = None
- if name_pieces.group(2) != None:
+ if name_pieces.group(2) is not None:
if name_pieces.group(2) == "_1":
p = [{}, {}]
p[0]["reader"] = s.files()[name_pieces.group(0)]
@@ -114,7 +114,7 @@ for s in inp.all_streams():
p = [{}]
p[0]["reader"] = s.files()[name_pieces.group(0)]
- if p != None:
+ if p is not None:
if chunking:
splitfastq(p)
else:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list