[ARVADOS] updated: b5393c72ddd855564cae2f2d798a3e4496a8d2ec
git at public.curoverse.com
git at public.curoverse.com
Fri Aug 8 16:39:16 EDT 2014
Summary of changes:
crunch_scripts/run-command | 44 +++++++++++++++++------------------
crunch_scripts/split-fastq.py | 7 ++----
sdk/cli/bin/arv-run-pipeline-instance | 4 ++--
3 files changed, 25 insertions(+), 30 deletions(-)
via b5393c72ddd855564cae2f2d798a3e4496a8d2ec (commit)
via db4c2a1cbf4d526a16e59b58fbac81a703dee87b (commit)
via 839f0ec7e4931f77eaf537d4642bb769b22fabe6 (commit)
via cb880a9fa932988892ffc9f95d38166e9dec5458 (commit)
via 5edfdafe30ea9d74a5a49602d61934b793762f4b (commit)
from b1365ce7c1ccc74f479b1ebdf31b5da52028da84 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit b5393c72ddd855564cae2f2d798a3e4496a8d2ec
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Aug 8 16:39:05 2014 -0400
3373: Fix wrong regex group
diff --git a/crunch_scripts/split-fastq.py b/crunch_scripts/split-fastq.py
index 1d81393..e242102 100755
--- a/crunch_scripts/split-fastq.py
+++ b/crunch_scripts/split-fastq.py
@@ -108,10 +108,7 @@ for s in inp.all_streams():
if name_pieces.group(2) == "_1":
p = [{}, {}]
p[0]["reader"] = s.files()[name_pieces.group(0)]
- if name_pieces.group(2) != None:
- p[1]["reader"] = s.files()[name_pieces.group(1) + "_2.fastq" + name_pieces.group(2)]
- else:
- p[1]["reader"] = s.files()[name_pieces.group(1) + "_2.fastq"]
+ p[1]["reader"] = s.files()[name_pieces.group(1) + "_2.fastq" + (name_pieces.group(3) if name_pieces.group(3) else '')]
else:
p = [{}]
p[0]["reader"] = s.files()[name_pieces.group(0)]
commit db4c2a1cbf4d526a16e59b58fbac81a703dee87b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Aug 8 16:33:52 2014 -0400
3373: Fix syntax error in split-fastq
diff --git a/crunch_scripts/split-fastq.py b/crunch_scripts/split-fastq.py
index 8382eb1..1d81393 100755
--- a/crunch_scripts/split-fastq.py
+++ b/crunch_scripts/split-fastq.py
@@ -101,7 +101,7 @@ for s in inp.all_streams():
# directories in the input, the choice is either to forget
# there are directories (which might lead to name conflicts) or
# just fail.
- print >>sys.stderr, "fastq must be at the root of the collection")
+ print >>sys.stderr, "fastq must be at the root of the collection"
sys.exit(1)
if name_pieces.group(2) != None:
commit 839f0ec7e4931f77eaf537d4642bb769b22fabe6
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Aug 8 16:15:55 2014 -0400
3373: more run-command cleanup, global p -> taskp, tweaked logging levels
diff --git a/crunch_scripts/run-command b/crunch_scripts/run-command
index 2ad08d8..7d77248 100755
--- a/crunch_scripts/run-command
+++ b/crunch_scripts/run-command
@@ -35,7 +35,7 @@ outdir = os.getcwd()
taskp = None
jobp = arvados.current_job()['script_parameters']
if len(arvados.current_task()['parameters']) > 0:
- p = arvados.current_task()['parameters']
+ taskp = arvados.current_task()['parameters']
links = []
@@ -71,8 +71,6 @@ subst.default_subs["node.cores"] = sub_cores
subst.default_subs["job.uuid"] = sub_jobid
subst.default_subs["task.uuid"] = sub_taskid
-rcode = 1
-
def machine_progress(bytes_written, bytes_expected):
return "run-command: wrote {} total {}\n".format(
bytes_written, -1 if (bytes_expected is None) else bytes_expected)
@@ -125,13 +123,14 @@ def get_items(p, value):
stdoutname = None
stdoutfile = None
+rcode = 1
try:
if "task.foreach" in jobp:
if arvados.current_task()['sequence'] == 0:
var = jobp["task.foreach"]
items = get_items(jobp, jobp[var])
- logging.info("run-command: parallelizing on %s with items %s" % (var, items))
+ logging.info("parallelizing on %s with items %s" % (var, items))
if items != None:
for i in items:
params = copy.copy(jobp)
@@ -148,21 +147,20 @@ try:
else:
sys.exit(1)
else:
- p = jobp
+ taskp = jobp
- cmd = expand_list(p, p["command"])
+ cmd = expand_list(taskp, taskp["command"])
- if "save.stdout" in p:
- stdoutname = subst.do_substitution(p, p["save.stdout"])
+ if "save.stdout" in taskp:
+ stdoutname = subst.do_substitution(taskp, taskp["save.stdout"])
stdoutfile = open(stdoutname, "wb")
- logging.info("run-command: {}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
+ logging.info("{}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
except Exception as e:
- logging.info("run-command: caught exception:")
- traceback.print_exc(file=sys.stdout)
- logging.info("run-command: task parameters was:")
- pprint.plogging.info(p)
+ logging.exception("caught exception")
+ logging.error("task parameters was:")
+ logging.error(pprint.pformat(taskp))
sys.exit(1)
try:
@@ -178,14 +176,13 @@ try:
rcode = sp.wait()
if sig.sig != None:
- logging.warning("run-command: terminating on signal %s" % sig.sig)
+ logging.critical("terminating on signal %s" % sig.sig)
sys.exit(2)
else:
- logging.info("run-command: completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
+ logging.info("completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
except Exception as e:
- logging.error("run-command: caught exception:")
- traceback.print_exc(file=sys.stdout)
+ logging.exception("caught exception")
# restore default signal handlers.
signal.signal(signal.SIGINT, signal.SIG_DFL)
@@ -195,11 +192,11 @@ signal.signal(signal.SIGQUIT, signal.SIG_DFL)
for l in links:
os.unlink(l)
-logging.info("run-command: the following output files will be saved to keep:")
+logging.info("the following output files will be saved to keep:")
-subprocess.call(["find", ".", "-type", "f", "-printf", "run-command: %12.12s %h/%f\\n"])
+subprocess.call(["find", ".", "-type", "f", "-printf", "run-command: %12.12s %h/%f\\n"], stdout=sys.stderr)
-logging.info("run-command: start writing output to keep")
+logging.info("start writing output to keep")
done = False
resume_cache = put.ResumeCache(os.path.join(arvados.current_task().tmpdir, "upload-output-checkpoint"))
@@ -219,11 +216,10 @@ while not done:
}).execute()
done = True
except KeyboardInterrupt:
- logging.error("run-command: terminating on signal 2")
+ logging.critical("terminating on signal 2")
sys.exit(2)
except Exception as e:
- logging.error("run-command: caught exception:")
- traceback.print_exc(file=sys.stdout)
+ logging.exception("caught exception:")
time.sleep(5)
sys.exit(rcode)
commit cb880a9fa932988892ffc9f95d38166e9dec5458
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Aug 8 16:01:06 2014 -0400
3373: arv-run-pipeline-instance: removed .to_json where objects are supplied to :body_object and will be serialized anyway.
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index ab3702c..0fdf27d 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -244,7 +244,7 @@ class PipelineInstance
def self.create(attributes)
result = $client.execute(:api_method => $arvados.pipeline_instances.create,
:body_object => {
- :pipeline_instance => attributes.to_json
+ :pipeline_instance => attributes
},
:authenticated => false,
:headers => {
@@ -263,7 +263,7 @@ class PipelineInstance
:uuid => @pi[:uuid]
},
:body_object => {
- :pipeline_instance => @attributes_to_update.to_json
+ :pipeline_instance => @attributes_to_update
},
:authenticated => false,
:headers => {
commit 5edfdafe30ea9d74a5a49602d61934b793762f4b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Aug 8 15:55:48 2014 -0400
3373: Switch run-command to use python logging instead of print.
diff --git a/crunch_scripts/run-command b/crunch_scripts/run-command
index d9f575b..2ad08d8 100755
--- a/crunch_scripts/run-command
+++ b/crunch_scripts/run-command
@@ -15,8 +15,10 @@ import copy
import traceback
import pprint
import multiprocessing
+import logging
os.umask(0077)
+logging.basicConfig(format="run-command: %(message)s")
t = arvados.current_task().tmpdir
@@ -129,7 +131,7 @@ try:
if arvados.current_task()['sequence'] == 0:
var = jobp["task.foreach"]
items = get_items(jobp, jobp[var])
- print("run-command: parallelizing on %s with items %s" % (var, items))
+ logging.info("run-command: parallelizing on %s with items %s" % (var, items))
if items != None:
for i in items:
params = copy.copy(jobp)
@@ -154,13 +156,13 @@ try:
stdoutname = subst.do_substitution(p, p["save.stdout"])
stdoutfile = open(stdoutname, "wb")
- print("run-command: {}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
+ logging.info("run-command: {}{}".format(' '.join(cmd), (" > " + stdoutname) if stdoutname != None else ""))
except Exception as e:
- print("run-command: caught exception:")
+ logging.info("run-command: caught exception:")
traceback.print_exc(file=sys.stdout)
- print("run-command: task parameters was:")
- pprint.pprint(p)
+ logging.info("run-command: task parameters was:")
+ pprint.plogging.info(p)
sys.exit(1)
try:
@@ -176,13 +178,13 @@ try:
rcode = sp.wait()
if sig.sig != None:
- print("run-command: terminating on signal %s" % sig.sig)
+ logging.warning("run-command: terminating on signal %s" % sig.sig)
sys.exit(2)
else:
- print("run-command: completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
+ logging.info("run-command: completed with exit code %i (%s)" % (rcode, "success" if rcode == 0 else "failed"))
except Exception as e:
- print("run-command: caught exception:")
+ logging.error("run-command: caught exception:")
traceback.print_exc(file=sys.stdout)
# restore default signal handlers.
@@ -193,11 +195,11 @@ signal.signal(signal.SIGQUIT, signal.SIG_DFL)
for l in links:
os.unlink(l)
-print("run-command: the following output files will be saved to keep:")
+logging.info("run-command: the following output files will be saved to keep:")
subprocess.call(["find", ".", "-type", "f", "-printf", "run-command: %12.12s %h/%f\\n"])
-print("run-command: start writing output to keep")
+logging.info("run-command: start writing output to keep")
done = False
resume_cache = put.ResumeCache(os.path.join(arvados.current_task().tmpdir, "upload-output-checkpoint"))
@@ -217,10 +219,10 @@ while not done:
}).execute()
done = True
except KeyboardInterrupt:
- print("run-command: terminating on signal 2")
+ logging.error("run-command: terminating on signal 2")
sys.exit(2)
except Exception as e:
- print("run-command: caught exception:")
+ logging.error("run-command: caught exception:")
traceback.print_exc(file=sys.stdout)
time.sleep(5)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list