[ARVADOS] updated: be4dfebdc550a1d8ce663244afd7481995938712
git at public.curoverse.com
git at public.curoverse.com
Mon Apr 6 15:56:02 EDT 2015
Summary of changes:
sdk/python/arvados/commands/cwl_job.py | 41 +++++++++++++++++++++++------
sdk/python/tests/test_cwl_job.py | 15 +++++++----
services/api/config/application.default.yml | 2 +-
3 files changed, 44 insertions(+), 14 deletions(-)
via be4dfebdc550a1d8ce663244afd7481995938712 (commit)
from 39fc7bc4040c014140e56dc277a72777e0799ca7 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit be4dfebdc550a1d8ce663244afd7481995938712
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Apr 6 15:59:47 2015 -0400
4685: First integration test case passes!
diff --git a/sdk/python/arvados/commands/cwl_job.py b/sdk/python/arvados/commands/cwl_job.py
index 28894de..dd136af 100644
--- a/sdk/python/arvados/commands/cwl_job.py
+++ b/sdk/python/arvados/commands/cwl_job.py
@@ -208,8 +208,11 @@ class TaskEvents(object):
def cancel_tasks(self):
for slot in self.slots:
if self.slots[slot]["task"] and self.slots[slot]["task"].get("__subprocess"):
- self.slots[slot]["task"]["__subprocess"].terminate()
- self.slots[slot]["task"]["__subprocess"].wait()
+ try:
+ self.slots[slot]["task"]["__subprocess"].terminate()
+ self.slots[slot]["task"]["__subprocess"].wait()
+ except OSError:
+ pass
def on_event(self, ev):
if ev.get('event_type') == "update" and ev['object_kind'] == "arvados#job":
@@ -225,7 +228,20 @@ class TaskEvents(object):
if ev["properties"]["new_attributes"].get("success") is not None:
self.finish_task(ev["properties"]["new_attributes"])
-def run_executive(resources, api, job, api_config):
+class JobMonitor(threading.Thread):
+ def __init__(self, sub, api_config, job):
+ super(JobMonitor, self).__init__()
+ self.sub = sub
+ self.api_config = api_config
+ self.job = job
+
+ def run(self):
+ self.sub.wait()
+ api = api_from_config("v1", self.api_config)
+ api.jobs().update(uuid=self.job["uuid"], body={"state":"Complete" if self.sub.returncode == 0 else "Failed"}).execute()
+
+
+def run_executive(resources, job, api_config):
execution_script = Template(script_header + """
cd $tmpdir
git init
@@ -250,6 +266,7 @@ docker run \
tmpdir = "/tmp/%s-%i" % (job["uuid"], random.randint(1, 100000))
+ api = api_from_config("v1", api_config)
docker_hash = arvados.commands.keepdocker.image_hash_in_collection(arvados.CollectionReader(job["docker_image_locator"], api_client=api))
repo = job["repository"]
@@ -274,15 +291,23 @@ docker run \
if resources["have_slurm"]:
pass
else:
- return subprocess.Popen(ex, shell=True)
+ sub = subprocess.Popen(ex, shell=True)
+ JobMonitor(sub, api_config, job).start()
+ return sub
+
class SigHandler(object):
- def __init__(self, ts):
+ def __init__(self, sub, ts):
+ self.sub = sub
self.ts = ts
def send_signal(self, signum, fram):
- self.ts.ws.close()
+ try:
+ self.sub.terminate()
+ except OSError:
+ pass
self.ts.cancel_tasks()
+ self.ts.ws.close()
def main(arguments=None):
@@ -327,10 +352,10 @@ def main(arguments=None):
if job_uuid:
ts = TaskEvents(api_config, resources, job_uuid)
- run_executive(resources, api, job, api_config)
+ sub = run_executive(resources, job, api_config)
# Set up signal handling
- sig = SigHandler(ts)
+ sig = SigHandler(sub, ts)
# Forward terminate signals to the subprocesses.
signal.signal(signal.SIGINT, sig.send_signal)
diff --git a/sdk/python/tests/test_cwl_job.py b/sdk/python/tests/test_cwl_job.py
index 365415b..e45bee5 100644
--- a/sdk/python/tests/test_cwl_job.py
+++ b/sdk/python/tests/test_cwl_job.py
@@ -29,10 +29,12 @@ class CwlJobTestCase(run_test_server.TestCaseWithServers):
os.mkdir(os.path.join(run_test_server.ARVADOS_DIR, "services/api/tmp/git/testrepo"))
os.chdir(os.path.join(run_test_server.ARVADOS_DIR, "services/api/tmp/git/testrepo"))
subprocess.check_call(["git", "init"])
- with open("foo.cwl", "w") as f:
- f.write("foo")
- os.chmod("foo.cwl", stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH)
- subprocess.check_call(["git", "add", "foo.cwl"])
+ with open("foo", "w") as f:
+ f.write("""#!/bin/sh
+ echo foo
+ """)
+ os.chmod("foo", stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH)
+ subprocess.check_call(["git", "add", "foo"])
subprocess.check_call(["git", "commit", "-mTest"])
def test_parse_sinfo(self):
@@ -61,7 +63,7 @@ class CwlJobTestCase(run_test_server.TestCaseWithServers):
def test_run_job(self):
job = CwlJobTestCase.api_client.jobs().create(body={"job": {
- "script": "foo.cwl",
+ "script": "foo",
"script_version": "master",
"script_parameters": { },
"repository": "testrepo",
@@ -71,3 +73,6 @@ class CwlJobTestCase(run_test_server.TestCaseWithServers):
} } }).execute()
cwl_job.main(["--job", job["uuid"],
"--job-api-token", os.environ["ARVADOS_API_TOKEN"]])
+
+ job2 = CwlJobTestCase.api_client.jobs().get(uuid=job["uuid"]).execute()
+ self.assertEqual(job2["state"], "Complete")
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index d6a58f9..53b787b 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -46,7 +46,7 @@ test:
user_profile_notification_address: arvados at example.com
workbench_address: https://localhost:3001/
git_repositories_dir: <%= "#{Dir.pwd}/tmp/git" %>
- git_host: http://localhost:3005
+ git_http_base: http://localhost:3005
common:
# The prefix used for all database identifiers to identify the record as
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list