[ARVADOS] updated: deac6ece295b9c2049f8c91e1492324a97bf1b01
Git user
git at public.curoverse.com
Mon Dec 5 10:40:49 EST 2016
Summary of changes:
sdk/cwl/arvados_cwl/__init__.py | 5 +-
sdk/cwl/arvados_cwl/fsaccess.py | 32 +++++++++++--
sdk/cwl/tests/test_job.py | 6 +++
sdk/cwl/tests/test_submit.py | 12 +++++
sdk/cwl/tests/test_urljoin.py | 56 ++++++++++++++++++++++
sdk/cwl/tests/wf/expect_packed.cwl | 2 +-
.../wf/{submit_wf.cwl => submit_keepref_wf.cwl} | 13 +++--
7 files changed, 113 insertions(+), 13 deletions(-)
create mode 100644 sdk/cwl/tests/test_urljoin.py
copy sdk/cwl/tests/wf/{submit_wf.cwl => submit_keepref_wf.cwl} (70%)
via deac6ece295b9c2049f8c91e1492324a97bf1b01 (commit)
via ebf7f34bfd6d20273f8a5f8fded07c3f0387b39f (commit)
from 9f0854e394baf712f1fbcf3ebd21af215276c0fb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit deac6ece295b9c2049f8c91e1492324a97bf1b01
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Dec 5 10:40:42 2016 -0500
10576: Add resolver to execute from keep references and arvados workflow
records. Add test for keep references in workflow default input.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index c7c1863..c953b4e 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -29,7 +29,7 @@ from .arvjob import ArvadosJob, RunnerJob, RunnerTemplate
from. runner import Runner, upload_instance
from .arvtool import ArvadosCommandTool
from .arvworkflow import ArvadosWorkflow, upload_workflow
-from .fsaccess import CollectionFsAccess, CollectionFetcher
+from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver
from .perf import Perf
from .pathmapper import FinalOutputPathMapper
from ._version import __version__
@@ -623,4 +623,5 @@ def main(args, stdout, stderr, api_client=None, keep_client=None):
keep_client=keep_client),
fetcher_constructor=partial(CollectionFetcher,
api_client=api_client,
- keep_client=keep_client))
+ keep_client=keep_client),
+ resolver=partial(collectionResolver, api_client))
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 9cb73d3..9201ab6 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -2,9 +2,11 @@ import fnmatch
import os
import errno
import urlparse
+import re
import cwltool.stdfsaccess
from cwltool.pathmapper import abspath
+import cwltool.resolver
import arvados.util
import arvados.collection
@@ -127,17 +129,23 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
class CollectionFetcher(DefaultFetcher):
def __init__(self, cache, session, api_client=None, keep_client=None):
super(CollectionFetcher, self).__init__(cache, session)
+ self.api_client = api_client
self.fsaccess = CollectionFsAccess("", api_client=api_client, keep_client=keep_client)
def fetch_text(self, url):
if url.startswith("keep:"):
with self.fsaccess.open(url) as f:
return f.read()
+ if url.startswith("arv:"):
+ return self.api_client.workflows().get(uuid=url[4:]).execute()["definition"]
return super(CollectionFetcher, self).fetch_text(url)
def check_exists(self, url):
if url.startswith("keep:"):
return self.fsaccess.exists(url)
+ if url.startswith("arv:"):
+ if self.fetch_text(url):
+ return True
return super(CollectionFetcher, self).check_exists(url)
def urljoin(self, base_url, url):
@@ -172,3 +180,20 @@ class CollectionFetcher(DefaultFetcher):
return urlparse.urlunsplit(("keep", "", path, "", urlsp.fragment))
return super(CollectionFetcher, self).urljoin(base_url, url)
+
+workflow_uuid_pattern = re.compile(r'[a-z0-9]{5}-7fd4e-[a-z0-9]{15}')
+
+def collectionResolver(api_client, document_loader, uri):
+ if workflow_uuid_pattern.match(uri):
+ return "arv:%s" % uri
+
+ p = uri.split("/")
+ if arvados.util.keep_locator_pattern.match(p[0]):
+ return "keep:" + uri
+
+ if arvados.util.collection_uuid_pattern.match(p[0]):
+ return "keep:%s%s" % (self.api_client.collections().
+ get(uuid=uri).execute()["portable_data_hash"],
+ uri[len(p[0]):])
+
+ return cwltool.resolver.tool_resolver(document_loader, uri)
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index c8813ad..7dbc9c8 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -164,6 +164,7 @@ class TestJob(unittest.TestCase):
arvjob.builder = mock.MagicMock()
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
+ arvjob.collect_outputs.return_value = {"out": "stuff"}
arvjob.done({
"state": "Complete",
@@ -189,6 +190,8 @@ class TestJob(unittest.TestCase):
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'name': 'Output 9999999 of testjob'})
+ arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
+
@mock.patch("arvados.collection.CollectionReader")
def test_done_use_existing_collection(self, reader):
api = mock.MagicMock()
@@ -211,6 +214,7 @@ class TestJob(unittest.TestCase):
arvjob.builder = mock.MagicMock()
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
+ arvjob.collect_outputs.return_value = {"out": "stuff"}
arvjob.done({
"state": "Complete",
@@ -228,6 +232,8 @@ class TestJob(unittest.TestCase):
self.assertFalse(api.collections().create.called)
+ arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
+
class TestWorkflow(unittest.TestCase):
# The test passes no builder.resources
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index dea04b3..70397fa 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -489,6 +489,18 @@ class TestSubmit(unittest.TestCase):
self.assertEqual(capture_stdout.getvalue(),
stubs.expect_container_request_uuid + '\n')
+ @mock.patch("arvados.collection.CollectionReader")
+ @mock.patch("time.sleep")
+ @stubs
+ def test_submit_keepref(self, stubs, tm, collectionReader):
+ capture_stdout = cStringIO.StringIO()
+ exited = arvados_cwl.main(
+ ["--submit", "--no-wait", "--api=containers", "--debug",
+ "tests/wf/submit_keepref_wf.cwl"],
+ capture_stdout, sys.stderr, api_client=stubs.api)
+ self.assertEqual(exited, 0)
+
+
@mock.patch("arvados.commands.keepdocker.find_one_image_hash")
@mock.patch("cwltool.docker.get_image")
@mock.patch("arvados.api")
diff --git a/sdk/cwl/tests/test_urljoin.py b/sdk/cwl/tests/test_urljoin.py
index eaede53..b9c8cea 100644
--- a/sdk/cwl/tests/test_urljoin.py
+++ b/sdk/cwl/tests/test_urljoin.py
@@ -51,3 +51,6 @@ class TestUrljoin(unittest.TestCase):
self.assertEquals("keep:99999999999999999999999999999991+99/dir/wh.py",
cf.urljoin("keep:99999999999999999999999999999991+99/dir/", "wh.py"))
+
+ def test_resolver(self):
+ pass
diff --git a/sdk/cwl/tests/wf/expect_packed.cwl b/sdk/cwl/tests/wf/expect_packed.cwl
index 25d02b2..1622f48 100644
--- a/sdk/cwl/tests/wf/expect_packed.cwl
+++ b/sdk/cwl/tests/wf/expect_packed.cwl
@@ -9,7 +9,7 @@ $graph:
type: File
outputs: []
requirements:
- - {class: DockerRequirement, dockerImageId: 'debian:8', dockerPull: 'debian:8'}
+ - {class: DockerRequirement, dockerPull: 'debian:8'}
- class: Workflow
id: '#main'
inputs:
diff --git a/sdk/cwl/tests/wf/submit_keepref_wf.cwl b/sdk/cwl/tests/wf/submit_keepref_wf.cwl
new file mode 100644
index 0000000..f07714e
--- /dev/null
+++ b/sdk/cwl/tests/wf/submit_keepref_wf.cwl
@@ -0,0 +1,20 @@
+# Test case for arvados-cwl-runner
+#
+# Used to test whether scanning a workflow file for dependencies
+# (e.g. submit_tool.cwl) and uploading to Keep works as intended.
+
+class: Workflow
+cwlVersion: v1.0
+inputs:
+ x:
+ type: File
+ default:
+ class: File
+ location: keep:99999999999999999999999999999994+99/blorp.txt
+outputs: []
+steps:
+ step1:
+ in:
+ x: x
+ out: []
+ run: ../tool/submit_tool.cwl
commit ebf7f34bfd6d20273f8a5f8fded07c3f0387b39f
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Dec 2 17:17:25 2016 -0500
10576: Tests & fixes for keep ref url join.
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 9d3fe1c..9cb73d3 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -145,7 +145,7 @@ class CollectionFetcher(DefaultFetcher):
return base_url
urlsp = urlparse.urlsplit(url)
- if urlsp.scheme:
+ if urlsp.scheme or not base_url:
return url
basesp = urlparse.urlsplit(base_url)
@@ -154,7 +154,7 @@ class CollectionFetcher(DefaultFetcher):
raise IOError(errno.EINVAL, "Invalid Keep locator", base_url)
baseparts = basesp.path.split("/")
- urlparts = urlsp.path.split("/")
+ urlparts = urlsp.path.split("/") if urlsp.path else []
pdh = baseparts.pop(0)
@@ -163,8 +163,9 @@ class CollectionFetcher(DefaultFetcher):
if urlsp.path.startswith("/"):
baseparts = []
+ urlparts.pop(0)
- if baseparts and urlparts:
+ if baseparts and urlsp.path:
baseparts.pop()
path = "/".join([pdh] + baseparts + urlparts)
diff --git a/sdk/cwl/tests/test_urljoin.py b/sdk/cwl/tests/test_urljoin.py
new file mode 100644
index 0000000..eaede53
--- /dev/null
+++ b/sdk/cwl/tests/test_urljoin.py
@@ -0,0 +1,53 @@
+import functools
+import mock
+import sys
+import unittest
+import json
+import logging
+import os
+
+import arvados
+import arvados.keep
+import arvados.collection
+import arvados_cwl
+
+from arvados_cwl.fsaccess import CollectionFetcher
+
+class TestUrljoin(unittest.TestCase):
+ def test_urljoin(self):
+ """Test path joining for keep references."""
+
+ cf = CollectionFetcher({}, None)
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/hw.py",
+ cf.urljoin("keep:99999999999999999999999999999991+99", "hw.py"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/hw.py",
+ cf.urljoin("keep:99999999999999999999999999999991+99/", "hw.py"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/hw.py#main",
+ cf.urljoin("keep:99999999999999999999999999999991+99", "hw.py#main"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/hw.py#main",
+ cf.urljoin("keep:99999999999999999999999999999991+99/hw.py", "#main"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/dir/hw.py#main",
+ cf.urljoin("keep:99999999999999999999999999999991+99/dir/hw.py", "#main"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/dir/wh.py",
+ cf.urljoin("keep:99999999999999999999999999999991+99/dir/hw.py", "wh.py"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/wh.py",
+ cf.urljoin("keep:99999999999999999999999999999991+99/dir/hw.py", "/wh.py"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/wh.py#main",
+ cf.urljoin("keep:99999999999999999999999999999991+99/dir/hw.py", "/wh.py#main"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/wh.py",
+ cf.urljoin("keep:99999999999999999999999999999991+99/hw.py#main", "wh.py"))
+
+ self.assertEquals("keep:99999999999999999999999999999992+99",
+ cf.urljoin("keep:99999999999999999999999999999991+99", "keep:99999999999999999999999999999992+99"))
+
+ self.assertEquals("keep:99999999999999999999999999999991+99/dir/wh.py",
+ cf.urljoin("keep:99999999999999999999999999999991+99/dir/", "wh.py"))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list