[ARVADOS] created: 2.1.0-1349-g2bf1d2d55
Git user
git at public.arvados.org
Wed Sep 15 19:34:18 UTC 2021
at 2bf1d2d554c33de560d6d1232c927010ac06e243 (commit)
commit 2bf1d2d554c33de560d6d1232c927010ac06e243
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Sep 15 15:33:50 2021 -0400
18002: Fixes for cwltool update
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 1029ca0e0..1e79566f4 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -57,6 +57,12 @@ class ArvadosContainer(JobBase):
def update_pipeline_component(self, r):
pass
+ def _required_env(self):
+ env = {}
+ env["HOME"] = self.outdir
+ env["TMPDIR"] = self.tmpdir
+ return env
+
def run(self, runtimeContext):
# ArvadosCommandTool subclasses from cwltool.CommandLineTool,
# which calls makeJobRunner() to get a new ArvadosContainer
@@ -234,8 +240,6 @@ class ArvadosContainer(JobBase):
"path": "%s/%s" % (self.outdir, self.stdout)}
(docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
- if not docker_req:
- docker_req = {"dockerImageId": "arvados/jobs:"+__version__}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index 13664a8df..83648f46a 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -6,6 +6,7 @@ from cwltool.command_line_tool import CommandLineTool, ExpressionTool
from .arvcontainer import ArvadosContainer
from .pathmapper import ArvPathMapper
from .runner import make_builder
+from ._version import __version__
from functools import partial
from schema_salad.sourceline import SourceLine
from cwltool.errors import WorkflowException
@@ -57,6 +58,12 @@ class ArvadosCommandTool(CommandLineTool):
def __init__(self, arvrunner, toolpath_object, loadingContext):
super(ArvadosCommandTool, self).__init__(toolpath_object, loadingContext)
+
+ (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
+ if not docker_req:
+ self.hints.append({"class": "DockerRequirement",
+ "dockerImageId": "arvados/jobs:"+__version__})
+
self.arvrunner = arvrunner
def make_job_runner(self, runtimeContext):
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 77d4027cc..1e04dd577 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -41,3 +41,15 @@ class ArvRuntimeContext(RuntimeContext):
if self.submit_request_uuid:
self.submit_runner_cluster = self.submit_request_uuid[0:5]
+
+ def get_outdir(self) -> str:
+ """Return self.outdir or create one with self.tmp_outdir_prefix."""
+ return self.outdir
+
+ def get_tmpdir(self) -> str:
+ """Return self.tmpdir or create one with self.tmpdir_prefix."""
+ return self.tmpdir
+
+ def create_tmpdir(self) -> str:
+ """Return self.tmpdir or create one with self.tmpdir_prefix."""
+ return self.tmpdir
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 28eb50603..d6cc4173f 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -40,7 +40,7 @@ setup(name='arvados-cwl-runner',
# build.
install_requires=[
'cwltool==3.1.20210816212154',
- 'schema-salad==8.1.20210627200047',
+ 'schema-salad==8.2.20210902094147',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0',
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 91283b0b6..8a380ff80 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -123,7 +123,7 @@ class TestContainer(unittest.TestCase):
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}],
"id": "#",
- "class": "CommandLineTool"
+ "class": "org.w3id.cwl.cwl.CommandLineTool"
})
loadingContext, runtimeContext = self.helper(runner, enable_reuse)
@@ -206,7 +206,7 @@ class TestContainer(unittest.TestCase):
}],
"baseCommand": "ls",
"id": "#",
- "class": "CommandLineTool"
+ "class": "org.w3id.cwl.cwl.CommandLineTool"
})
loadingContext, runtimeContext = self.helper(runner)
@@ -314,7 +314,7 @@ class TestContainer(unittest.TestCase):
}],
"baseCommand": "ls",
"id": "#",
- "class": "CommandLineTool"
+ "class": "org.w3id.cwl.cwl.CommandLineTool"
})
loadingContext, runtimeContext = self.helper(runner)
@@ -414,7 +414,7 @@ class TestContainer(unittest.TestCase):
"stdin": "/keep/99999999999999999999999999999996+99/file.txt",
"arguments": [{"valueFrom": "$(runtime.outdir)"}],
"id": "#",
- "class": "CommandLineTool"
+ "class": "org.w3id.cwl.cwl.CommandLineTool"
})
loadingContext, runtimeContext = self.helper(runner)
@@ -639,7 +639,7 @@ class TestContainer(unittest.TestCase):
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}],
"id": "#",
- "class": "CommandLineTool"
+ "class": "org.w3id.cwl.cwl.CommandLineTool"
})
loadingContext, runtimeContext = self.helper(runner)
@@ -720,7 +720,7 @@ class TestContainer(unittest.TestCase):
document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.1")
tool = cmap({"arguments": ["md5sum", "example.conf"],
- "class": "CommandLineTool",
+ "class": "org.w3id.cwl.cwl.CommandLineTool",
"hints": [
{
"class": "http://commonwl.org/cwltool#Secrets",
@@ -819,7 +819,7 @@ class TestContainer(unittest.TestCase):
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}],
"id": "#",
- "class": "CommandLineTool",
+ "class": "org.w3id.cwl.cwl.CommandLineTool",
"hints": [
{
"class": "ToolTimeLimit",
@@ -862,7 +862,7 @@ class TestContainer(unittest.TestCase):
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}],
"id": "#",
- "class": "CommandLineTool",
+ "class": "org.w3id.cwl.cwl.CommandLineTool",
"hints": [
{
"class": "http://arvados.org/cwl#OutputStorageClass",
@@ -936,7 +936,7 @@ class TestContainer(unittest.TestCase):
"baseCommand": "ls",
"arguments": [{"valueFrom": "$(runtime.outdir)"}],
"id": "#",
- "class": "CommandLineTool",
+ "class": "org.w3id.cwl.cwl.CommandLineTool",
"hints": [
{
"class": "http://arvados.org/cwl#ProcessProperties",
commit 58e740a6ea84242b423163ae66f0dc9399ddd788
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Sep 15 13:41:02 2021 -0400
18002: Updating Python dependencies WIP
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 34fe0c15d..28eb50603 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -39,8 +39,8 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20210319143721',
- 'schema-salad==7.1.20210611090601',
+ 'cwltool==3.1.20210816212154',
+ 'schema-salad==8.1.20210627200047',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0',
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 9dfe0436d..bc0785183 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -720,11 +720,11 @@ class KeepClient(object):
result = service.last_result()
if not success:
- if result.get('status_code', None):
+ if result.get('status_code'):
_logger.debug("Request fail: PUT %s => %s %s",
self.data_hash,
- result['status_code'],
- result['body'])
+ result.get('status_code'),
+ result.get('body'))
raise self.TaskFailed()
_logger.debug("KeepWriterThread %s succeeded %s+%i %s",
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index ef95674a6..40e6e18ab 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -48,10 +48,10 @@ setup(name='arvados-python-client',
install_requires=[
'ciso8601 >=2.0.0',
'future',
- 'google-api-python-client >=1.6.2, <1.7',
+ 'google-api-python-client >=1.6.2, <2',
'httplib2 >=0.9.2',
'pycurl >=7.19.5.1',
- 'ruamel.yaml >=0.15.54, <=0.16.5',
+ 'ruamel.yaml >=0.15.54, <=0.17.11',
'setuptools',
'ws4py >=0.4.2',
],
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index b1c42fd2b..b2160e549 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -1300,6 +1300,8 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock):
def last_result(self):
if self.will_succeed:
return self._result
+ else:
+ return {"status_code": 500, "body": "didn't succeed"}
def finished(self):
return False
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list