[ARVADOS] updated: 1.3.0-142-gd6cccb3ea
Git user
git at public.curoverse.com
Thu Jan 10 09:06:58 EST 2019
Summary of changes:
.gitignore | 3 +-
sdk/cwl/arvados_cwl/__init__.py | 2 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 7 ++-
sdk/cwl/arvados_cwl/arvjob.py | 10 ++--
sdk/cwl/arvados_cwl/arvtool.py | 2 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 3 +-
sdk/cwl/arvados_cwl/crunch_script.py | 3 +-
sdk/cwl/arvados_cwl/done.py | 6 +-
sdk/cwl/arvados_cwl/executor.py | 10 +++-
sdk/cwl/arvados_cwl/fsaccess.py | 19 ++++---
sdk/cwl/arvados_cwl/http.py | 22 +++++---
sdk/cwl/arvados_cwl/pathmapper.py | 16 ++++--
sdk/cwl/arvados_cwl/perf.py | 1 +
sdk/cwl/arvados_cwl/runner.py | 14 +++--
sdk/cwl/arvados_cwl/task_queue.py | 14 +++--
sdk/cwl/arvados_version.py | 2 +
sdk/cwl/gittaggers.py | 2 +
sdk/cwl/tests/matcher.py | 1 +
sdk/cwl/tests/test_container.py | 2 +
sdk/cwl/tests/test_http.py | 4 +-
sdk/cwl/tests/test_job.py | 9 ++-
sdk/cwl/tests/test_make_output.py | 6 +-
sdk/cwl/tests/test_submit.py | 105 ++++++++++++++++++-----------------
sdk/cwl/tests/wf/check_mem.py | 4 +-
24 files changed, 158 insertions(+), 109 deletions(-)
via d6cccb3ea4e5f076a436d9935e3835d4b620b859 (commit)
from 523dbe6efe87983ad3f1ac93a5dedfa104302650 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d6cccb3ea4e5f076a436d9935e3835d4b620b859
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Thu Jan 10 09:06:42 2019 -0500
13306: Changes to arvados-cwl-runner code after running futurize --stage2
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/.gitignore b/.gitignore
index 517166c3f..db3020ae8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,4 +29,5 @@ services/api/config/arvados-clients.yml
.DS_Store
.vscode
.Rproj.user
-_version.py
\ No newline at end of file
+_version.py
+*.bak
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7e1495283..eb2a0df59 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -271,7 +271,7 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
add_arv_hints()
- for key, val in cwltool.argparser.get_default_args().items():
+ for key, val in list(cwltool.argparser.get_default_args().items()):
if not hasattr(arvargs, key):
setattr(arvargs, key, val)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 6a91d6ff3..d2f04b5dd 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -1,3 +1,6 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -5,7 +8,7 @@
import logging
import json
import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
import time
import datetime
import ciso8601
@@ -136,7 +139,7 @@ class ArvadosContainer(JobBase):
generatemapper = NoFollowPathMapper(self.generatefiles["listing"], "", "",
separateDirs=False)
- sorteditems = sorted(generatemapper.items(), None, key=lambda n: n[1].target)
+ sorteditems = sorted(list(generatemapper.items()), None, key=lambda n: n[1].target)
logger.debug("generatemapper is %s", sorteditems)
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 87d6d1049..bd8ab1137 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -1,3 +1,5 @@
+from past.builtins import basestring
+from builtins import object
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -71,7 +73,7 @@ class ArvadosJob(JobBase):
separateDirs=False)
with Perf(metrics, "createfiles %s" % self.name):
- for f, p in generatemapper.items():
+ for f, p in list(generatemapper.items()):
if p.type == "CreateFile":
with vwd.open(p.target, "w") as n:
n.write(p.resolved.encode("utf-8"))
@@ -85,7 +87,7 @@ class ArvadosJob(JobBase):
trash_at=info["trash_at"],
properties=info["properties"])
- for f, p in generatemapper.items():
+ for f, p in list(generatemapper.items()):
if p.type == "File":
script_parameters["task.vwd"][p.target] = p.resolved
if p.type == "CreateFile":
@@ -243,7 +245,7 @@ class ArvadosJob(JobBase):
api_client=self.arvrunner.api,
keep_client=self.arvrunner.keep_client,
num_retries=self.arvrunner.num_retries)
- log = logc.open(logc.keys()[0])
+ log = logc.open(list(logc.keys())[0])
dirs = {
"tmpdir": "/tmpdir",
"outdir": "/outdir",
@@ -343,7 +345,7 @@ class RunnerJob(Runner):
find_or_create=self.enable_reuse
).execute(num_retries=self.arvrunner.num_retries)
- for k,v in job_spec["script_parameters"].items():
+ for k,v in list(job_spec["script_parameters"].items()):
if v is False or v is None or isinstance(v, dict):
job_spec["script_parameters"][k] = {"value": v}
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index c4e9f44ab..31e6be12b 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -16,7 +16,7 @@ def validate_cluster_target(arvrunner, runtimeContext):
runtimeContext.submit_runner_cluster not in arvrunner.api._rootDesc["remoteHosts"] and
runtimeContext.submit_runner_cluster != arvrunner.api._rootDesc["uuidPrefix"]):
raise WorkflowException("Unknown or invalid cluster id '%s' known remote clusters are %s" % (runtimeContext.submit_runner_cluster,
- ", ".join(arvrunner.api._rootDesc["remoteHosts"].keys())))
+ ", ".join(list(arvrunner.api._rootDesc["remoteHosts"].keys()))))
def set_cluster_target(tool, arvrunner, builder, runtimeContext):
cluster_target_req = None
for field in ("hints", "requirements"):
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index ea167d404..b878c4d9b 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -1,3 +1,4 @@
+from past.builtins import basestring
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -135,7 +136,7 @@ class ArvadosWorkflowStep(WorkflowStep):
runtimeContext = runtimeContext.copy()
runtimeContext.toplevel = True # Preserve behavior for #13365
- builder = make_builder({shortname(k): v for k,v in joborder.items()}, self.hints, self.requirements, runtimeContext)
+ builder = make_builder({shortname(k): v for k,v in list(joborder.items())}, self.hints, self.requirements, runtimeContext)
runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
return super(ArvadosWorkflowStep, self).job(joborder, output_callback, runtimeContext)
diff --git a/sdk/cwl/arvados_cwl/crunch_script.py b/sdk/cwl/arvados_cwl/crunch_script.py
index 61f9cbbe0..fc2d0e876 100644
--- a/sdk/cwl/arvados_cwl/crunch_script.py
+++ b/sdk/cwl/arvados_cwl/crunch_script.py
@@ -1,3 +1,4 @@
+from past.builtins import basestring
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -61,7 +62,7 @@ def run():
if "location" in v:
v["location"] = keeppath(v["location"])
- for k,v in job_order_object.items():
+ for k,v in list(job_order_object.items()):
if isinstance(v, basestring) and arvados.util.keep_locator_pattern.match(v):
job_order_object[k] = {
"class": "File",
diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 6d46e79cb..9c9195a6a 100644
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -63,7 +63,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
containersapi = ("crunch-run.txt" in logcollection)
mergelogs = {}
- for log in logcollection.keys():
+ for log in list(logcollection.keys()):
if not containersapi or log in ("crunch-run.txt", "stdout.txt", "stderr.txt"):
logname = log[:-4]
logt = deque([], maxlen)
@@ -77,7 +77,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
logt.append(l)
if containersapi:
- keys = mergelogs.keys()
+ keys = list(mergelogs.keys())
loglines = []
while True:
earliest = None
@@ -91,7 +91,7 @@ def logtail(logcollection, logfunc, header, maxlen=25):
loglines.append("%s %s %s" % (ts, earliest, msg))
loglines = loglines[-maxlen:]
else:
- loglines = mergelogs.values()[0]
+ loglines = list(mergelogs.values())[0]
logtxt = "\n ".join(l.strip() for l in loglines)
logfunc("%s\n\n %s", header, logtxt)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 31ac4c247..dbe4992cb 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -1,3 +1,7 @@
+from __future__ import division
+from builtins import next
+from builtins import object
+from past.utils import old_div
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -401,7 +405,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
"Option 'dockerOutputDirectory' must be an absolute path.")
if obj.get("class") == "http://commonwl.org/cwltool#Secrets" and self.work_api != "containers":
raise SourceLine(obj, "class", UnsupportedRequirement).makeError("Secrets not supported with --api=jobs")
- for v in obj.itervalues():
+ for v in obj.values():
self.check_features(v)
elif isinstance(obj, list):
for i,v in enumerate(obj):
@@ -424,7 +428,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
keep_client=self.keep_client,
num_retries=self.num_retries)
- for k,v in generatemapper.items():
+ for k,v in list(generatemapper.items()):
if k.startswith("_:"):
if v.type == "Directory":
continue
@@ -608,7 +612,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
visited.add(m.group(1))
estimated_size[0] += int(m.group(2))
visit_class(job_order, ("File", "Directory"), estimate_collection_cache)
- runtimeContext.collection_cache_size = max(((estimated_size[0]*192) / (1024*1024))+1, 256)
+ runtimeContext.collection_cache_size = max((old_div((estimated_size[0]*192), (1024*1024)))+1, 256)
self.collection_cache.set_cap(runtimeContext.collection_cache_size*1024*1024)
logger.info("Using collection cache size %s MiB", runtimeContext.collection_cache_size)
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index 0816ee8fc..708b81818 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -1,3 +1,6 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -5,7 +8,7 @@
import fnmatch
import os
import errno
-import urlparse
+import urllib.parse
import re
import logging
import threading
@@ -48,7 +51,7 @@ class CollectionCache(object):
def cap_cache(self, required):
# ordered dict iterates from oldest to newest
- for pdh, v in self.collections.items():
+ for pdh, v in list(self.collections.items()):
available = self.cap - self.total
if available >= required or len(self.collections) < self.min_entries:
return
@@ -90,7 +93,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
p = sp[0]
if p.startswith("keep:") and arvados.util.keep_locator_pattern.match(p[5:]):
pdh = p[5:]
- return (self.collection_cache.get(pdh), urlparse.unquote(sp[1]) if len(sp) == 2 else None)
+ return (self.collection_cache.get(pdh), urllib.parse.unquote(sp[1]) if len(sp) == 2 else None)
else:
return (None, path)
@@ -188,7 +191,7 @@ class CollectionFsAccess(cwltool.stdfsaccess.StdFsAccess):
raise IOError(errno.ENOENT, "Directory '%s' in '%s' not found" % (rest, collection.portable_data_hash()))
if not isinstance(dir, arvados.collection.RichCollectionBase):
raise IOError(errno.ENOENT, "Path '%s' in '%s' is not a Directory" % (rest, collection.portable_data_hash()))
- return [abspath(l, fn) for l in dir.keys()]
+ return [abspath(l, fn) for l in list(dir.keys())]
else:
return super(CollectionFsAccess, self).listdir(fn)
@@ -243,11 +246,11 @@ class CollectionFetcher(DefaultFetcher):
if not url:
return base_url
- urlsp = urlparse.urlsplit(url)
+ urlsp = urllib.parse.urlsplit(url)
if urlsp.scheme or not base_url:
return url
- basesp = urlparse.urlsplit(base_url)
+ basesp = urllib.parse.urlsplit(base_url)
if basesp.scheme in ("keep", "arvwf"):
if not basesp.path:
raise IOError(errno.EINVAL, "Invalid Keep locator", base_url)
@@ -268,7 +271,7 @@ class CollectionFetcher(DefaultFetcher):
baseparts.pop()
path = "/".join([pdh] + baseparts + urlparts)
- return urlparse.urlunsplit((basesp.scheme, "", path, "", urlsp.fragment))
+ return urllib.parse.urlunsplit((basesp.scheme, "", path, "", urlsp.fragment))
return super(CollectionFetcher, self).urljoin(base_url, url)
@@ -290,7 +293,7 @@ def collectionResolver(api_client, document_loader, uri, num_retries=4):
if pipeline_template_uuid_pattern.match(uri):
pt = api_client.pipeline_templates().get(uuid=uri).execute(num_retries=num_retries)
- return "keep:" + pt["components"].values()[0]["script_parameters"]["cwl:tool"]
+ return "keep:" + list(pt["components"].values())[0]["script_parameters"]["cwl:tool"]
p = uri.split("/")
if arvados.util.keep_locator_pattern.match(p[0]):
diff --git a/sdk/cwl/arvados_cwl/http.py b/sdk/cwl/arvados_cwl/http.py
index 4516de021..9b3649a61 100644
--- a/sdk/cwl/arvados_cwl/http.py
+++ b/sdk/cwl/arvados_cwl/http.py
@@ -1,3 +1,7 @@
+from __future__ import division
+from future import standard_library
+standard_library.install_aliases()
+from past.utils import old_div
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -9,7 +13,7 @@ import datetime
import re
import arvados
import arvados.collection
-import urlparse
+import urllib.parse
import logging
import calendar
@@ -91,13 +95,13 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
if fresh_cache(url, properties, now):
# Do nothing
cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
- return "keep:%s/%s" % (item["portable_data_hash"], cr.keys()[0])
+ return "keep:%s/%s" % (item["portable_data_hash"], list(cr.keys())[0])
if not changed(url, properties, now):
# ETag didn't change, same content, just update headers
api.collections().update(uuid=item["uuid"], body={"collection":{"properties": properties}}).execute()
cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
- return "keep:%s/%s" % (item["portable_data_hash"], cr.keys()[0])
+ return "keep:%s/%s" % (item["portable_data_hash"], list(cr.keys())[0])
properties = {}
req = requests.get(url, stream=True, allow_redirects=True)
@@ -123,7 +127,7 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
else:
name = grp.group(4)
else:
- name = urlparse.urlparse(url).path.split("/")[-1]
+ name = urllib.parse.urlparse(url).path.split("/")[-1]
count = 0
start = time.time()
@@ -134,14 +138,14 @@ def http_to_keep(api, project_uuid, url, utcnow=datetime.datetime.utcnow):
f.write(chunk)
loopnow = time.time()
if (loopnow - checkpoint) > 20:
- bps = (float(count)/float(loopnow - start))
+ bps = (old_div(float(count),float(loopnow - start)))
if cl is not None:
logger.info("%2.1f%% complete, %3.2f MiB/s, %1.0f seconds left",
- float(count * 100) / float(cl),
- bps/(1024*1024),
- (cl-count)/bps)
+ old_div(float(count * 100), float(cl)),
+ old_div(bps,(1024*1024)),
+ old_div((cl-count),bps))
else:
- logger.info("%d downloaded, %3.2f MiB/s", count, bps/(1024*1024))
+ logger.info("%d downloaded, %3.2f MiB/s", count, old_div(bps,(1024*1024)))
checkpoint = loopnow
c.save_new(name="Downloaded from %s" % url, owner_uuid=project_uuid, ensure_unique_name=True)
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 0b2a22788..7353d7239 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -1,3 +1,7 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from past.builtins import basestring
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -6,7 +10,7 @@ import re
import logging
import uuid
import os
-import urllib
+import urllib.request, urllib.parse, urllib.error
import arvados_cwl.util
import arvados.commands.run
@@ -60,7 +64,7 @@ class ArvPathMapper(PathMapper):
src = src[:src.index("#")]
if isinstance(src, basestring) and ArvPathMapper.pdh_dirpath.match(src):
- self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.unquote(src[5:]), srcobj["class"], True)
+ self._pathmap[src] = MapperEnt(src, self.collection_pattern % urllib.parse.unquote(src[5:]), srcobj["class"], True)
debug = logger.isEnabledFor(logging.DEBUG)
@@ -77,7 +81,7 @@ class ArvPathMapper(PathMapper):
if isinstance(st, arvados.commands.run.UploadFile):
uploadfiles.add((src, ab, st))
elif isinstance(st, arvados.commands.run.ArvFile):
- self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.unquote(st.fn[5:]), "File", True)
+ self._pathmap[src] = MapperEnt(st.fn, self.collection_pattern % urllib.parse.unquote(st.fn[5:]), "File", True)
else:
raise WorkflowException("Input file path '%s' is invalid" % st)
elif src.startswith("_:"):
@@ -176,7 +180,7 @@ class ArvPathMapper(PathMapper):
packed=False)
for src, ab, st in uploadfiles:
- self._pathmap[src] = MapperEnt(urllib.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
+ self._pathmap[src] = MapperEnt(urllib.parse.quote(st.fn, "/:+@"), self.collection_pattern % st.fn[5:],
"Directory" if os.path.isdir(ab) else "File", True)
for srcobj in referenced_files:
@@ -228,7 +232,7 @@ class ArvPathMapper(PathMapper):
ab, "File", True)
if srcobj.get("secondaryFiles"):
ab = self.collection_pattern % c.portable_data_hash()
- self._pathmap["_:" + unicode(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
+ self._pathmap["_:" + str(uuid.uuid4())] = MapperEnt("keep:"+c.portable_data_hash(), ab, "Directory", True)
if remap:
for loc, sub in remap:
@@ -301,7 +305,7 @@ class VwdPathMapper(StagingPathMapper):
# with any secondary files.
self.visitlisting(referenced_files, self.stagedir, basedir)
- for path, (ab, tgt, type, staged) in self._pathmap.items():
+ for path, (ab, tgt, type, staged) in list(self._pathmap.items()):
if type in ("File", "Directory") and ab.startswith("keep:"):
self._pathmap[path] = MapperEnt("$(task.keep)/%s" % ab[5:], tgt, type, staged)
diff --git a/sdk/cwl/arvados_cwl/perf.py b/sdk/cwl/arvados_cwl/perf.py
index 39f475fe8..e736d9ef0 100644
--- a/sdk/cwl/arvados_cwl/perf.py
+++ b/sdk/cwl/arvados_cwl/perf.py
@@ -1,3 +1,4 @@
+from builtins import object
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 4b3275fa3..9f6390e30 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -1,16 +1,18 @@
+from future import standard_library
+standard_library.install_aliases()
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
import os
-import urlparse
+import urllib.parse
from functools import partial
import logging
import json
import subprocess32 as subprocess
from collections import namedtuple
-from StringIO import StringIO
+from io import StringIO
from schema_salad.sourceline import SourceLine, cmap
@@ -61,7 +63,7 @@ def find_defaults(d, op):
if "default" in d:
op(d)
else:
- for i in d.itervalues():
+ for i in d.values():
find_defaults(i, op)
def setSecondary(t, fileobj, discovered):
@@ -98,7 +100,7 @@ def upload_dependencies(arvrunner, name, document_loader,
loaded = set()
def loadref(b, u):
joined = document_loader.fetcher.urljoin(b, u)
- defrg, _ = urlparse.urldefrag(joined)
+ defrg, _ = urllib.parse.urldefrag(joined)
if defrg not in loaded:
loaded.add(defrg)
# Use fetch_text to get raw file (before preprocessing).
@@ -232,7 +234,7 @@ def packed_workflow(arvrunner, tool, merged_map):
packed = pack(tool.doc_loader, tool.doc_loader.fetch(tool.tool["id"]),
tool.tool["id"], tool.metadata, rewrite_out=rewrites)
- rewrite_to_orig = {v: k for k,v in rewrites.items()}
+ rewrite_to_orig = {v: k for k,v in list(rewrites.items())}
def visit(v, cur_id):
if isinstance(v, dict):
@@ -314,7 +316,7 @@ def upload_workflow_deps(arvrunner, tool):
discovered_secondaryfiles=discovered_secondaryfiles)
document_loader.idx[deptool["id"]] = deptool
toolmap = {}
- for k,v in pm.items():
+ for k,v in list(pm.items()):
toolmap[k] = v.resolved
merged_map[deptool["id"]] = FileUpdates(toolmap, discovered_secondaryfiles)
diff --git a/sdk/cwl/arvados_cwl/task_queue.py b/sdk/cwl/arvados_cwl/task_queue.py
index 1c233fac0..f7d7c071c 100644
--- a/sdk/cwl/arvados_cwl/task_queue.py
+++ b/sdk/cwl/arvados_cwl/task_queue.py
@@ -1,8 +1,12 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import range
+from builtins import object
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
-import Queue
+import queue
import threading
import logging
@@ -11,13 +15,13 @@ logger = logging.getLogger('arvados.cwl-runner')
class TaskQueue(object):
def __init__(self, lock, thread_count):
self.thread_count = thread_count
- self.task_queue = Queue.Queue(maxsize=self.thread_count)
+ self.task_queue = queue.Queue(maxsize=self.thread_count)
self.task_queue_threads = []
self.lock = lock
self.in_flight = 0
self.error = None
- for r in xrange(0, self.thread_count):
+ for r in range(0, self.thread_count):
t = threading.Thread(target=self.task_queue_func)
self.task_queue_threads.append(t)
t.start()
@@ -51,7 +55,7 @@ class TaskQueue(object):
return
self.task_queue.put(task, block=True, timeout=3)
return
- except Queue.Full:
+ except queue.Full:
pass
finally:
unlock.acquire()
@@ -62,7 +66,7 @@ class TaskQueue(object):
# Drain queue
while not self.task_queue.empty():
self.task_queue.get(True, .1)
- except Queue.Empty:
+ except queue.Empty:
pass
def join(self):
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index d13dd5ec5..bd23cdd0a 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import next
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
diff --git a/sdk/cwl/gittaggers.py b/sdk/cwl/gittaggers.py
index 4dc844847..f32ab8000 100644
--- a/sdk/cwl/gittaggers.py
+++ b/sdk/cwl/gittaggers.py
@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import next
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
diff --git a/sdk/cwl/tests/matcher.py b/sdk/cwl/tests/matcher.py
index 50c0c60b1..674e8fc60 100644
--- a/sdk/cwl/tests/matcher.py
+++ b/sdk/cwl/tests/matcher.py
@@ -1,3 +1,4 @@
+from builtins import object
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index a34d550f6..9ad7300b3 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import object
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
diff --git a/sdk/cwl/tests/test_http.py b/sdk/cwl/tests/test_http.py
index 0c66c39c0..34f2bbb9d 100644
--- a/sdk/cwl/tests/test_http.py
+++ b/sdk/cwl/tests/test_http.py
@@ -1,9 +1,11 @@
+from future import standard_library
+standard_library.install_aliases()
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
import copy
-import cStringIO
+import io
import functools
import hashlib
import json
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 2aaac0ae5..387d12f38 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -1,3 +1,6 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -9,7 +12,7 @@ import mock
import os
import unittest
import copy
-import StringIO
+import io
import arvados
import arvados_cwl
@@ -213,7 +216,7 @@ class TestJob(unittest.TestCase):
runner.num_retries = 0
runner.ignore_docker_for_reuse = False
- reader().open.return_value = StringIO.StringIO(
+ reader().open.return_value = io.StringIO(
"""2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
@@ -286,7 +289,7 @@ class TestJob(unittest.TestCase):
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.num_retries = 0
- reader().open.return_value = StringIO.StringIO(
+ reader().open.return_value = io.StringIO(
"""2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
diff --git a/sdk/cwl/tests/test_make_output.py b/sdk/cwl/tests/test_make_output.py
index baeb4145e..73f9fc90c 100644
--- a/sdk/cwl/tests/test_make_output.py
+++ b/sdk/cwl/tests/test_make_output.py
@@ -1,3 +1,5 @@
+from future import standard_library
+standard_library.install_aliases()
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
@@ -7,7 +9,7 @@ import json
import logging
import mock
import os
-import StringIO
+import io
import unittest
import arvados
@@ -35,7 +37,7 @@ class TestMakeOutput(unittest.TestCase):
final_uuid = final.manifest_locator()
num_retries = runner.num_retries
- cwlout = StringIO.StringIO()
+ cwlout = io.StringIO()
openmock = mock.MagicMock()
final.open.return_value = openmock
openmock.__enter__.return_value = cwlout
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 90dab0147..5d2d0d011 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1,9 +1,12 @@
+from future import standard_library
+standard_library.install_aliases()
+from builtins import object
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
import copy
-import cStringIO
+import io
import functools
import hashlib
import json
@@ -33,7 +36,7 @@ def stubs(func):
@mock.patch("arvados.keep.KeepClient")
@mock.patch("arvados.events.subscribe")
def wrapped(self, events, keep_client1, keep_client2, keepdocker, *args, **kwargs):
- class Stubs:
+ class Stubs(object):
pass
stubs = Stubs()
stubs.events = events
@@ -89,7 +92,7 @@ def stubs(func):
return CollectionExecute(created_collections[uuid])
def collection_getstub(created_collections, uuid):
- for v in created_collections.itervalues():
+ for v in created_collections.values():
if uuid in (v["uuid"], v["portable_data_hash"]):
return CollectionExecute(v)
@@ -318,7 +321,7 @@ class TestSubmit(unittest.TestCase):
return '999999999999999999999999999999d4+99'
arvdock.side_effect = get_image
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=jobs", "--debug",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -361,7 +364,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_no_reuse(self, stubs, tm):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=jobs", "--debug", "--disable-reuse",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -389,7 +392,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_on_error(self, stubs, tm):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=jobs", "--debug", "--on-error=stop",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -408,7 +411,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_runner_ram(self, stubs, tm):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--debug", "--submit-runner-ram=2048",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -427,7 +430,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_invalid_runner_ram(self, stubs, tm):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--debug", "--submit-runner-ram=-2048",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -439,7 +442,7 @@ class TestSubmit(unittest.TestCase):
def test_submit_output_name(self, stubs, tm):
output_name = "test_output_name"
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--debug", "--output-name", output_name,
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -458,7 +461,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_pipeline_name(self, stubs, tm):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--debug", "--name=hello job 123",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -478,7 +481,7 @@ class TestSubmit(unittest.TestCase):
def test_submit_output_tags(self, stubs, tm):
output_tags = "tag0,tag1,tag2"
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--debug", "--output-tags", output_tags,
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
@@ -512,7 +515,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
@@ -544,7 +547,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_no_reuse(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--disable-reuse",
@@ -572,7 +575,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_reuse_disabled_by_workflow(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
@@ -608,7 +611,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_on_error(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--on-error=stop",
@@ -635,7 +638,7 @@ class TestSubmit(unittest.TestCase):
def test_submit_container_output_name(self, stubs):
output_name = "test_output_name"
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--output-name", output_name,
@@ -661,7 +664,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_storage_classes(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--debug", "--submit", "--no-wait", "--api=containers", "--storage-classes=foo",
@@ -728,7 +731,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_output_ttl(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--intermediate-output-ttl", "3600",
@@ -754,7 +757,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_trash_intermediate(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--trash-intermediate",
@@ -782,7 +785,7 @@ class TestSubmit(unittest.TestCase):
def test_submit_container_output_tags(self, stubs):
output_tags = "tag0,tag1,tag2"
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--output-tags", output_tags,
@@ -807,7 +810,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_runner_ram(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-ram=2048",
@@ -829,7 +832,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_file_keepref(self, stubs, tm, collectionReader):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
collectionReader().find.return_value = arvados.arvfile.ArvadosFile(mock.MagicMock(), "blorp.txt")
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
@@ -842,7 +845,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_keepref(self, stubs, tm, reader):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
with open("tests/wf/expect_arvworkflow.cwl") as f:
reader().open().__enter__().read.return_value = f.read()
@@ -904,7 +907,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_jobs_keepref(self, stubs, tm, reader):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
with open("tests/wf/expect_arvworkflow.cwl") as f:
reader().open().__enter__().read.return_value = f.read()
@@ -927,7 +930,7 @@ class TestSubmit(unittest.TestCase):
@mock.patch("time.sleep")
@stubs
def test_submit_arvworkflow(self, stubs, tm):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
with open("tests/wf/expect_arvworkflow.cwl") as f:
stubs.api.workflows().get().execute.return_value = {"definition": f.read(), "name": "a test workflow"}
@@ -1026,7 +1029,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_name(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--name=hello container 123",
@@ -1047,14 +1050,14 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_missing_input(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
"tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
self.assertEqual(exited, 0)
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
"tests/wf/submit_wf.cwl", "tests/submit_test_job_missing.json"],
@@ -1065,7 +1068,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_project(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--project-uuid="+project_uuid,
@@ -1093,7 +1096,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_eval_timeout(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--eval-timeout=60",
@@ -1119,7 +1122,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_collection_cache(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--collection-cache-size=500",
@@ -1147,7 +1150,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_thread_count(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--thread-count=20",
@@ -1173,7 +1176,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_job_runner_image(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=jobs", "--debug", "--submit-runner-image=arvados/jobs:123",
@@ -1193,7 +1196,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_runner_image(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-image=arvados/jobs:123",
@@ -1213,7 +1216,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_priority(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--priority=669",
@@ -1234,7 +1237,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_wf_runner_resources(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
@@ -1315,7 +1318,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_secrets(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug",
@@ -1492,7 +1495,7 @@ class TestSubmit(unittest.TestCase):
"state": "Queued"
}
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
try:
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--submit-request-uuid=zzzzz-xvhdp-yyyyyyyyyyyyyyy",
@@ -1509,7 +1512,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_container_cluster_id(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
try:
exited = arvados_cwl.main(
@@ -1530,7 +1533,7 @@ class TestSubmit(unittest.TestCase):
@stubs
def test_submit_validate_cluster_id(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
stubs.api._rootDesc["remoteHosts"]["zbbbb"] = "123"
exited = arvados_cwl.main(
["--submit", "--no-wait", "--api=containers", "--debug", "--submit-runner-cluster=zcccc",
@@ -1565,7 +1568,7 @@ class TestCreateTemplate(unittest.TestCase):
def test_create(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--create-workflow", "--debug",
@@ -1598,7 +1601,7 @@ class TestCreateTemplate(unittest.TestCase):
def test_create_name(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--create-workflow", "--debug",
@@ -1632,7 +1635,7 @@ class TestCreateTemplate(unittest.TestCase):
def test_update_name(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--update-workflow", self.existing_template_uuid,
@@ -1673,7 +1676,7 @@ class TestCreateWorkflow(unittest.TestCase):
def test_create(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--create-workflow", "--debug",
@@ -1705,7 +1708,7 @@ class TestCreateWorkflow(unittest.TestCase):
def test_create_name(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--create-workflow", "--debug",
@@ -1735,7 +1738,7 @@ class TestCreateWorkflow(unittest.TestCase):
@stubs
def test_incompatible_api(self, stubs):
- capture_stderr = cStringIO.StringIO()
+ capture_stderr = io.StringIO()
logging.getLogger('arvados.cwl-runner').addHandler(
logging.StreamHandler(capture_stderr))
@@ -1752,7 +1755,7 @@ class TestCreateWorkflow(unittest.TestCase):
@stubs
def test_update(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--update-workflow", self.existing_workflow_uuid,
@@ -1777,7 +1780,7 @@ class TestCreateWorkflow(unittest.TestCase):
@stubs
def test_update_name(self, stubs):
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--update-workflow", self.existing_workflow_uuid,
@@ -1804,7 +1807,7 @@ class TestCreateWorkflow(unittest.TestCase):
def test_create_collection_per_tool(self, stubs):
project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
- capture_stdout = cStringIO.StringIO()
+ capture_stdout = io.StringIO()
exited = arvados_cwl.main(
["--create-workflow", "--debug",
@@ -1884,7 +1887,7 @@ class TestTemplateInputs(unittest.TestCase):
exited = arvados_cwl.main(
["--create-template",
"tests/wf/inputs_test.cwl", "tests/order/empty_order.json"],
- cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
+ io.StringIO(), sys.stderr, api_client=stubs.api)
self.assertEqual(exited, 0)
stubs.api.pipeline_templates().create.assert_called_with(
@@ -1895,7 +1898,7 @@ class TestTemplateInputs(unittest.TestCase):
exited = arvados_cwl.main(
["--create-template",
"tests/wf/inputs_test.cwl", "tests/order/inputs_test_order.json"],
- cStringIO.StringIO(), sys.stderr, api_client=stubs.api)
+ io.StringIO(), sys.stderr, api_client=stubs.api)
self.assertEqual(exited, 0)
expect_template = copy.deepcopy(self.expect_template)
diff --git a/sdk/cwl/tests/wf/check_mem.py b/sdk/cwl/tests/wf/check_mem.py
index bad16ed06..88eb2fd51 100644
--- a/sdk/cwl/tests/wf/check_mem.py
+++ b/sdk/cwl/tests/wf/check_mem.py
@@ -3,6 +3,8 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import print_function
+from __future__ import division
+from past.utils import old_div
import arvados
import sys
import os
@@ -10,7 +12,7 @@ import os
if "JOB_UUID" in os.environ:
requested = arvados.api().jobs().get(uuid=os.environ["JOB_UUID"]).execute()["runtime_constraints"]["min_ram_mb_per_node"]
else:
- requested = arvados.api().containers().current().execute()["runtime_constraints"]["ram"]/(1024*1024)
+ requested = old_div(arvados.api().containers().current().execute()["runtime_constraints"]["ram"],(1024*1024))
print("Requested %d expected %d" % (requested, int(sys.argv[1])))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list