[ARVADOS] updated: 1.1.4-555-g9b16a843a
Git user
git at public.curoverse.com
Mon Jul 9 09:11:55 EDT 2018
Summary of changes:
apps/workbench/config/application.default.yml | 1 +
apps/workbench/test/test_helper.rb | 30 +--
build/build.list | 1 +
build/run-build-packages.sh | 4 +
build/run-library.sh | 5 +-
build/run-tests.sh | 26 ++-
cmd/arvados-client/cmd.go | 25 +--
.../arvados-server/arvados-controller.service | 7 +-
cmd/arvados-server/cmd.go | 27 +++
doc/_config.yml | 8 +
.../_container_scheduling_parameters.liquid | 2 +
doc/admin/activation.html.textile.liquid | 229 +++++++++++++++++++
doc/admin/spot-instances.html.textile.liquid | 78 +++++++
doc/admin/storage-classes.html.textile.liquid | 47 ++++
doc/api/tokens.html.textile.liquid | 4 +
.../arvados-on-kubernetes-GKE.html.textile.liquid | 62 +++++
...ados-on-kubernetes-minikube.html.textile.liquid | 34 +++
.../arvados-on-kubernetes.html.textile.liquid | 133 +++++++++++
...onfigure-azure-blob-storage.html.textile.liquid | 86 +++++--
.../configure-fs-storage.html.textile.liquid | 56 +++++
...configure-s3-object-storage.html.textile.liquid | 112 +++++++++
doc/install/index.html.textile.liquid | 20 +-
doc/install/install-keepstore.html.textile.liquid | 250 ++++++++++++---------
.../topics/storage-classes.html.textile.liquid | 53 +++++
lib/cmd/cmd.go | 27 ++-
lib/cmd/cmd_test.go | 12 +-
lib/controller/cmd.go | 17 ++
lib/controller/handler.go | 141 ++++++++++++
lib/controller/handler_test.go | 122 ++++++++++
lib/dispatchcloud/node_size.go | 34 +--
lib/dispatchcloud/node_size_test.go | 62 ++---
lib/service/cmd.go | 120 ++++++++++
sdk/R/R/zzz.R | 10 +
sdk/R/README.Rmd | 2 +
sdk/cwl/arvados_cwl/__init__.py | 234 ++++++++++---------
sdk/cwl/arvados_cwl/arvcontainer.py | 97 ++++----
sdk/cwl/arvados_cwl/arvjob.py | 76 +++----
sdk/cwl/arvados_cwl/arvtool.py | 73 +++---
sdk/cwl/arvados_cwl/arvworkflow.py | 54 ++---
sdk/cwl/arvados_cwl/context.py | 33 +++
sdk/cwl/arvados_cwl/crunch_script.py | 19 +-
sdk/cwl/arvados_cwl/done.py | 10 +-
sdk/cwl/arvados_cwl/pathmapper.py | 42 +---
sdk/cwl/arvados_cwl/runner.py | 12 +-
sdk/cwl/arvados_cwl/util.py | 28 +++
sdk/cwl/arvados_version.py | 17 +-
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_container.py | 183 +++++++++------
sdk/cwl/tests/test_job.py | 157 +++++++------
sdk/cwl/tests/test_make_output.py | 4 +-
sdk/cwl/tests/test_pathmapper.py | 24 --
sdk/cwl/tests/test_submit.py | 75 +++++++
sdk/cwl/tests/test_util.py | 44 ++++
sdk/go/arvados/byte_size.go | 91 ++++++++
sdk/go/arvados/byte_size_test.go | 70 ++++++
sdk/go/arvados/config.go | 115 +++++++---
sdk/go/arvados/config_test.go | 37 +++
sdk/go/arvados/container.go | 3 +-
sdk/go/health/aggregator.go | 16 +-
sdk/go/health/aggregator_test.go | 14 +-
sdk/go/httpserver/error.go | 21 ++
sdk/go/httpserver/logger.go | 30 ++-
sdk/go/httpserver/logger_test.go | 14 +-
sdk/python/arvados/keep.py | 19 +-
sdk/python/tests/nginx.conf | 38 +++-
sdk/python/tests/run_test_server.py | 85 ++++++-
sdk/python/tests/test_keep_client.py | 25 +++
.../api/app/models/api_client_authorization.rb | 3 +-
services/api/app/models/container_request.rb | 35 ++-
services/api/app/models/user.rb | 3 +-
services/api/config/application.default.yml | 22 +-
services/api/lib/crunch_dispatch.rb | 8 +-
services/api/lib/update_priority.rb | 25 ++-
services/api/lib/whitelist_update.rb | 2 +-
services/api/test/integration/remote_user_test.rb | 33 +++
services/api/test/unit/container_request_test.rb | 49 +++-
services/api/test/unit/update_priority_test.rb | 30 +++
services/api/test/unit/user_test.rb | 4 +-
.../crunch-dispatch-slurm_test.go | 16 +-
services/crunch-run/crunchrun.go | 9 +
services/crunch-run/crunchrun_test.go | 22 +-
services/health/main.go | 2 +-
services/keep-web/server.go | 2 +-
services/keepproxy/keepproxy.go | 2 +-
services/keepstore/config.go | 3 +-
services/keepstore/handlers.go | 2 +-
.../arvnodeman/computenode/driver/ec2.py | 2 +-
services/nodemanager/arvnodeman/config.py | 6 +-
services/nodemanager/arvnodeman/jobqueue.py | 4 +-
services/nodemanager/doc/ec2.example.cfg | 12 +
.../tests/test_computenode_driver_ec2.py | 4 +-
services/nodemanager/tests/test_config.py | 14 +-
services/nodemanager/tests/testutil.py | 4 +-
93 files changed, 3046 insertions(+), 881 deletions(-)
copy services/health/arvados-health.service => cmd/arvados-server/arvados-controller.service (83%)
create mode 100644 cmd/arvados-server/cmd.go
create mode 100644 doc/admin/activation.html.textile.liquid
create mode 100644 doc/admin/spot-instances.html.textile.liquid
create mode 100644 doc/admin/storage-classes.html.textile.liquid
create mode 100644 doc/install/arvados-on-kubernetes-GKE.html.textile.liquid
create mode 100644 doc/install/arvados-on-kubernetes-minikube.html.textile.liquid
create mode 100644 doc/install/arvados-on-kubernetes.html.textile.liquid
create mode 100644 doc/install/configure-fs-storage.html.textile.liquid
create mode 100644 doc/install/configure-s3-object-storage.html.textile.liquid
create mode 100644 doc/user/topics/storage-classes.html.textile.liquid
create mode 100644 lib/controller/cmd.go
create mode 100644 lib/controller/handler.go
create mode 100644 lib/controller/handler_test.go
create mode 100644 lib/service/cmd.go
create mode 100644 sdk/R/R/zzz.R
create mode 100644 sdk/cwl/arvados_cwl/context.py
create mode 100644 sdk/cwl/arvados_cwl/util.py
create mode 100644 sdk/cwl/tests/test_util.py
create mode 100644 sdk/go/arvados/byte_size.go
create mode 100644 sdk/go/arvados/byte_size_test.go
create mode 100644 sdk/go/arvados/config_test.go
create mode 100644 sdk/go/httpserver/error.go
create mode 100644 services/api/test/unit/update_priority_test.rb
via 9b16a843a641159867ea390c4c2384b320183b15 (commit)
via 5c4d9d38dcee73a7ffb6221c80f707c3924da64f (commit)
via ee4c01999aff1ebc1e2dc338a70d1d7b812c633c (commit)
via a12e4d36901ab3b3027f1168f1ea06b26122829e (commit)
via c1ab715afeb9c095efed7e6064ce10021c067c35 (commit)
via 0695b41ae52b87adcdf49c6411f0da353d7213a9 (commit)
via 816764a283c2cbf2d41b4582113065922b99bd52 (commit)
via 8f0815985dfb74fc1c7468426359ac3fbbd8c12d (commit)
via 1f9519fba9a34f2a596c683ed6395b2e291935b7 (commit)
via c425f01b6dd63dfe5d99a19f57c8155c2f0fb195 (commit)
via 3813e892be7f3136e5b14580e7ee64d6c786bf8e (commit)
via 4a2dc82a1acce855151928abe0030e1dd7dbf728 (commit)
via 9a9930cebb7f36131490d0f144e23f5066e7f01d (commit)
via d128fe6bc8da01fe8e7829db988819dd1c159298 (commit)
via 76a2c8a1eb1771b5255e8582b7e9823d072e8ef4 (commit)
via 62f6a306eb965cee6bd814b194bf5ce3bb910116 (commit)
via 11ab2e23511c7e8962e0110c3aad44b74fea2dbd (commit)
via 7d4123813b42d8f6478e239895e62f6f600a1c6f (commit)
via 596884ea30c551d7c2699b2cf3fdea2cede3101a (commit)
via bd059862d1dd7e68642dd365f0a536621ff6735f (commit)
via 0765fb8b19dd7b76ec5d2d05edda3f4fd4347194 (commit)
via 286f7fb2dbdcd860275a24f3480e938b5bd414ec (commit)
via aeaee020e585b69f3fea9381deb0c595eebf7c9a (commit)
via 32d954c32f20a982bec4b3615bfd7cfca449c280 (commit)
via 57fd9fa6bf0ee3062d7d38aceb7e97543791d241 (commit)
via 9df6d2c2152b5b1968649c970664c4f69d9e92e8 (commit)
via edf5021cddc7646c920d3c938a2a3b34d6d8c0a5 (commit)
via b004059a3372af7aa347ef7b8b73522d489ffec2 (commit)
via c90adfb66544a4e6ff34b13268fe3e05847144c2 (commit)
via e9e440847c574d1152b128cfd508cf473f49121b (commit)
via a4ea40603c5f9c71ce7051fe13e3b493e0829128 (commit)
via eed6ea7af40d0d7ee0a11bcb484113bd606f51ed (commit)
via ab7bb79f2da9f44eae4b1fd910680ca472b9c5a7 (commit)
via ba908bbe90213f1d50422611052c92280eae0dcd (commit)
via 945258f22c3c02d12e0dada049b8c37fa5139af2 (commit)
via a31c1accfc353ed6bb3c9982ee694f98f6c965ec (commit)
via 42d62e3d140360a179293b8995aaf535e8c4c30c (commit)
via 059d04053d1a7ac62c796ad5757191b9c5dd5aae (commit)
via 93661ec76c6c1affcde86563dccda5843a879239 (commit)
via 7f9465d37fcc3277128d3f4a611b778e24e530a5 (commit)
via ebb7681e5cf4bc2825e8786ecda895e219158703 (commit)
via 6df0096f2b246959bb9449a0940e9c539bcc9100 (commit)
via 4fdb030e1ad940215b75b4b8a8f7aca5635211e3 (commit)
via 78a9021dc8baa5603feb512c45c1fbca5f3accfc (commit)
via b7acf6db5beab0b082c863fe5a9692d68152a132 (commit)
via 0bcbbb13f9e278347e500fa344ee600891a9bcb8 (commit)
via 0e94528878341faa941247eb3d990230c941868c (commit)
via 645e9829bec9147f52141b17b439f9b561ed3445 (commit)
via 5d3a682b0fc41f4152051b43b8e5311b9a130f13 (commit)
via ec88f99b1eeb811159b3aea9a6f810dc06e9417b (commit)
via bd14cd71274109789ea076b1b4219bdf3beef657 (commit)
via 75f835af6ac337c490a76ac60453242719f6ceca (commit)
via 905a7305413f57e3188a66fe55d35cecc484b1f2 (commit)
via c7414891876cd6d0d02abe71c7376b8104bfbc32 (commit)
via a200bee21cc9632428c6ed9e37dd3a4f90682f9d (commit)
via 731ff645f6de779a324999f3b78bf90469dbb4fb (commit)
via 9fa635c9dc288317f19471291accecf8690f5718 (commit)
via f9e94997cb5c2166d8b71874f263544cfc2fe5ba (commit)
via e677e6b27fa35ba2f81a8a35b00fe860926a69ad (commit)
via a7cd9cfe43468acd6cbc1b674241fb3c7ffcf0cf (commit)
via fa4fc4f452c5b29427512758f4bd14df2fcc3c42 (commit)
via cc556a86ae6a83ba4cf5b7125fbabe704207ae60 (commit)
via 911abf131639401b2ccfd23a6bc3ec983f7ed8f4 (commit)
via 41e15e626cc301d867c48ebb1f945bbec6b7d555 (commit)
via af1978730ccd9bd3584fc1cdace37809eff63a13 (commit)
via 43e349a8916b4e952dfafee2f88d4d2952be12c1 (commit)
via a8bfbac314335eb0bff3f4ff5e856d3c327de31d (commit)
via 73ad2ee9af3b97c46293bdfc9e2925a67726b786 (commit)
via af3bfbb6126ebce381c52ced87fc8af372d84118 (commit)
via 6f1a5b9cb0bc761115ef4bc31472ee47db83a906 (commit)
via 2538d2a88afa75f4c23961d77fe9ffccb048368a (commit)
via 5f478e932583f04fc1bd067e437e2d929060dd18 (commit)
via 519850a6a801553c4cfbc7384236536b5a9cedf2 (commit)
via 6ac7e6d3d1f75cc6d6fe625d2ceaaf00b86a720d (commit)
via bb5b4b99bd9957fd1b4713c5168b0d7e148899ee (commit)
via cb97316068f201ffd03f54d67074a83601c2bb45 (commit)
via aace3aa52c1b9fe3b2e907f43e8d6801ceeb28f9 (commit)
via 75adb33e1b2038691e1f62078ee530d1ba01c873 (commit)
via 18e820bdfe2598ec1901fd52fd80c8dae4f55c32 (commit)
via 3fbabc1b5236b6667f453a2949849d96f6e683df (commit)
via 187d6318298adca84193d8c78952e023f303bc2d (commit)
via 87c2b7450a906fbfc87c90b1e30cff11ac5e2aca (commit)
via c4b2d43234205a0d9b1310e55b1f00dee9c20625 (commit)
via 3ac93fa684e01ee2e8ababa6d9cc617cdbd3cbea (commit)
via fe561d69b42d55191d6d1f01d4f3cdcf1fc9faaa (commit)
via 23d57ba45b348b580fc584bbd77fe3960796622d (commit)
via f777c74882e6b0f52b15f62d1d6251cd180979e4 (commit)
via 1b5156270c5cb8d7a4a1b095d981f1a84a98554f (commit)
via b58c06e93fc1392aea0347ea099376b41ec4b7c3 (commit)
via a1f0e517f6b37ea987c0146a4ca93f50715f00f2 (commit)
via d9c1f53cde2e3a09c92e7e6f009f35237b2272a9 (commit)
via 21c5372c6b670820e842e01336eb6b191d6e10b7 (commit)
via 4a98eba9ae08ffccb822842f74b1b805302a1ad1 (commit)
via e26dc8ebc182bec997624213c771f06e9b0179e8 (commit)
via 46434cfe5a053097440bcccc35d0ce7d00bbcfee (commit)
via 679d5507d6c390fd03e5c71300001f2696b1f5b1 (commit)
via 18d6239d25924545ba91825011d467861cd5513c (commit)
via 48fd863c654325eefceb8dfd182c88a8149ca309 (commit)
via 8666f138c10e2a201ee288770f29c5a20b9fc706 (commit)
via 488bc59b2d90e0a9a23801b034c8a54525d83da4 (commit)
via e2f03263c7c2496ff3ee84e43eb133fe171905f9 (commit)
via db7330822cb7dbdd1b61a34737d1b24158d8068d (commit)
via 2e0b7fcafcccc50602f8fd4df11b6312467e95fa (commit)
via fa8fd28e3ca22518a147cf34bf7146ef2a173257 (commit)
via 36f8e449321e4fa02d88fee1fded14aa8ff81723 (commit)
via 8f76037ba8a37c488612285ffe70d26d0d038124 (commit)
via 4d82174d21dd5cdf0668a10ca016e72338fd97e6 (commit)
via 1a37db7b0a70b5672cd0f7c86b19b85ba824378f (commit)
via 28225aeb0336a4872bbaa3aae5d331172f1e4068 (commit)
via 296ac256c898c145f7e4566d86d5f09f4e966b43 (commit)
via 58fa1d8438cb613c6bf7bece8702146f3eed5205 (commit)
via bd1aa20c5878436505b31aa987473ac3fbb6395c (commit)
via 035b113f60302f6d9c265e6e3a63dbb3c5873153 (commit)
via 29fe8719ae4f4463cf5c63ad20cf2eb1ae39cd19 (commit)
via 72bd39971753efa7e951b945d07e8d9704a07221 (commit)
via ceb5ad39a5b94ba26d4a4a059f7801b758ddcec8 (commit)
via 220778381f3a6aa6988c682f914fb9baeada85be (commit)
via 25a80e9318880fbff91289ac8f70e1cae4c132a2 (commit)
via a788135c352c36d1a905c7630423ba57b2ae072a (commit)
via b01b480adb45c3bbfcfab13e343e08c16854dedc (commit)
via 088268fc00cb466c4b4576a2dbbaff720cb3e3af (commit)
via 95c01d22571043d7b6405f928538204cf930d453 (commit)
via cdd57a73f2f90d6b14b8652ab94adc14e4b99c8b (commit)
from 4a78e8e91fdad38e567fef0cd43aa8cb6bd33580 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 9b16a843a641159867ea390c4c2384b320183b15
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Mon Jul 9 15:08:59 2018 +0200
Remove _get_intermediate_collection_info method and create function
with same name
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 03f5a5eb6..2618ddeca 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -11,6 +11,7 @@ import datetime
import ciso8601
import uuid
+from arvados_cwl.util import get_current_container, get_intermediate_collection_info
import ruamel.yaml as yaml
from cwltool.errors import WorkflowException
@@ -21,7 +22,6 @@ from cwltool.job import JobBase
import arvados.collection
-from arvados.errors import ApiError
from .arvdocker import arv_docker_get_image
from . import done
from .runner import Runner, arvados_jobs_image, packed_workflow, trim_anonymous_location, remove_redundant_fields
@@ -166,7 +166,9 @@ class ArvadosContainer(JobBase):
keepemptydirs(vwd)
- info = self._get_intermediate_collection_info()
+ if not runtimeContext.current_container:
+ runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = get_intermediate_collection_info(runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
ensure_unique_name=True,
trash_at=info["trash_at"],
@@ -341,26 +343,6 @@ class ArvadosContainer(JobBase):
finally:
self.output_callback(outputs, processStatus)
- def _get_intermediate_collection_info(self):
- trash_time = None
- if self.arvrunner.intermediate_output_ttl > 0:
- trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
-
- current_container_uuid = None
- try:
- current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
- current_container_uuid = current_container['uuid']
- except ApiError as e:
- # Status code 404 just means we're not running in a container.
- if e.resp.status != 404:
- logger.info("Getting current container: %s", e)
- props = {"type": "Intermediate",
- "container": current_container_uuid}
-
- return {"name" : "Intermediate collection",
- "trash_at" : trash_time,
- "properties" : props}
-
class RunnerContainer(Runner):
"""Submit and manage a container that runs arvados-cwl-runner."""
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 2d112c87a..36a047213 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -6,7 +6,6 @@ import logging
import re
import copy
import json
-import datetime
import time
from cwltool.process import shortname, UnsupportedRequirement
@@ -19,6 +18,7 @@ from cwltool.job import JobBase
from schema_salad.sourceline import SourceLine
+from arvados_cwl.util import get_current_container, get_intermediate_collection_info
import ruamel.yaml as yaml
import arvados.collection
@@ -77,7 +77,9 @@ class ArvadosJob(JobBase):
if vwd:
with Perf(metrics, "generatefiles.save_new %s" % self.name):
- info = self._get_intermediate_collection_info()
+ if not runtimeContext.current_container:
+ runtimeContext.current_container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = get_intermediate_collection_info(runtimeContext.current_container, runtimeContext.intermediate_output_ttl)
vwd.save_new(name=info["name"],
ensure_unique_name=True,
trash_at=info["trash_at"],
@@ -282,26 +284,6 @@ class ArvadosJob(JobBase):
finally:
self.output_callback(outputs, processStatus)
- def _get_intermediate_collection_info(self):
- trash_time = None
- if self.arvrunner.intermediate_output_ttl > 0:
- trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
-
- current_container_uuid = None
- try:
- current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
- current_container_uuid = current_container['uuid']
- except ApiError as e:
- # Status code 404 just means we're not running in a container.
- if e.resp.status != 404:
- logger.info("Getting current container: %s", e)
- props = {"type": "Intermediate",
- "container": current_container_uuid}
-
- return {"name" : "Intermediate collection",
- "trash_at" : trash_time,
- "properties" : props}
-
class RunnerJob(Runner):
"""Submit and manage a Crunch job that runs crunch_scripts/cwl-runner."""
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index cf0c1fb7e..81e256ed5 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
from cwltool.context import LoadingContext, RuntimeContext
class ArvLoadingContext(LoadingContext):
@@ -24,5 +28,6 @@ class ArvRuntimeContext(RuntimeContext):
self.wait = True
self.cwl_runner_job = None
self.storage_classes = "default"
+ self.current_container = None
super(ArvRuntimeContext, self).__init__(kwargs)
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 9a76f81da..a3b6a4dd9 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -7,8 +7,8 @@ import logging
import uuid
import os
import urllib
-import datetime
+from arvados_cwl.util import get_current_container, get_intermediate_collection_info
import arvados.commands.run
import arvados.collection
@@ -155,7 +155,8 @@ class ArvPathMapper(PathMapper):
for l in srcobj.get("listing", []):
self.addentry(l, c, ".", remap)
- info = self._get_intermediate_collection_info()
+ container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = get_intermediate_collection_info(container, self.arvrunner.intermediate_output_ttl)
c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
@@ -173,7 +174,8 @@ class ArvPathMapper(PathMapper):
num_retries=self.arvrunner.num_retries )
self.addentry(srcobj, c, ".", remap)
- info = self._get_intermediate_collection_info()
+ container = get_current_container(self.arvrunner.api, self.arvrunner.num_retries, logger)
+ info = get_intermediate_collection_info(container, self.arvrunner.intermediate_output_ttl)
c.save_new(name=info["name"],
owner_uuid=self.arvrunner.project_uuid,
@@ -212,26 +214,6 @@ class ArvPathMapper(PathMapper):
else:
return None
- def _get_intermediate_collection_info(self):
- trash_time = None
- if self.arvrunner.intermediate_output_ttl > 0:
- trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
-
- current_container_uuid = None
- try:
- current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
- current_container_uuid = current_container['uuid']
- except ApiError as e:
- # Status code 404 just means we're not running in a container.
- if e.resp.status != 404:
- logger.info("Getting current container: %s", e)
- props = {"type": "Intermediate",
- "container": current_container_uuid}
-
- return {"name" : "Intermediate collection",
- "trash_at" : trash_time,
- "properties" : props}
-
class StagingPathMapper(PathMapper):
_follow_dirs = True
diff --git a/sdk/cwl/arvados_cwl/util.py b/sdk/cwl/arvados_cwl/util.py
new file mode 100644
index 000000000..7dafedac2
--- /dev/null
+++ b/sdk/cwl/arvados_cwl/util.py
@@ -0,0 +1,28 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import datetime
+from arvados.errors import ApiError
+
+def get_intermediate_collection_info(current_container, intermediate_output_ttl):
+ name = "Intermediate collection"
+ trash_time = None
+ if intermediate_output_ttl > 0:
+ trash_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=intermediate_output_ttl)
+ container_uuid = None
+ if current_container:
+ container_uuid = current_container['uuid']
+ props = {"type": "intermediate", "container": container_uuid}
+
+ return {"name" : name, "trash_at" : trash_time, "properties" : props}
+
+def get_current_container(api, num_retries=0, logger=None):
+ current_container = None
+ try:
+ current_container = api.containers().current().execute(num_retries=num_retries)
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404 and logger:
+ logger.info("Getting current container: %s", e)
+ return current_container;
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index bd5b1a1b6..e7ce9c370 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -9,7 +9,6 @@ import logging
import mock
import unittest
import os
-import datetime
import functools
import cwltool.process
import cwltool.secrets
@@ -22,13 +21,6 @@ if not os.getenv('ARVADOS_DEBUG'):
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
-class MockDateTime(datetime.datetime):
- @classmethod
- def now(cls):
- return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
-
-datetime.datetime = MockDateTime
-
class TestContainer(unittest.TestCase):
def helper(self, runner, enable_reuse=True):
@@ -698,17 +690,3 @@ class TestContainer(unittest.TestCase):
_, kwargs = runner.api.container_requests().create.call_args
self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
-
-
- def test_get_intermediate_collection_info(self):
- arvrunner = mock.MagicMock()
- arvrunner.intermediate_output_ttl = 60
- arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
-
- container = arvados_cwl.ArvadosContainer(arvrunner)
-
- info = container._get_intermediate_collection_info()
-
- self.assertEqual(info["name"], "Intermediate collection")
- self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
- self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index b9f1c3961..c110bc5d5 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -10,7 +10,6 @@ import os
import unittest
import copy
import StringIO
-import datetime
import arvados
import arvados_cwl
@@ -25,13 +24,6 @@ if not os.getenv('ARVADOS_DEBUG'):
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
-class MockDateTime(datetime.datetime):
- @classmethod
- def now(cls):
- return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
-
-datetime.datetime = MockDateTime
-
class TestJob(unittest.TestCase):
def helper(self, runner, enable_reuse=True):
@@ -339,20 +331,6 @@ class TestJob(unittest.TestCase):
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
- def test_get_intermediate_collection_info(self):
- arvrunner = mock.MagicMock()
- arvrunner.intermediate_output_ttl = 60
- arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
-
- job = arvados_cwl.ArvadosJob(arvrunner)
-
- info = job._get_intermediate_collection_info()
-
- self.assertEqual(info["name"], "Intermediate collection")
- self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
- self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
-
-
class TestWorkflow(unittest.TestCase):
def helper(self, runner, enable_reuse=True):
diff --git a/sdk/cwl/tests/test_pathmapper.py b/sdk/cwl/tests/test_pathmapper.py
index e5326147d..eaa571142 100644
--- a/sdk/cwl/tests/test_pathmapper.py
+++ b/sdk/cwl/tests/test_pathmapper.py
@@ -9,7 +9,6 @@ import unittest
import json
import logging
import os
-import datetime
import arvados
import arvados.keep
@@ -27,13 +26,6 @@ def upload_mock(files, api, dry_run=False, num_retries=0, project=None, fnPatter
c.keepref = "%s/%s" % (pdh, os.path.basename(c.fn))
c.fn = fnPattern % (pdh, os.path.basename(c.fn))
-class MockDateTime(datetime.datetime):
- @classmethod
- def now(cls):
- return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
-
-datetime.datetime = MockDateTime
-
class TestPathmap(unittest.TestCase):
def setUp(self):
self.api = mock.MagicMock()
@@ -109,19 +101,3 @@ class TestPathmap(unittest.TestCase):
"class": "File",
"location": "file:tests/hw.py"
}], "", "/test/%s", "/test/%s/%s")
-
- def test_get_intermediate_collection_info(self):
- self.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
- arvrunner = arvados_cwl.ArvCwlRunner(self.api)
- arvrunner.intermediate_output_ttl = 60
-
- path_mapper = ArvPathMapper(arvrunner, [{
- "class": "File",
- "location": "keep:99999999999999999999999999999991+99/hw.py"
- }], "", "/test/%s", "/test/%s/%s")
-
- info = path_mapper._get_intermediate_collection_info()
-
- self.assertEqual(info["name"], "Intermediate collection")
- self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
- self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
diff --git a/sdk/cwl/tests/test_util.py b/sdk/cwl/tests/test_util.py
new file mode 100644
index 000000000..e42197e21
--- /dev/null
+++ b/sdk/cwl/tests/test_util.py
@@ -0,0 +1,44 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import unittest
+import mock
+import datetime
+import httplib2
+
+from arvados_cwl.util import *
+from arvados.errors import ApiError
+
+class MockDateTime(datetime.datetime):
+ @classmethod
+ def utcnow(cls):
+ return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
+
+class TestUtil(unittest.TestCase):
+ def test_get_intermediate_collection_info(self):
+ current_container = {"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+ intermediate_output_ttl = 120
+
+ info = get_intermediate_collection_info(current_container, intermediate_output_ttl)
+
+ self.assertEqual(info["name"], "Intermediate collection")
+ self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 2, 0, 0))
+ self.assertEqual(info["properties"], {"type" : "intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
+
+ def test_get_current_container_success(self):
+ api = mock.MagicMock()
+ api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+
+ current_container = get_current_container(api)
+
+ self.assertEqual(current_container, {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
+
+ def test_get_current_container_error(self):
+ api = mock.MagicMock()
+ api.containers().current().execute.side_effect = ApiError(httplib2.Response({"status": 300}), "")
+ logger = mock.MagicMock()
+
+ self.assertRaises(ApiError, get_current_container(api, num_retries=0, logger=logger))
commit 5c4d9d38dcee73a7ffb6221c80f707c3924da64f
Merge: ee4c01999 a12e4d369
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Thu Jul 5 17:36:24 2018 +0200
Merge branch 'master' of git.curoverse.com:arvados into 13330-cwl-intermediate-collections-cleanup
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --cc sdk/cwl/arvados_cwl/arvjob.py
index 332c6a7b9,70c2173db..2d112c87a
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@@ -6,10 -6,9 +6,10 @@@ import loggin
import re
import copy
import json
+import datetime
import time
- from cwltool.process import get_feature, shortname, UnsupportedRequirement
+ from cwltool.process import shortname, UnsupportedRequirement
from cwltool.errors import WorkflowException
from cwltool.command_line_tool import revmap_file, CommandLineTool
from cwltool.load_tool import fetch_document
diff --cc sdk/cwl/tests/test_container.py
index a8a91efd3,dd484690c..bd5b1a1b6
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@@ -641,15 -649,45 +656,59 @@@ class TestContainer(unittest.TestCase)
}
}))
+ # The test passes no builder.resources
+ # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+ @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+ def test_timelimit(self, keepdocker):
+ arv_docker_clear_cache()
+
+ runner = mock.MagicMock()
+ runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+ runner.ignore_docker_for_reuse = False
+ runner.intermediate_output_ttl = 0
+ runner.secret_store = cwltool.secrets.SecretStore()
+
+ keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+ runner.api.collections().get().execute.return_value = {
+ "portable_data_hash": "99999999999999999999999999999993+99"}
+
+ tool = cmap({
+ "inputs": [],
+ "outputs": [],
+ "baseCommand": "ls",
+ "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+ "id": "#",
+ "class": "CommandLineTool",
+ "hints": [
+ {
+ "class": "http://commonwl.org/cwltool#TimeLimit",
+ "timelimit": 42
+ }
+ ]
+ })
+
+ loadingContext, runtimeContext = self.helper(runner)
+ runtimeContext.name = "test_timelimit"
+
+ arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
+ arvtool.formatgraph = None
+
+ for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+ j.run(runtimeContext)
+
+ _, kwargs = runner.api.container_requests().create.call_args
+ self.assertEqual(42, kwargs['body']['scheduling_parameters'].get('max_run_time'))
++
++
+ def test_get_intermediate_collection_info(self):
+ arvrunner = mock.MagicMock()
+ arvrunner.intermediate_output_ttl = 60
+ arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+
+ container = arvados_cwl.ArvadosContainer(arvrunner)
+
+ info = container._get_intermediate_collection_info()
+
+ self.assertEqual(info["name"], "Intermediate collection")
+ self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
+ self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
diff --cc sdk/cwl/tests/test_job.py
index 1841fd3f8,c110bc5d5..b9f1c3961
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@@ -25,15 -24,30 +25,37 @@@ if not os.getenv('ARVADOS_DEBUG')
logging.getLogger('arvados.cwl-runner').setLevel(logging.WARN)
logging.getLogger('arvados.arv-run').setLevel(logging.WARN)
+class MockDateTime(datetime.datetime):
+ @classmethod
+ def now(cls):
+ return datetime.datetime(2018, 1, 1, 0, 0, 0, 0)
+
+datetime.datetime = MockDateTime
+
class TestJob(unittest.TestCase):
+ def helper(self, runner, enable_reuse=True):
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+ loadingContext = arvados_cwl.context.ArvLoadingContext(
+ {"avsc_names": avsc_names,
+ "basedir": "",
+ "make_fs_access": make_fs_access,
+ "loader": Loader({}),
+ "metadata": {"cwlVersion": "v1.0"},
+ "makeTool": runner.arv_make_tool})
+ runtimeContext = arvados_cwl.context.ArvRuntimeContext(
+ {"work_api": "jobs",
+ "basedir": "",
+ "name": "test_run_job_"+str(enable_reuse),
+ "make_fs_access": make_fs_access,
+ "enable_reuse": enable_reuse,
+ "priority": 500})
+
+ return loadingContext, runtimeContext
+
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch('arvados.commands.keepdocker.list_images_in_arv')
@@@ -315,22 -331,36 +339,50 @@@
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
+ def test_get_intermediate_collection_info(self):
+ arvrunner = mock.MagicMock()
+ arvrunner.intermediate_output_ttl = 60
+ arvrunner.api.containers().current().execute.return_value = {"uuid" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"}
+
+ job = arvados_cwl.ArvadosJob(arvrunner)
+
+ info = job._get_intermediate_collection_info()
+
+ self.assertEqual(info["name"], "Intermediate collection")
+ self.assertEqual(info["trash_at"], datetime.datetime(2018, 1, 1, 0, 1))
+ self.assertEqual(info["properties"], {"type" : "Intermediate", "container" : "zzzzz-8i9sb-zzzzzzzzzzzzzzz"})
+
+
class TestWorkflow(unittest.TestCase):
+ def helper(self, runner, enable_reuse=True):
+ document_loader, avsc_names, schema_metadata, metaschema_loader = cwltool.process.get_schema("v1.0")
+
+ make_fs_access=functools.partial(arvados_cwl.CollectionFsAccess,
+ collection_cache=arvados_cwl.CollectionCache(runner.api, None, 0))
+
+ document_loader.fetcher_constructor = functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=make_fs_access(""))
+ document_loader.fetcher = document_loader.fetcher_constructor(document_loader.cache, document_loader.session)
+ document_loader.fetch_text = document_loader.fetcher.fetch_text
+ document_loader.check_exists = document_loader.fetcher.check_exists
+
+ loadingContext = arvados_cwl.context.ArvLoadingContext(
+ {"avsc_names": avsc_names,
+ "basedir": "",
+ "make_fs_access": make_fs_access,
+ "loader": document_loader,
+ "metadata": {"cwlVersion": "v1.0"},
+ "construct_tool_object": runner.arv_make_tool})
+ runtimeContext = arvados_cwl.context.ArvRuntimeContext(
+ {"work_api": "jobs",
+ "basedir": "",
+ "name": "test_run_wf_"+str(enable_reuse),
+ "make_fs_access": make_fs_access,
+ "enable_reuse": enable_reuse,
+ "priority": 500})
+
+ return loadingContext, runtimeContext
+
# The test passes no builder.resources
# Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
@mock.patch("arvados.collection.CollectionReader")
commit ee4c01999aff1ebc1e2dc338a70d1d7b812c633c
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Thu Jul 5 17:20:51 2018 +0200
Remove trailig white-space
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 014e8e58c..a7f160c6c 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -158,9 +158,9 @@ class ArvadosContainer(object):
keepemptydirs(vwd)
info = self._get_intermediate_collection_info()
- vwd.save_new(name=info["name"],
- ensure_unique_name=True,
- trash_at=info["trash_at"],
+ vwd.save_new(name=info["name"],
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
properties=info["properties"])
prev = None
@@ -330,19 +330,19 @@ class ArvadosContainer(object):
self.output_callback(outputs, processStatus)
def _get_intermediate_collection_info(self):
- trash_time = None
- if self.arvrunner.intermediate_output_ttl > 0:
- trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
-
- current_container_uuid = None
- try:
- current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
- current_container_uuid = current_container['uuid']
- except ApiError as e:
- # Status code 404 just means we're not running in a container.
- if e.resp.status != 404:
+ trash_time = None
+ if self.arvrunner.intermediate_output_ttl > 0:
+ trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+
+ current_container_uuid = None
+ try:
+ current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+ current_container_uuid = current_container['uuid']
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404:
logger.info("Getting current container: %s", e)
- props = {"type": "Intermediate",
+ props = {"type": "Intermediate",
"container": current_container_uuid}
return {"name" : "Intermediate collection",
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index ea599ea83..332c6a7b9 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -69,9 +69,9 @@ class ArvadosJob(object):
if vwd:
with Perf(metrics, "generatefiles.save_new %s" % self.name):
info = self._get_intermediate_collection_info()
- vwd.save_new(name=info["name"],
- ensure_unique_name=True,
- trash_at=info["trash_at"],
+ vwd.save_new(name=info["name"],
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
properties=info["properties"])
for f, p in generatemapper.items():
@@ -271,19 +271,19 @@ class ArvadosJob(object):
self.output_callback(outputs, processStatus)
def _get_intermediate_collection_info(self):
- trash_time = None
- if self.arvrunner.intermediate_output_ttl > 0:
- trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
-
- current_container_uuid = None
- try:
- current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
- current_container_uuid = current_container['uuid']
- except ApiError as e:
- # Status code 404 just means we're not running in a container.
- if e.resp.status != 404:
+ trash_time = None
+ if self.arvrunner.intermediate_output_ttl > 0:
+ trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+
+ current_container_uuid = None
+ try:
+ current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+ current_container_uuid = current_container['uuid']
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404:
logger.info("Getting current container: %s", e)
- props = {"type": "Intermediate",
+ props = {"type": "Intermediate",
"container": current_container_uuid}
return {"name" : "Intermediate collection",
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index b54fcdcc7..766769443 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -158,9 +158,9 @@ class ArvPathMapper(PathMapper):
info = self._get_intermediate_collection_info()
c.save_new(name=info["name"],
- owner_uuid=self.arvrunner.project_uuid,
- ensure_unique_name=True,
- trash_at=info["trash_at"],
+ owner_uuid=self.arvrunner.project_uuid,
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
properties=info["properties"])
ab = self.collection_pattern % c.portable_data_hash()
@@ -176,9 +176,9 @@ class ArvPathMapper(PathMapper):
info = self._get_intermediate_collection_info()
c.save_new(name=info["name"],
- owner_uuid=self.arvrunner.project_uuid,
- ensure_unique_name=True,
- trash_at=info["trash_at"],
+ owner_uuid=self.arvrunner.project_uuid,
+ ensure_unique_name=True,
+ trash_at=info["trash_at"],
properties=info["properties"])
ab = self.file_pattern % (c.portable_data_hash(), srcobj["basename"])
@@ -213,19 +213,19 @@ class ArvPathMapper(PathMapper):
return None
def _get_intermediate_collection_info(self):
- trash_time = None
- if self.arvrunner.intermediate_output_ttl > 0:
- trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
-
- current_container_uuid = None
- try:
- current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
- current_container_uuid = current_container['uuid']
- except ApiError as e:
- # Status code 404 just means we're not running in a container.
- if e.resp.status != 404:
+ trash_time = None
+ if self.arvrunner.intermediate_output_ttl > 0:
+ trash_time = datetime.datetime.now() + datetime.timedelta(seconds=self.arvrunner.intermediate_output_ttl)
+
+ current_container_uuid = None
+ try:
+ current_container = self.arvrunner.api.containers().current().execute(num_retries=self.arvrunner.num_retries)
+ current_container_uuid = current_container['uuid']
+ except ApiError as e:
+ # Status code 404 just means we're not running in a container.
+ if e.resp.status != 404:
logger.info("Getting current container: %s", e)
- props = {"type": "Intermediate",
+ props = {"type": "Intermediate",
"container": current_container_uuid}
return {"name" : "Intermediate collection",
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list