[ARVADOS] updated: e3a2f5a80c1e4bd645e7e8bff27ef94f89ceae3e
Git user
git at public.curoverse.com
Fri Jun 9 17:12:24 EDT 2017
Summary of changes:
apps/workbench/app/assets/images/trash-icon.png | Bin 0 -> 18381 bytes
.../workbench/app/assets/javascripts/filterable.js | 1 +
.../app/assets/javascripts/select_modal.js | 1 +
.../app/assets/javascripts/selection.js.erb | 4 +
.../app/controllers/trash_items_controller.rb | 88 +++++++++
apps/workbench/app/models/arvados_base.rb | 4 +
apps/workbench/app/models/arvados_resource_list.rb | 6 +
apps/workbench/app/models/collection.rb | 3 +
.../app/views/application/_breadcrumbs.html.erb | 7 +
.../views/container_requests/_show_recent.html.erb | 6 +-
.../container_requests/_show_recent_rows.html.erb | 2 +-
.../app/views/projects/_show_dashboard.html.erb | 24 ++-
.../trash_items/_create_new_object_button.html.erb | 1 +
.../views/trash_items/_show_recent_trash.html.erb | 54 ++++++
.../views/trash_items/_show_trash_rows.html.erb | 32 ++++
.../app/views/trash_items/_untrash_item.html.erb | 7 +
.../workbench/app/views/trash_items/index.html.erb | 1 +
.../untrash_items.js.erb} | 3 +-
.../views/work_units/_show_all_processes.html.erb | 2 +-
apps/workbench/config/routes.rb | 5 +
apps/workbench/test/integration/trash_test.rb | 87 +++++++++
build/run-tests.sh | 2 +-
doc/api/methods/groups.html.textile.liquid | 1 +
sdk/cwl/arvados_cwl/arvjob.py | 32 +---
sdk/cwl/arvados_cwl/done.py | 62 ++++---
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_job.py | 47 ++++-
sdk/go/arvados/client.go | 19 +-
sdk/go/arvadosclient/arvadosclient.go | 98 +++++-----
sdk/go/arvadosclient/arvadosclient_test.go | 18 +-
sdk/go/keepclient/discover.go | 204 +++++++++++++--------
sdk/go/keepclient/discover_test.go | 17 +-
sdk/go/keepclient/keepclient.go | 167 ++++++++++++-----
sdk/go/keepclient/keepclient_test.go | 19 +-
sdk/go/keepclient/support.go | 57 +-----
sdk/python/arvados/arvfile.py | 27 ++-
sdk/python/tests/test_arvfile.py | 4 +-
sdk/python/tests/test_collections.py | 2 +-
.../arvados/v1/collections_controller.rb | 13 +-
.../controllers/arvados/v1/groups_controller.rb | 14 +-
.../arvados/v1/keep_services_controller.rb | 1 +
services/api/app/models/user.rb | 37 ++--
services/api/config/routes.rb | 1 +
services/api/lib/can_be_an_owner.rb | 33 ++++
services/api/lib/create_ancestor_view.sql | 14 ++
services/api/lib/crunch_dispatch.rb | 2 +-
services/api/test/fixtures/collections.yml | 2 +-
.../arvados/v1/collections_controller_test.rb | 18 ++
.../arvados/v1/groups_controller_test.rb | 47 +++++
.../arvados/v1/keep_services_controller_test.rb | 6 +-
services/api/test/unit/crunch_dispatch_test.rb | 2 +-
services/keep-balance/integration_test.go | 9 +-
services/keep-web/handler.go | 7 +-
services/keepproxy/keepproxy.go | 48 +++--
services/keepproxy/keepproxy_test.go | 2 +-
services/keepstore/keepstore.go | 1 -
services/keepstore/pull_worker_integration_test.go | 15 +-
services/nodemanager/arvnodeman/jobqueue.py | 6 +-
services/nodemanager/tests/test_jobqueue.py | 8 +-
tools/keep-block-check/keep-block-check.go | 14 +-
tools/keep-block-check/keep-block-check_test.go | 13 +-
tools/keep-exercise/keep-exercise.go | 8 +-
tools/keep-rsync/keep-rsync.go | 14 +-
tools/keep-rsync/keep-rsync_test.go | 149 +++++++--------
64 files changed, 1086 insertions(+), 514 deletions(-)
create mode 100644 apps/workbench/app/assets/images/trash-icon.png
create mode 100644 apps/workbench/app/controllers/trash_items_controller.rb
create mode 100644 apps/workbench/app/views/trash_items/_create_new_object_button.html.erb
create mode 100644 apps/workbench/app/views/trash_items/_show_recent_trash.html.erb
create mode 100644 apps/workbench/app/views/trash_items/_show_trash_rows.html.erb
create mode 100644 apps/workbench/app/views/trash_items/_untrash_item.html.erb
create mode 100644 apps/workbench/app/views/trash_items/index.html.erb
copy apps/workbench/app/views/{projects/remove_items.js.erb => trash_items/untrash_items.js.erb} (58%)
create mode 100644 apps/workbench/test/integration/trash_test.rb
create mode 100644 services/api/lib/create_ancestor_view.sql
via e3a2f5a80c1e4bd645e7e8bff27ef94f89ceae3e (commit)
via 6ab526bb7fef3d7d42ff728fa30444e75de7be38 (commit)
via b745c25cb15077f4f5cd658a6d59392c48bc8add (commit)
via 29665e2d9a543bffb237d148c3484c03b03e30aa (commit)
via 4daf12ad8cb2f74c3f3d72eafef7ed7a0621a4ac (commit)
via 87ea4388edb977d246b09b78d9d4bfa5c2ba5170 (commit)
via b42d7d9ed4640296ae5d01d26268c4fdba1ca10c (commit)
via f9ff1c5444f7030fa0b372446bd40d4feed203c8 (commit)
via cc77c62e148a632c5b690c275fd9fbf659ad2715 (commit)
via 55315b668b8fa04572a44fd7db6598478f54130a (commit)
via 0a6adabd77bf483186a29a7cffef38866a5ee8e0 (commit)
via 0560a5812377315a92c8a4e9b41a68677832ea65 (commit)
via 695a100d4bd3bf4f5534c7e489c118c2917bf35a (commit)
via dcd20d41fde4216345a20ddfa950b40de74b85b5 (commit)
via cb230b07e0125d819991bc74a1f528740068157d (commit)
via a10eb3ebf97a29687c6f43d705e30fb0349364d2 (commit)
via 51549f93489f5d57f71b7767528ee7dec9079fb6 (commit)
via 5f8a7ee855778644abf70e7058317891738d2ce1 (commit)
via 5d03f6499055ef109ca2c8d5d59941b25de1fa47 (commit)
via fd4bdb760eb47013b8b60b40859ba7f004ee8383 (commit)
via efcfd2cdc9a9ed6ab3af54e9a4e149d4bdf1f8d3 (commit)
via f834a9841fe27c18bc31e58636ec1bcdb833b54b (commit)
via 0f5295ae31362eefe182f3a2329b3903d6f82a3b (commit)
via c5e4c7027338235435e9698057847686baaee6cc (commit)
via 41372eb0088174d02505f657f0fa2dabbc845791 (commit)
via 75e00445b6de230493e9ee37483dd4c469db29b1 (commit)
via 88abf688d7545fc663e0422f0c586107d4c56e5a (commit)
via 6c40c1c5b000b2eb0967df0f7ebaf613393501bc (commit)
via 916cf89b440fd13a9b9c055d817b34d339442ea3 (commit)
via 6d9ac912d421951ec363e40fbcfde2428aa2f8d9 (commit)
via 5a625a53394fa99ef7fb44ab63104b8195d963df (commit)
via 9ff2df8cfa1f0798c22e4b3c6ef91d4677961295 (commit)
via 7c7dc19ffa0a20fff6d97e51c874bfaca9596b24 (commit)
via 554873d2fb3aa095caaa6fece1c2321008a85b24 (commit)
via 7a98a4695a031c4b7b07d7244f3f293db7ebe0fc (commit)
via 63cfe7a9b391e1a07c6c6c8b0fab10b1c875f1be (commit)
via fd1b1b760f96e5c86e70dd8eb2e629b55f55fe55 (commit)
via a84b665349051a06df5728f435203c8353b5795a (commit)
via 27355f62088566a9388972e128bfb8ba77790e9a (commit)
via 75c5b123e0b4cbfebed9b15364a97c2209f94740 (commit)
via 4d0d27f83183b6659a570ae1fc07f95aefe0b23e (commit)
via 65622f423c2ee35250856657b06118481d53edc8 (commit)
via 741a9b3912c382b25507ca76fae15ddac020f302 (commit)
via 1cdeb90e271f2a0c50d3552ff702b8e6f9489c74 (commit)
via fc10c21d75b15e36c4f861fdd17bf985ca83d888 (commit)
via 9da81b40a1c0e65f2459106a4718e4adcf7551c4 (commit)
via 028e052db597498ee5c1412b606fa178c621b3ca (commit)
via 31ef294c700811eae8ade8dfc5fcb7ec40d1ad2e (commit)
via 5931243d745188851c8240e3e766d97aafdf935c (commit)
via b5d352d6099b60db5dcdd9183dcab3e8e17d729e (commit)
via 8e1efc4c74f86a986dd0c6f15f53777d0d2bf0bb (commit)
via 6fe6390690471cee8ba23984e3560fc4ced8b180 (commit)
via 505b5c4d23da212bf285b3279a756b4f02d75567 (commit)
via 23c8dd0d911c4b238169f399ff455f2dab9f0b4f (commit)
via 1ceeafd5411b47834b836019dfd21d4050158171 (commit)
via e3da56d96e191df11444877ed1989bd8f34bfa5a (commit)
via 9d6fb27b6a0ef28062be37efdea20d1a4c7079d3 (commit)
via d094ae4ee58f26e0585445eccb3be2d019ab020f (commit)
via 3b12ef6b6d7ff6852f6109ab71dbec382322a686 (commit)
from 7cdfe579a8ad314cf303a280b0de68b026244748 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e3a2f5a80c1e4bd645e7e8bff27ef94f89ceae3e
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Fri Jun 9 18:11:15 2017 -0300
11557: Moved logic to done.done(). Updated related tests.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at curoverse.com>
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index a620a20..59ae69c 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -145,6 +145,14 @@ class ArvadosJob(object):
if response["state"] == "Complete":
logger.info("%s reused job %s", self.arvrunner.label(self), response["uuid"])
+ # Give read permission to the desired project on reused jobs
+ for job_name, job_uuid in response.get('components', {}).items():
+ self.arvrunner.api.links().create(body={
+ 'link_class': 'can_read',
+ 'tail_uuid': self.arvrunner.project_uuid,
+ 'head_uuid': job_uuid,
+ }).execute(num_retries=self.arvrunner.num_retries)
+
with Perf(metrics, "done %s" % self.name):
self.done(response)
else:
@@ -291,30 +299,6 @@ class RunnerJob(Runner):
find_or_create=self.enable_reuse
).execute(num_retries=self.arvrunner.num_retries)
- if self.enable_reuse:
- # When reusing jobs, copy its output/log collection to the desired project
- reused_collections = [('Output', job.get('output', None)),
- ('Log', job.get('log', None))]
- for col_type, pdh in [(n, p) for n, p in reused_collections if p]:
- c = arvados.collection.Collection(pdh,
- api_client=self.arvrunner.api,
- keep_client=self.arvrunner.keep_client,
- num_retries=self.arvrunner.num_retries)
- c.save_new(name="{} of {}".format(col_type, self.name),
- owner_uuid=self.arvrunner.project_uuid,
- ensure_unique_name=True,
- num_retries=self.arvrunner.num_retries)
- logger.info("Copied reused job's %s to collection %s",
- col_type.lower(),
- c.manifest_locator())
- # Give read permission to the desired project on reused jobs
- for job_name, job_uuid in job.get('components', {}).items():
- self.arvrunner.api.links().create(body={
- 'link_class': 'can_read',
- 'tail_uuid': self.arvrunner.project_uuid,
- 'head_uuid': job_uuid,
- }).execute(num_retries=self.arvrunner.num_retries)
-
for k,v in job_spec["script_parameters"].items():
if v is False or v is None or isinstance(v, dict):
job_spec["script_parameters"][k] = {"value": v}
diff --git a/sdk/cwl/arvados_cwl/done.py b/sdk/cwl/arvados_cwl/done.py
index 69b074c..48466f0 100644
--- a/sdk/cwl/arvados_cwl/done.py
+++ b/sdk/cwl/arvados_cwl/done.py
@@ -3,39 +3,43 @@ from cwltool.errors import WorkflowException
from collections import deque
def done(self, record, tmpdir, outdir, keepdir):
- colname = "Output %s of %s" % (record["output"][0:7], self.name)
+ cols = [
+ ("output", "Output %s of %s" % (record["output"][0:7], self.name), record["output"]),
+ ("log", "Log of %s" % (record["uuid"]), record["log"])
+ ]
- # check if collection already exists with same owner, name and content
- collection_exists = self.arvrunner.api.collections().list(
- filters=[["owner_uuid", "=", self.arvrunner.project_uuid],
- ['portable_data_hash', '=', record["output"]],
- ["name", "=", colname]]
- ).execute(num_retries=self.arvrunner.num_retries)
-
- if not collection_exists["items"]:
- # Create a collection located in the same project as the
- # pipeline with the contents of the output.
- # First, get output record.
- collections = self.arvrunner.api.collections().list(
- limit=1,
- filters=[['portable_data_hash', '=', record["output"]]],
- select=["manifest_text"]
+ for coltype, colname, colpdh in cols:
+ # check if collection already exists with same owner, name and content
+ collection_exists = self.arvrunner.api.collections().list(
+ filters=[["owner_uuid", "=", self.arvrunner.project_uuid],
+ ['portable_data_hash', '=', colpdh],
+ ["name", "=", colname]]
).execute(num_retries=self.arvrunner.num_retries)
- if not collections["items"]:
- raise WorkflowException(
- "[job %s] output '%s' cannot be found on API server" % (
- self.name, record["output"]))
+ if not collection_exists["items"]:
+ # Create a collection located in the same project as the
+ # pipeline with the contents of the output/log.
+ # First, get output/log record.
+ collections = self.arvrunner.api.collections().list(
+ limit=1,
+ filters=[['portable_data_hash', '=', colpdh]],
+ select=["manifest_text"]
+ ).execute(num_retries=self.arvrunner.num_retries)
+
+ if not collections["items"]:
+ raise WorkflowException(
+ "[job %s] %s '%s' cannot be found on API server" % (
+ self.name, coltype, colpdh))
- # Create new collection in the parent project
- # with the output contents.
- self.arvrunner.api.collections().create(body={
- "owner_uuid": self.arvrunner.project_uuid,
- "name": colname,
- "portable_data_hash": record["output"],
- "manifest_text": collections["items"][0]["manifest_text"]
- }, ensure_unique_name=True).execute(
- num_retries=self.arvrunner.num_retries)
+ # Create new collection in the parent project
+ # with the output/log contents.
+ self.arvrunner.api.collections().create(body={
+ "owner_uuid": self.arvrunner.project_uuid,
+ "name": colname,
+ "portable_data_hash": colpdh,
+ "manifest_text": collections["items"][0]["manifest_text"]
+ }, ensure_unique_name=True).execute(
+ num_retries=self.arvrunner.num_retries)
return done_outputs(self, record, tmpdir, outdir, keepdir)
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 99dd3cb..14e006e 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -161,7 +161,9 @@ class TestJob(unittest.TestCase):
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
""")
api.collections().list().execute.side_effect = ({"items": []},
- {"items": [{"manifest_text": "XYZ"}]})
+ {"items": [{"manifest_text": "XYZ"}]},
+ {"items": []},
+ {"items": [{"manifest_text": "ABC"}]})
arvjob = arvados_cwl.ArvadosJob(runner)
arvjob.name = "testjob"
@@ -179,20 +181,37 @@ class TestJob(unittest.TestCase):
api.collections().list.assert_has_calls([
mock.call(),
+ # Output collection check
mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
['portable_data_hash', '=', '99999999999999999999999999999993+99'],
['name', '=', 'Output 9999999 of testjob']]),
mock.call().execute(num_retries=0),
mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999993+99']],
select=['manifest_text']),
+ mock.call().execute(num_retries=0),
+ # Log collection's turn
+ mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
+ ['portable_data_hash', '=', '99999999999999999999999999999994+99'],
+ ['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
+ mock.call().execute(num_retries=0),
+ mock.call(limit=1, filters=[['portable_data_hash', '=', '99999999999999999999999999999994+99']],
+ select=['manifest_text']),
mock.call().execute(num_retries=0)])
- api.collections().create.assert_called_with(
- ensure_unique_name=True,
- body={'portable_data_hash': '99999999999999999999999999999993+99',
- 'manifest_text': 'XYZ',
- 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
- 'name': 'Output 9999999 of testjob'})
+ api.collections().create.assert_has_calls([
+ mock.call(ensure_unique_name=True,
+ body={'portable_data_hash': '99999999999999999999999999999993+99',
+ 'manifest_text': 'XYZ',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'name': 'Output 9999999 of testjob'}),
+ mock.call().execute(num_retries=0),
+ mock.call(ensure_unique_name=True,
+ body={'portable_data_hash': '99999999999999999999999999999994+99',
+ 'manifest_text': 'ABC',
+ 'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+ 'name': 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz'}),
+ mock.call().execute(num_retries=0),
+ ])
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
@@ -211,7 +230,10 @@ class TestJob(unittest.TestCase):
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
""")
- api.collections().list().execute.side_effect = ({"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},)
+ api.collections().list().execute.side_effect = (
+ {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
+ {"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
+ )
arvjob = arvados_cwl.ArvadosJob(runner)
arvjob.name = "testjob"
@@ -229,10 +251,17 @@ class TestJob(unittest.TestCase):
api.collections().list.assert_has_calls([
mock.call(),
+ # Output collection
mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
['portable_data_hash', '=', '99999999999999999999999999999993+99'],
['name', '=', 'Output 9999999 of testjob']]),
- mock.call().execute(num_retries=0)])
+ mock.call().execute(num_retries=0),
+ # Log collection
+ mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
+ ['portable_data_hash', '=', '99999999999999999999999999999994+99'],
+ ['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
+ mock.call().execute(num_retries=0)
+ ])
self.assertFalse(api.collections().create.called)
commit 6ab526bb7fef3d7d42ff728fa30444e75de7be38
Merge: 7cdfe57 b745c25
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Jun 7 10:45:53 2017 -0300
11557: Merge branch 'master' into 11557-acr-output-col-perms
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list