[ARVADOS] updated: 028e052db597498ee5c1412b606fa178c621b3ca
Git user
git at public.curoverse.com
Tue May 30 16:59:37 EDT 2017
Summary of changes:
.../app/helpers/pipeline_instances_helper.rb | 2 +-
apps/workbench/app/models/proxy_work_unit.rb | 40 ++---
.../views/container_requests/_show_recent.html.erb | 6 +-
.../container_requests/_show_recent_rows.html.erb | 2 +-
.../app/views/projects/_show_dashboard.html.erb | 24 ++-
.../app/views/work_units/_show_child.html.erb | 5 +-
.../controllers/application_controller_test.rb | 2 +-
doc/user/cwl/cwl-extensions.html.textile.liquid | 17 ++-
doc/user/cwl/cwl-run-options.html.textile.liquid | 20 +++
doc/user/cwl/cwl-runner.html.textile.liquid | 2 +-
sdk/cwl/arvados_cwl/__init__.py | 65 +++++++-
sdk/cwl/arvados_cwl/arv-cwl-schema.yml | 35 +++++
sdk/cwl/arvados_cwl/arvcontainer.py | 34 ++++-
sdk/cwl/arvados_cwl/crunch_script.py | 2 +
sdk/cwl/arvados_cwl/fsaccess.py | 7 +-
sdk/cwl/arvados_cwl/pathmapper.py | 65 +++++---
sdk/cwl/arvados_cwl/runner.py | 10 +-
sdk/cwl/setup.py | 5 +-
.../cwl/tests/collection_per_tool/a.txt | 0
.../cwl/tests/collection_per_tool/b.txt | 0
.../cwl/tests/collection_per_tool/c.txt | 0
.../collection_per_tool/collection_per_tool.cwl | 13 ++
.../collection_per_tool_packed.cwl | 45 ++++++
sdk/cwl/tests/collection_per_tool/step1.cwl | 15 ++
sdk/cwl/tests/collection_per_tool/step2.cwl | 15 ++
sdk/cwl/tests/test_container.py | 163 +++++++++++---------
sdk/cwl/tests/test_pathmapper.py | 23 ++-
sdk/cwl/tests/test_submit.py | 164 +++++++++++++++------
sdk/cwl/tests/wf/expect_packed.cwl | 4 +-
sdk/cwl/tests/wf/submit_wf_packed.cwl | 2 +-
sdk/python/arvados/arvfile.py | 12 +-
sdk/python/arvados/commands/run.py | 68 +++++----
sdk/python/tests/test_arvfile.py | 4 +-
services/api/lib/crunch_dispatch.rb | 2 +-
services/api/test/unit/crunch_dispatch_test.rb | 2 +-
services/nodemanager/arvnodeman/jobqueue.py | 18 ++-
services/nodemanager/tests/test_jobqueue.py | 36 ++++-
37 files changed, 686 insertions(+), 243 deletions(-)
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/collection_per_tool/a.txt (100%)
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/collection_per_tool/b.txt (100%)
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/collection_per_tool/c.txt (100%)
create mode 100644 sdk/cwl/tests/collection_per_tool/collection_per_tool.cwl
create mode 100644 sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
create mode 100644 sdk/cwl/tests/collection_per_tool/step1.cwl
create mode 100644 sdk/cwl/tests/collection_per_tool/step2.cwl
via 028e052db597498ee5c1412b606fa178c621b3ca (commit)
via b5d352d6099b60db5dcdd9183dcab3e8e17d729e (commit)
via 8e1efc4c74f86a986dd0c6f15f53777d0d2bf0bb (commit)
via 505b5c4d23da212bf285b3279a756b4f02d75567 (commit)
via 23c8dd0d911c4b238169f399ff455f2dab9f0b4f (commit)
via 7c03ff7b9ddac8dc232a910364fcecd32f86e03d (commit)
via 9d6fb27b6a0ef28062be37efdea20d1a4c7079d3 (commit)
via f7a665dcd8dfe96b49e63f3caa352f5e5f8b19a1 (commit)
via b436b04a6e0cd28eaa465bb928eea3195b114322 (commit)
via 3cd9054aa2699473f1af45c165441d7276162b46 (commit)
via dbd91c317d54da7123bee4f2d496b440b2ba9393 (commit)
via d0811968704eb02e3e98e46345fadf22cf95b5ec (commit)
via 0da65aef99133cb76661f56f2adac83a77ad8ac9 (commit)
via 4dac499d30988fd8289a40ad4128d6d471037180 (commit)
via d0b58511f396325bf083463d460567bf6f76e468 (commit)
via 5223e68fa0d1f1fc52c7285fa6844d7932ba9acb (commit)
via 79b826befbc0b62a822e2d0c922e5c4ea0a3bb7c (commit)
via 8ffcfd1523d2a023c0286976a1aed50fb70dea3a (commit)
via 9a7ea56065ed47ef114d569f137ff50d1efffc48 (commit)
via 932a318673b17b63a2f2d99a6ba170c908ef3b20 (commit)
via e578f043c4d3b45a08f15d365adea0386228a4bf (commit)
via 1d77dec0f7bc1cca34d52288bd39ecd0a79be250 (commit)
via d451eabbcbb8d5885fb30c2c341d93204a008596 (commit)
via 456dde006228871c522597abbc5fe8006e44ad4d (commit)
via bee4fa82c15b2188c2105fff3b52c305b38f04a6 (commit)
via 180074677191e4f30f90e8b8390245ab356b1c3f (commit)
via 6eae76355f346740cc50ddba4656239c507461ac (commit)
via f60885adec1fe9a7b92f30ffbd4f5bb112e1aeae (commit)
via 72a4000d8dc81879afae18ca440ab2ffaf712823 (commit)
via df6578e918d8c29baf314e671c28a96aea1c1d00 (commit)
via 4a1df4f3aed07a31da8570363b5eaceccf4dffa8 (commit)
via b310fc04b9e60b974f4bfc2c86a5c45ecd7c65f2 (commit)
via 8645888f12c25edaaac8e03fb5691cfcfbcdb9b2 (commit)
via 43dbdec9f0f6361e4705671bbdade34cd18adefa (commit)
via dd58470216d6a416e4d831a3b9c25bfdaa255fff (commit)
via 67fbf00f0d868384f1585f2473b5f89455001638 (commit)
via 2e83bfa0e38b87632871f91cead42c9ead3d7bb7 (commit)
via ccc2ef472f5048059194e67b973d663c07816de9 (commit)
via cee8f888b442448b2bffe243da2e9cc5dcf29b04 (commit)
from 1ceeafd5411b47834b836019dfd21d4050158171 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 028e052db597498ee5c1412b606fa178c621b3ca
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue May 30 17:56:07 2017 -0300
11684: ArvadosFile.flush() now check if it is the only owner of a bufferblock before deleting it, so the extra argument is not required.
On commit_all(), always check if the owner attribute is an instance of ArvadosFile before calling flush()
Fixed a couple tests that were mocking bufferblock.owner so that they work with this new behavior.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 2c653d7..a1d87a4 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -799,7 +799,7 @@ class _BlockManager(object):
if v.state() != _BufferBlock.COMMITTED and v.owner:
# Ignore blocks with a list of owners, as if they're not in COMMITTED
# state, they're already being committed asynchronously.
- if not isinstance(v.owner, list):
+ if isinstance(v.owner, ArvadosFile):
v.owner.flush(sync=False)
with self.lock:
@@ -824,7 +824,7 @@ class _BlockManager(object):
# of repacking small blocks, so don't delete it when flushing
# its owners, just do it after flushing them all.
for owner in v.owner:
- owner.flush(sync=True, delete_bufferblock=False)
+ owner.flush(sync=True)
self.delete_bufferblock(k)
def block_prefetch(self, locator):
@@ -1129,7 +1129,7 @@ class ArvadosFile(object):
return len(data)
@synchronized
- def flush(self, sync=True, num_retries=0, delete_bufferblock=True):
+ def flush(self, sync=True, num_retries=0):
"""Flush the current bufferblock to Keep.
:sync:
@@ -1155,8 +1155,10 @@ class ArvadosFile(object):
self.parent._my_block_manager().commit_bufferblock(bb, sync=True)
to_delete.add(s.locator)
s.locator = bb.locator()
- if delete_bufferblock:
- for s in to_delete:
+ for s in to_delete:
+ # Don't delete the bufferblock if it's owned by many files. It'll be
+ # deleted after all of its owners are flush()ed.
+ if self.parent._my_block_manager().get_bufferblock(s).owner is self:
self.parent._my_block_manager().delete_bufferblock(s)
self.parent.notify(MOD, self.parent, self.name, (self, self))
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index a800913..d241f73 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -832,7 +832,7 @@ class BlockManagerTest(unittest.TestCase):
mockkeep = mock.MagicMock()
with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
bufferblock = blockmanager.alloc_bufferblock()
- bufferblock.owner = mock.MagicMock()
+ bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile)
def flush(sync=None):
blockmanager.commit_bufferblock(bufferblock, sync)
bufferblock.owner.flush.side_effect = flush
@@ -863,7 +863,7 @@ class BlockManagerTest(unittest.TestCase):
mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
bufferblock = blockmanager.alloc_bufferblock()
- bufferblock.owner = mock.MagicMock()
+ bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile)
def flush(sync=None):
blockmanager.commit_bufferblock(bufferblock, sync)
bufferblock.owner.flush.side_effect = flush
commit b5d352d6099b60db5dcdd9183dcab3e8e17d729e
Merge: 1ceeafd 8e1efc4
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue May 30 14:07:41 2017 -0300
11684: Merge branch 'master' into 11684-unsigned-locator-fix
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list