[ARVADOS] updated: 028e052db597498ee5c1412b606fa178c621b3ca

Git user git at public.curoverse.com
Tue May 30 16:59:37 EDT 2017


Summary of changes:
 .../app/helpers/pipeline_instances_helper.rb       |   2 +-
 apps/workbench/app/models/proxy_work_unit.rb       |  40 ++---
 .../views/container_requests/_show_recent.html.erb |   6 +-
 .../container_requests/_show_recent_rows.html.erb  |   2 +-
 .../app/views/projects/_show_dashboard.html.erb    |  24 ++-
 .../app/views/work_units/_show_child.html.erb      |   5 +-
 .../controllers/application_controller_test.rb     |   2 +-
 doc/user/cwl/cwl-extensions.html.textile.liquid    |  17 ++-
 doc/user/cwl/cwl-run-options.html.textile.liquid   |  20 +++
 doc/user/cwl/cwl-runner.html.textile.liquid        |   2 +-
 sdk/cwl/arvados_cwl/__init__.py                    |  65 +++++++-
 sdk/cwl/arvados_cwl/arv-cwl-schema.yml             |  35 +++++
 sdk/cwl/arvados_cwl/arvcontainer.py                |  34 ++++-
 sdk/cwl/arvados_cwl/crunch_script.py               |   2 +
 sdk/cwl/arvados_cwl/fsaccess.py                    |   7 +-
 sdk/cwl/arvados_cwl/pathmapper.py                  |  65 +++++---
 sdk/cwl/arvados_cwl/runner.py                      |  10 +-
 sdk/cwl/setup.py                                   |   5 +-
 .../cwl/tests/collection_per_tool/a.txt            |   0
 .../cwl/tests/collection_per_tool/b.txt            |   0
 .../cwl/tests/collection_per_tool/c.txt            |   0
 .../collection_per_tool/collection_per_tool.cwl    |  13 ++
 .../collection_per_tool_packed.cwl                 |  45 ++++++
 sdk/cwl/tests/collection_per_tool/step1.cwl        |  15 ++
 sdk/cwl/tests/collection_per_tool/step2.cwl        |  15 ++
 sdk/cwl/tests/test_container.py                    | 163 +++++++++++---------
 sdk/cwl/tests/test_pathmapper.py                   |  23 ++-
 sdk/cwl/tests/test_submit.py                       | 164 +++++++++++++++------
 sdk/cwl/tests/wf/expect_packed.cwl                 |   4 +-
 sdk/cwl/tests/wf/submit_wf_packed.cwl              |   2 +-
 sdk/python/arvados/arvfile.py                      |  12 +-
 sdk/python/arvados/commands/run.py                 |  68 +++++----
 sdk/python/tests/test_arvfile.py                   |   4 +-
 services/api/lib/crunch_dispatch.rb                |   2 +-
 services/api/test/unit/crunch_dispatch_test.rb     |   2 +-
 services/nodemanager/arvnodeman/jobqueue.py        |  18 ++-
 services/nodemanager/tests/test_jobqueue.py        |  36 ++++-
 37 files changed, 686 insertions(+), 243 deletions(-)
 copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/collection_per_tool/a.txt (100%)
 copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/collection_per_tool/b.txt (100%)
 copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/collection_per_tool/c.txt (100%)
 create mode 100644 sdk/cwl/tests/collection_per_tool/collection_per_tool.cwl
 create mode 100644 sdk/cwl/tests/collection_per_tool/collection_per_tool_packed.cwl
 create mode 100644 sdk/cwl/tests/collection_per_tool/step1.cwl
 create mode 100644 sdk/cwl/tests/collection_per_tool/step2.cwl

       via  028e052db597498ee5c1412b606fa178c621b3ca (commit)
       via  b5d352d6099b60db5dcdd9183dcab3e8e17d729e (commit)
       via  8e1efc4c74f86a986dd0c6f15f53777d0d2bf0bb (commit)
       via  505b5c4d23da212bf285b3279a756b4f02d75567 (commit)
       via  23c8dd0d911c4b238169f399ff455f2dab9f0b4f (commit)
       via  7c03ff7b9ddac8dc232a910364fcecd32f86e03d (commit)
       via  9d6fb27b6a0ef28062be37efdea20d1a4c7079d3 (commit)
       via  f7a665dcd8dfe96b49e63f3caa352f5e5f8b19a1 (commit)
       via  b436b04a6e0cd28eaa465bb928eea3195b114322 (commit)
       via  3cd9054aa2699473f1af45c165441d7276162b46 (commit)
       via  dbd91c317d54da7123bee4f2d496b440b2ba9393 (commit)
       via  d0811968704eb02e3e98e46345fadf22cf95b5ec (commit)
       via  0da65aef99133cb76661f56f2adac83a77ad8ac9 (commit)
       via  4dac499d30988fd8289a40ad4128d6d471037180 (commit)
       via  d0b58511f396325bf083463d460567bf6f76e468 (commit)
       via  5223e68fa0d1f1fc52c7285fa6844d7932ba9acb (commit)
       via  79b826befbc0b62a822e2d0c922e5c4ea0a3bb7c (commit)
       via  8ffcfd1523d2a023c0286976a1aed50fb70dea3a (commit)
       via  9a7ea56065ed47ef114d569f137ff50d1efffc48 (commit)
       via  932a318673b17b63a2f2d99a6ba170c908ef3b20 (commit)
       via  e578f043c4d3b45a08f15d365adea0386228a4bf (commit)
       via  1d77dec0f7bc1cca34d52288bd39ecd0a79be250 (commit)
       via  d451eabbcbb8d5885fb30c2c341d93204a008596 (commit)
       via  456dde006228871c522597abbc5fe8006e44ad4d (commit)
       via  bee4fa82c15b2188c2105fff3b52c305b38f04a6 (commit)
       via  180074677191e4f30f90e8b8390245ab356b1c3f (commit)
       via  6eae76355f346740cc50ddba4656239c507461ac (commit)
       via  f60885adec1fe9a7b92f30ffbd4f5bb112e1aeae (commit)
       via  72a4000d8dc81879afae18ca440ab2ffaf712823 (commit)
       via  df6578e918d8c29baf314e671c28a96aea1c1d00 (commit)
       via  4a1df4f3aed07a31da8570363b5eaceccf4dffa8 (commit)
       via  b310fc04b9e60b974f4bfc2c86a5c45ecd7c65f2 (commit)
       via  8645888f12c25edaaac8e03fb5691cfcfbcdb9b2 (commit)
       via  43dbdec9f0f6361e4705671bbdade34cd18adefa (commit)
       via  dd58470216d6a416e4d831a3b9c25bfdaa255fff (commit)
       via  67fbf00f0d868384f1585f2473b5f89455001638 (commit)
       via  2e83bfa0e38b87632871f91cead42c9ead3d7bb7 (commit)
       via  ccc2ef472f5048059194e67b973d663c07816de9 (commit)
       via  cee8f888b442448b2bffe243da2e9cc5dcf29b04 (commit)
      from  1ceeafd5411b47834b836019dfd21d4050158171 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 028e052db597498ee5c1412b606fa178c621b3ca
Author: Lucas Di Pentima <lucas at curoverse.com>
Date:   Tue May 30 17:56:07 2017 -0300

    11684: ArvadosFile.flush() now check if it is the only owner of a bufferblock before deleting it, so the extra argument is not required.
    On commit_all(), always check if the owner attribute is an instance of ArvadosFile before calling flush()
    Fixed a couple tests that were mocking bufferblock.owner so that they work with this new behavior.

diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 2c653d7..a1d87a4 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -799,7 +799,7 @@ class _BlockManager(object):
             if v.state() != _BufferBlock.COMMITTED and v.owner:
                 # Ignore blocks with a list of owners, as if they're not in COMMITTED
                 # state, they're already being committed asynchronously.
-                if not isinstance(v.owner, list):
+                if isinstance(v.owner, ArvadosFile):
                     v.owner.flush(sync=False)
 
         with self.lock:
@@ -824,7 +824,7 @@ class _BlockManager(object):
                     # of repacking small blocks, so don't delete it when flushing
                     # its owners, just do it after flushing them all.
                     for owner in v.owner:
-                        owner.flush(sync=True, delete_bufferblock=False)
+                        owner.flush(sync=True)
                     self.delete_bufferblock(k)
 
     def block_prefetch(self, locator):
@@ -1129,7 +1129,7 @@ class ArvadosFile(object):
         return len(data)
 
     @synchronized
-    def flush(self, sync=True, num_retries=0, delete_bufferblock=True):
+    def flush(self, sync=True, num_retries=0):
         """Flush the current bufferblock to Keep.
 
         :sync:
@@ -1155,8 +1155,10 @@ class ArvadosFile(object):
                         self.parent._my_block_manager().commit_bufferblock(bb, sync=True)
                     to_delete.add(s.locator)
                     s.locator = bb.locator()
-            if delete_bufferblock:
-                for s in to_delete:
+            for s in to_delete:
+                # Don't delete the bufferblock if it's owned by many files. It'll be
+                # deleted after all of its owners are flush()ed.
+                if self.parent._my_block_manager().get_bufferblock(s).owner is self:
                     self.parent._my_block_manager().delete_bufferblock(s)
 
         self.parent.notify(MOD, self.parent, self.name, (self, self))
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index a800913..d241f73 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -832,7 +832,7 @@ class BlockManagerTest(unittest.TestCase):
         mockkeep = mock.MagicMock()
         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
             bufferblock = blockmanager.alloc_bufferblock()
-            bufferblock.owner = mock.MagicMock()
+            bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile)
             def flush(sync=None):
                 blockmanager.commit_bufferblock(bufferblock, sync)
             bufferblock.owner.flush.side_effect = flush
@@ -863,7 +863,7 @@ class BlockManagerTest(unittest.TestCase):
         mockkeep.put.side_effect = arvados.errors.KeepWriteError("fail")
         with arvados.arvfile._BlockManager(mockkeep) as blockmanager:
             bufferblock = blockmanager.alloc_bufferblock()
-            bufferblock.owner = mock.MagicMock()
+            bufferblock.owner = mock.MagicMock(spec=arvados.arvfile.ArvadosFile)
             def flush(sync=None):
                 blockmanager.commit_bufferblock(bufferblock, sync)
             bufferblock.owner.flush.side_effect = flush

commit b5d352d6099b60db5dcdd9183dcab3e8e17d729e
Merge: 1ceeafd 8e1efc4
Author: Lucas Di Pentima <lucas at curoverse.com>
Date:   Tue May 30 14:07:41 2017 -0300

    11684: Merge branch 'master' into 11684-unsigned-locator-fix


-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list