[ARVADOS] updated: e892c7ee96f28bef7d5b2a9314eb9549ee56634d
Git user
git at public.curoverse.com
Tue Oct 11 13:56:10 EDT 2016
Summary of changes:
.../app/controllers/application_controller.rb | 7 +-
.../app/views/projects/_show_dashboard.html.erb | 43 +++++--
apps/workbench/config/application.default.yml | 6 +
apps/workbench/test/helpers/share_object_helper.rb | 8 +-
.../test/integration/application_layout_test.rb | 5 +-
.../test/integration/pipeline_instances_test.rb | 8 +-
apps/workbench/test/integration/users_test.rb | 3 +-
apps/workbench/test/integration/work_units_test.rb | 6 +-
apps/workbench/test/integration_helper.rb | 18 ++-
apps/workbench/test/unit/work_unit_test.rb | 2 +-
build/run-build-packages.sh | 4 +-
crunch_scripts/cwl-runner | 40 ++----
.../_container_runtime_constraints.liquid | 1 +
.../install-dispatch.html.textile.liquid | 33 ++++-
sdk/cwl/arvados_cwl/__init__.py | 95 ++++++++++++--
sdk/cwl/arvados_cwl/arv-cwl-schema.yml | 38 ++++++
sdk/cwl/arvados_cwl/arvcontainer.py | 16 ++-
sdk/cwl/arvados_cwl/arvjob.py | 22 +++-
sdk/cwl/arvados_cwl/fsaccess.py | 6 +-
sdk/cwl/arvados_cwl/pathmapper.py | 28 +++-
sdk/cwl/arvados_cwl/runner.py | 10 +-
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/test_container.py | 10 +-
sdk/cwl/tests/test_job.py | 125 +++++++++---------
sdk/cwl/tests/test_make_output.py | 58 +++++++++
sdk/cwl/tests/test_submit.py | 22 ++--
sdk/go/arvados/container.go | 7 +-
sdk/go/crunchrunner/crunchrunner.go | 69 ++++++++--
sdk/go/crunchrunner/crunchrunner_test.go | 28 ++++
sdk/python/arvados/arvfile.py | 12 +-
services/api/app/models/container.rb | 27 ++++
services/api/app/models/container_request.rb | 12 +-
services/api/config/application.default.yml | 7 +
.../migrate/20160926194129_add_container_count.rb | 9 ++
services/api/db/structure.sql | 7 +-
services/api/test/fixtures/container_requests.yml | 20 +++
services/api/test/unit/container_request_test.rb | 44 ++++++-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 5 +-
.../crunch-dispatch-slurm_test.go | 16 ++-
services/crunch-dispatch-slurm/squeue.go | 2 +-
services/crunch-dispatch-slurm/usage.go | 1 +
.../crunchstat_summary/summarizer.py | 141 +++++++++++----------
42 files changed, 772 insertions(+), 251 deletions(-)
create mode 100644 sdk/cwl/tests/test_make_output.py
create mode 100644 services/api/db/migrate/20160926194129_add_container_count.rb
via e892c7ee96f28bef7d5b2a9314eb9549ee56634d (commit)
via f3d4ec1d93cadb51fb55355caa8c08f453bc4c21 (commit)
via 9f35a3cf10e5d261d83556d5b33f242966101f02 (commit)
via 87473fd874bfc6fe954089b74bef915543a48d60 (commit)
via fd36117b416e1190334d108058dc0c16ef24e964 (commit)
via 24bcfa0b87b87e4510fffe8a961a5d4a9fd34948 (commit)
via 77e64115189779a3d91f968946dfa82fa504d4c1 (commit)
via 18de85806e3717421accb89b093fc2bd56822100 (commit)
via 62a5bae0cf096fee69add487cd28a8bdbd4fa17c (commit)
via 25236212b8f726335ede5d4aa5da120b81ef19a4 (commit)
via f853c2c7f2283b57a4f215aeb288b2842e3d1843 (commit)
via f678c1f5a1b03a49eb0a40f614990b0fbcad3436 (commit)
via a5ce43cd7493e17fe1bb5f67451957b283881228 (commit)
via d68cb1845e82f44ecf126ae4fd1ba3abc00f4463 (commit)
via e9bf0a8dec771a53818ee76026c8bcc93a4722da (commit)
via 23b1c79162bd422f99e4d7a0e491d82da560d91f (commit)
via 304273fabb09ff72e94481a610e84f78776f05e3 (commit)
via f4c5d28ce98099b2f4204b5875449d76609c6a1c (commit)
via 2f86d9ac26cad9cd202ccfe9cf07c31ecbabcf7e (commit)
via 4c8559157a16bf3d2781714fdec0275fb7f53ac6 (commit)
via 38f17f1e09e1eb2af273148744ca193d2541d37e (commit)
via a4ef9968a1b59b99efac934076b8216f3b2f499a (commit)
via d70adee422f8f68debdf4425fea8afd53a488556 (commit)
via 47e42f1129363c2565e69c36ff26ce9c42731fb8 (commit)
via 78995d3bf5b9e160de50318bc83744dda92e4211 (commit)
via 6757824d984496469cd929cdbb62643c1cede7fd (commit)
via e25a377ed38698cb3eebbd0f43b80a5276595240 (commit)
via a894a61cc9004e184bc63f9f0a785f555d6c6829 (commit)
via 30d4c3f8f1d4d303c66fc568b11c34be7c51f2a8 (commit)
via a605fc29fd7f79b2882625c99a72e998157fa5bc (commit)
via 8d0f1cbda7cb58b15923813d5fca0448cb7b330e (commit)
via f35e22ef4abc012cad2260149657b95bfbcbac5d (commit)
via 8317b6e41e86f3e5c988906e0a9965479acf7e51 (commit)
via 74ec5b86db46257f75ea1eb94c136ce18e65c906 (commit)
via e68177bfa8de36bd54f705e16e5c4854eafb29a2 (commit)
via a3562a62e6b23035543f08226479813d5d1c31ed (commit)
via d5cc11240819c79a14197a37489964e0adf173be (commit)
via a3da8e980ccddf8312cec3e431517c212c964cd7 (commit)
via 9486b15595a667742ef541d3f31f78507afea8e7 (commit)
via 50d2969bdaf2fbbfbe00d35ec8339b166eee8001 (commit)
via 929e817fc1c91e1d9a0afcfcab93fe07304d9be5 (commit)
via 12e22cbf3bbf047c001624be54ec802999bb5c1e (commit)
via 0d3dd79bff8665c36e2442a76a9f7bb700702101 (commit)
via 84d86c1721bf549c2dc38df95f29f3579b36a5ae (commit)
via 068b2c9dbe1e467119680cbab61c62a03ad4fafd (commit)
via 6c0bf267d795a3ca49c3258c9490714c9e18d333 (commit)
via 8e6d577f1381da0d42481ef8dd0479241e3c50bf (commit)
via bd7dc68d12f1d134106fd21afe9cbf2d0512c831 (commit)
via c7a461e97640697e390aff96bf25126972396b19 (commit)
via ffe1fe1c77743147ee82aacdc50edde3672cd748 (commit)
via 0e4e5d028290bc8d5a4ffe5469a6993f640027a6 (commit)
via d182d19257f6cbe71fd75c29e1a81ba59a2aa79c (commit)
via f300b641735aa2657cfdcc4303050095c3bd60d5 (commit)
via 83139cd2df6b7d86f352bee46d1ad86146bd093b (commit)
via f72b0e8bcc350966ce54954711bed538c527eb00 (commit)
via ad4344e21db3c894869d09fdcb09e3202bf9cf78 (commit)
via a229ee1c2da50d247c5e7de12096f0f9d6ea3943 (commit)
via 27b732adbd4d2331436a433394658a0cfcf5fd94 (commit)
via 4da754586b4529a4480951218a69ec6f41fee98d (commit)
via 84f3d7048e63fc88b39f329ac5472238df2edc3d (commit)
via 5905f1299f3a55519d1c79991e405ae8d47711f9 (commit)
via eabe4efe7b0964f537f2e09fb0d820272f1c86e7 (commit)
via 986cdafd30a7bef4ccfbd18f808be52b2f6a458e (commit)
via bc67ce8d098c36e999739a9213b29bb72af6f23d (commit)
via 1ce0bd54f40a686959e95e63f7aaa8b233269932 (commit)
via 54cd184005d14ba93e93ae0eeb16ea70ee09a3b6 (commit)
via 49987769d924c1bc77cbdc9e9b182c3e2cc09b2d (commit)
via 3fb81a4db7abbaaa67b7a18d1c4a5ce82bc232dc (commit)
via a67a0dc7bf8946ea97157d32dc615f81d24e629b (commit)
from 31568ea331306a574be758fc60b090ecee3bc005 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e892c7ee96f28bef7d5b2a9314eb9549ee56634d
Merge: f3d4ec1 9f35a3c
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue Oct 11 14:54:14 2016 -0300
9701: Merge branch 'master' into 9701-collection-pack-small-files-alt
commit f3d4ec1d93cadb51fb55355caa8c08f453bc4c21
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue Oct 11 14:52:22 2016 -0300
9701: Wrapping actual alloc_bufferblock and delete_bufferblock methods inside synchronized calls, to be able to use them from within already synchronized BlockManager methods.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 5bac10e..1ca7ad8 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -432,6 +432,9 @@ class _BlockManager(object):
ArvadosFile that owns this block
"""
+ return self._alloc_bufferblock(blockid, starting_capacity, owner)
+
+ def _alloc_bufferblock(self, blockid=None, starting_capacity=2**14, owner=None):
if blockid is None:
blockid = "%s" % uuid.uuid4()
bufferblock = _BufferBlock(blockid, starting_capacity=starting_capacity, owner=owner)
@@ -566,8 +569,7 @@ class _BlockManager(object):
# Check if there are enough small blocks for filling up one in full
pending_write_size = sum([b.size() for b in small_blocks])
if force or (pending_write_size >= config.KEEP_BLOCK_SIZE):
- new_bb = _BufferBlock("%s" % uuid.uuid4(), 2**14, None)
- self._bufferblocks[new_bb.blockid] = new_bb
+ new_bb = self._alloc_bufferblock()
while len(small_blocks) > 0 and (new_bb.write_pointer + small_blocks[0].size()) <= config.KEEP_BLOCK_SIZE:
bb = small_blocks.pop(0)
arvfile = bb.owner
@@ -576,8 +578,7 @@ class _BlockManager(object):
0,
bb.size(),
new_bb.write_pointer - bb.size())])
- bb.clear()
- del self._bufferblocks[bb.blockid]
+ self._delete_bufferblock(bb.blockid)
self.commit_bufferblock(new_bb, sync=sync)
def commit_bufferblock(self, block, sync):
@@ -629,6 +630,9 @@ class _BlockManager(object):
@synchronized
def delete_bufferblock(self, locator):
+ self._delete_bufferblock(locator)
+
+ def _delete_bufferblock(self, locator):
bb = self._bufferblocks[locator]
bb.clear()
del self._bufferblocks[locator]
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list