[ARVADOS] updated: 1.1.3-103-g88a29cd
Git user
git at public.curoverse.com
Mon Feb 26 15:11:53 EST 2018
Summary of changes:
.../app/assets/javascripts/components/sessions.js | 10 +-
.../app/assets/javascripts/work_unit_log.js | 2 +-
.../_show_object_description_cell.html.erb | 2 +-
.../views/container_requests/_state_label.html.erb | 8 +
apps/workbench/app/views/jobs/_show_log.html.erb | 4 +-
.../views/pipeline_instances/_show_log.html.erb | 4 +-
.../app/views/work_units/_show_log.html.erb | 4 +-
.../container_requests_controller_test.rb | 4 +-
build/run-tests.sh | 16 +-
doc/_config.yml | 2 +-
...ml.textile.liquid => index.html.textile.liquid} | 0
doc/sdk/index.html.textile.liquid | 2 +-
{services/ws => lib/dispatchcloud}/gocheck_test.go | 5 +-
lib/dispatchcloud/node_size.go | 128 +++++++++++
lib/dispatchcloud/node_size_test.go | 93 ++++++++
sdk/R/install_deps.R | 18 ++
sdk/cwl/arvados_cwl/__init__.py | 3 +
sdk/cwl/arvados_cwl/arvcontainer.py | 2 +
sdk/cwl/arvados_cwl/arvworkflow.py | 52 +++--
sdk/cwl/arvados_cwl/fsaccess.py | 4 +-
sdk/cwl/tests/test_job.py | 4 +-
sdk/cwl/tests/test_submit.py | 45 +++-
sdk/cwl/tests/wf/echo_a.cwl | 1 +
sdk/cwl/tests/wf/echo_b.cwl | 1 +
sdk/go/arvados/collection_fs_test.go | 2 +-
sdk/go/arvados/config.go | 14 ++
sdk/go/arvados/container.go | 6 +-
sdk/go/httpserver/logger.go | 29 ++-
sdk/go/httpserver/responsewriter.go | 7 +
sdk/python/arvados/arvfile.py | 3 +
sdk/python/arvados/collection.py | 10 +-
sdk/python/arvados/keep.py | 180 +++++++--------
sdk/python/tests/test_keep_client.py | 28 ++-
.../api/app/controllers/application_controller.rb | 4 +
.../app/controllers/arvados/v1/nodes_controller.rb | 33 ++-
services/api/app/models/collection.rb | 47 +++-
services/api/app/models/node.rb | 44 ++--
...216203422_add_storage_classes_to_collections.rb | 17 ++
services/api/db/structure.sql | 7 +-
services/api/test/fixtures/collections.yml | 42 ++++
.../api/test/functional/arvados/v1/filters_test.rb | 38 +++-
.../functional/arvados/v1/nodes_controller_test.rb | 42 ++++
services/api/test/unit/collection_test.rb | 75 +++++++
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 210 ++++++++++-------
.../crunch-dispatch-slurm_test.go | 248 ++++++++++++---------
services/crunch-dispatch-slurm/squeue.go | 3 +-
services/crunch-dispatch-slurm/usage.go | 3 +-
services/fuse/arvados_fuse/__init__.py | 27 ++-
services/fuse/arvados_fuse/fresh.py | 4 +
services/fuse/arvados_fuse/fusefile.py | 4 +
services/keepstore/azure_blob_volume.go | 171 ++++++++------
services/keepstore/azure_blob_volume_test.go | 22 +-
services/keepstore/bufferpool.go | 2 -
services/keepstore/bufferpool_test.go | 3 +-
services/keepstore/config.go | 105 +++++++--
services/keepstore/config_test.go | 3 +-
services/keepstore/handlers.go | 50 +++--
services/keepstore/keepstore.go | 7 +-
services/keepstore/mounts_test.go | 75 +++++--
services/keepstore/pull_worker.go | 2 -
services/keepstore/s3_volume.go | 7 +-
services/keepstore/s3_volume_test.go | 14 +-
services/keepstore/trash_worker.go | 1 -
services/keepstore/volume.go | 31 ++-
services/keepstore/volume_test.go | 4 +
services/keepstore/volume_unix.go | 8 +-
services/keepstore/volume_unix_test.go | 13 ++
.../arvnodeman/computenode/dispatch/__init__.py | 25 ++-
.../arvnodeman/computenode/dispatch/slurm.py | 56 +++--
.../arvnodeman/computenode/driver/gce.py | 4 +-
services/nodemanager/arvnodeman/daemon.py | 1 -
services/nodemanager/arvnodeman/jobqueue.py | 56 +++--
services/nodemanager/arvnodeman/nodelist.py | 33 +--
services/nodemanager/tests/integration_test.py | 22 +-
.../nodemanager/tests/test_computenode_dispatch.py | 30 +--
.../tests/test_computenode_dispatch_slurm.py | 21 +-
.../tests/test_computenode_driver_gce.py | 2 +-
services/nodemanager/tests/test_daemon.py | 9 +-
services/nodemanager/tests/test_jobqueue.py | 20 +-
services/nodemanager/tests/test_nodelist.py | 24 +-
services/nodemanager/tests/testutil.py | 4 +-
.../lib/arvbox/docker/service/doc/run-service | 14 +-
tools/sync-groups/sync-groups.go | 2 +-
vendor/vendor.json | 146 +++++++++++-
84 files changed, 1851 insertions(+), 677 deletions(-)
copy apps/workbench/app/views/{pipeline_instances => container_requests}/_show_object_description_cell.html.erb (64%)
create mode 100644 apps/workbench/app/views/container_requests/_state_label.html.erb
rename doc/sdk/R/{R.html.textile.liquid => index.html.textile.liquid} (100%)
copy {services/ws => lib/dispatchcloud}/gocheck_test.go (70%)
create mode 100644 lib/dispatchcloud/node_size.go
create mode 100644 lib/dispatchcloud/node_size_test.go
create mode 100644 sdk/R/install_deps.R
create mode 100644 services/api/db/migrate/20180216203422_add_storage_classes_to_collections.rb
via 88a29cd091468feb98e5cd541c560f4d35bca716 (commit)
via 5c60c8c56304a0204f4e94128fb3deaae9e9b707 (commit)
via 704dd826a498db11a052fe77a4953b0caa245c10 (commit)
via 05ececf6335372a971bc26ae3566b374131ce66e (commit)
via 281fe0c40c16aacd82cd1ca122e38aa3a3854f2e (commit)
via 68cde7e2a496c4e57cb1576bb8bf29415d3c0b67 (commit)
via a4959ce78b69f1f4e0e59e2458af8d11bc1d5510 (commit)
via 6a731b2dd982f3453db9f5b6fcf6350899da48a8 (commit)
via f594d47faa4c606ccbfda0e1bcaac58552e4732d (commit)
via b0d7f37097e8b5c56bfbfa79a134a8074789ebb4 (commit)
via 52f37db5889f22e4dc02e0e468d561783e76c188 (commit)
via 92002b9af320e3229f59ca0778d1906e663f3066 (commit)
via 6509f6ec0a997f74b9992b8ef9acb7ae0dbd8c7a (commit)
via 8af8ef37ba4f673e47ba17899a83e2a683872b55 (commit)
via b3f083ce49a50521d96daa28f896d857384222fb (commit)
via b77a0df5575eb4d75b21b5c400fc8570efc97cea (commit)
via cff9f271514a559f8d7d130dea7cdd5521a81973 (commit)
via 00693ff029257e72565148e8748941ac20cc99d1 (commit)
via 1632cb9ffffc882987c69d2e5f8b91f849fa9d8c (commit)
via da5b20e29680ad6a172ee7875001a1a9c348b674 (commit)
via c28d87a0916349bfcaacc774aa1c05582e1f0154 (commit)
via 86183e3763202c53eb547a5432d291be9ed2986c (commit)
via 8404fc27cc5904f90ef429a2a6e2fd43d5d49996 (commit)
via d9c9b4b6f8fc895d3c91d3b49380d6397576a10c (commit)
via 2727970da3aa2537df0b465bca1f08142208d4d9 (commit)
via de8503c9b82fd64f9f25a3fd1ea2b0653ea25727 (commit)
via 9ea9735d7569f23a89fbcfaa69710f209eea6466 (commit)
via 24dc417363fad4b80efda414b9d31dad0772bb69 (commit)
via 5c0c6c7daa4f38b26efa7db4e53a0016b45f6416 (commit)
via de9a366cbd68a92e0189e9432e5e473320f3a803 (commit)
via f53e3ede584a9354ec2d73f67b6800949b3b4079 (commit)
via ad5dcfa3ea9c1b6500b4de1ba0cf8bc0560188da (commit)
via 359de7e4f707e744356e98b14e620d15041c409d (commit)
via b9fbe39f4b4f35f63b6513801f9b143beb906596 (commit)
via f344ada0769688fa7676ade420fbc22bd8dfd10e (commit)
via 9c699bfa811f94b2cd7c3a82ad1197dc07029b17 (commit)
via 1ec75d021efae1ca197de2e460fd87a932766c9d (commit)
via 9333d60538c8dcd9312e7b2dddb56dcf25e52d91 (commit)
via e0c29fdbeba0b237a5ad7154ef890c43edeaa2df (commit)
via 707ed2964a24ebe940ae440af22fdcb56781fcbd (commit)
via 90483e26cf9f0ced8e5b12586ae3a5e1ec620add (commit)
via ba1cbe5824ddc769d8b1fdb1f51a4e02187e778c (commit)
via 4d52d030d964a986d28cc0f35f74624a41406fb0 (commit)
via ce5f1a58b22bcf19bcedfb5d8602290d4a3025ff (commit)
via 2320020a96ae6438482d1c8bbdf43d9c4fd06482 (commit)
via 2cac74ad82619e2866c7ca277dd8e4ff2d828f5b (commit)
via 76b7cb2fc14586db860cd54cd66fc3e9e438143b (commit)
via 03f277bf4b616f41ef7ed4b195d44dab83d16144 (commit)
via 66b3d102b0ce68a893e5a49ea5f3bb3b9952f6ef (commit)
via 34513cb0f3f880e3e91655446fdea2590ed0a579 (commit)
via 43540c6be0228ffd5c01cdfaefed35ddc604fe5e (commit)
via 371ce4b14c96e0f515586c55387d95c17e1670b3 (commit)
via 3d6d097f42128bde90b7bc184057a84e99ea3e0a (commit)
via de70e67d9bd06783f42ba5ab2408b5778e6573ff (commit)
via f35131cad37610298a6cdc52b4b213a29cb1e735 (commit)
via cfb929643f5f5d24fdc9f98c1c83c79ffd708f0e (commit)
via 3826a6339ba1c901c054053920ed20547b3ba54d (commit)
via c93eb8717c79922f3e68b13405b5250bb1345d72 (commit)
via 496a9058bc2960a89b66440f8d30f5e92de918db (commit)
via 85398442d095524c5b2f315c294ea81f9d17853b (commit)
via 20c51bfacc584f54083da0b3317116089e3af6b4 (commit)
via 3c5378e79de03e312268e31a10fc93650cafbb10 (commit)
via 3ccc70f4bb06bab6c0b3c71f555cba24cc5c6a47 (commit)
via 5120666e073018f6821bab07f0bbb788098f97b1 (commit)
via 80da19707253af74bd78c374bfcab64b04d2dbde (commit)
via ba01bc2ca5bb7351442f58709b7f85aa84a20342 (commit)
via a88a3f5710af6eb159299231779a1e4554aa772a (commit)
via 13288c69988380210150124d7991af6ca6e3d62b (commit)
via 2b41877b842bbf47a0fdea713c9d2fc2fb80664a (commit)
via 428e8f9663ba05eb28667e02d01675e271c38fdf (commit)
via d2369c0fb8742d54f0cdfc88ef485beff62ea277 (commit)
via bad6499910388c17c4a54b3128f361aa36a670e1 (commit)
via 10f08c358c12468119dc2621c48b68d6d33417da (commit)
via 0c9586ca048805b404dd762f5cd7cabe5d1ed227 (commit)
via bfa436c97990c1a0cf39907acc0235a8535b6a43 (commit)
via f9e1447e5ab21dbc0fdb08328c31811fca2fd327 (commit)
from 8a2035547ad8bf6abad6a4a03bb0b59211a00932 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 88a29cd091468feb98e5cd541c560f4d35bca716
Merge: 5c60c8c 704dd82
Author: jiayong2 <jiayong at math.mit.edu>
Date: Mon Feb 26 20:11:08 2018 +0000
Merge branch 'master' into 11850-singlecontainer-max-requirements
Arvados-DCO-1.1-Signed-off-by: Jiayong Li <jiayong at math.mit.edu>
commit 5c60c8c56304a0204f4e94128fb3deaae9e9b707
Author: jiayong2 <jiayong at math.mit.edu>
Date: Mon Feb 26 20:06:59 2018 +0000
Take sum for outdir requirement in single container
Arvados-DCO-1.1-Signed-off-by: Jiayong Li <jiayong at math.mit.edu>
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 15704db..5aed871 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -27,6 +27,9 @@ from .perf import Perf
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
+max_res_pars = ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdirMax")
+sum_res_pars = ("outdirMin", "outdirMax")
+
def upload_workflow(arvRunner, tool, job_order, project_uuid, uuid=None,
submit_runner_ram=0, name=None, merged_map=None):
@@ -72,17 +75,19 @@ def dedup_reqs(reqs):
dedup[r["class"]] = r
return [dedup[r] for r in sorted(dedup.keys())]
-def get_max_res_req(res_reqs):
- """Take the max of a list of ResourceRequirement."""
+def get_overall_res_req(res_reqs):
+ """Take the overall of a list of ResourceRequirement,
+ i.e., the max of coresMin, coresMax, ramMin, ramMax, tmpdirMin, tmpdirMax
+ and the sum of outdirMin, outdirMax."""
- total_res_req = {}
+ all_res_req = {}
exception_msgs = []
- for a in ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdirMax", "outdirMin", "outdirMax"):
- total_res_req[a] = []
+ for a in max_res_pars + sum_res_pars:
+ all_res_req[a] = []
for res_req in res_reqs:
if a in res_req:
if isinstance(res_req[a], int): # integer check
- total_res_req[a].append(res_req[a])
+ all_res_req[a].append(res_req[a])
else:
msg = SourceLine(res_req).makeError(
"Non-top-level ResourceRequirement in single container cannot have expressions")
@@ -90,13 +95,16 @@ def get_max_res_req(res_reqs):
if exception_msgs:
raise WorkflowException("\n".join(exception_msgs))
else:
- max_res_req = {}
- for a in total_res_req:
- if total_res_req[a]:
- max_res_req[a] = max(total_res_req[a])
- if max_res_req:
- max_res_req["class"] = "ResourceRequirement"
- return cmap(max_res_req)
+ overall_res_req = {}
+ for a in all_res_req:
+ if all_res_req[a]:
+ if a in max_res_pars:
+ overall_res_req[a] = max(all_res_req[a])
+ elif a in sum_res_pars:
+ overall_res_req[a] = sum(all_res_req[a])
+ if overall_res_req:
+ overall_res_req["class"] = "ResourceRequirement"
+ return cmap(overall_res_req)
class ArvadosWorkflow(Workflow):
"""Wrap cwltool Workflow to override selected methods."""
@@ -134,8 +142,8 @@ class ArvadosWorkflow(Workflow):
builder = Builder()
builder.job = joborder
- builder.requirements = self.requirements
- builder.hints = self.hints
+ builder.requirements = workflowobj["requirements"]
+ builder.hints = workflowobj["hints"]
builder.resources = {}
res_reqs = {"requirements": [], "hints": []}
@@ -146,7 +154,7 @@ class ArvadosWorkflow(Workflow):
for req in item[t]:
if req["class"] == "ResourceRequirement":
eval_req = {"class": "ResourceRequirement"}
- for a in ("coresMin", "coresMax", "ramMin", "ramMax", "tmpdirMin", "tmpdirMax", "outdirMin", "outdirMax"):
+ for a in max_res_pars + sum_res_pars:
if a in req:
eval_req[a] = builder.do_eval(req[a])
res_reqs[t].append(eval_req)
@@ -154,16 +162,16 @@ class ArvadosWorkflow(Workflow):
for req in item[t]:
if req["class"] == "ResourceRequirement":
res_reqs[t].append(req)
- max_res_req = {"requirements": get_max_res_req(res_reqs["requirements"]),
- "hints": get_max_res_req(res_reqs["hints"])}
+ overall_res_req = {"requirements": get_overall_res_req(res_reqs["requirements"]),
+ "hints": get_overall_res_req(res_reqs["hints"])}
new_spec = {"requirements": self.requirements, "hints": self.hints}
for t in ("requirements", "hints"):
for req in new_spec[t]:
if req["class"] == "ResourceRequirement":
new_spec[t].remove(req)
- if max_res_req[t]:
- new_spec[t].append(max_res_req[t])
+ if overall_res_req[t]:
+ new_spec[t].append(overall_res_req[t])
upload_dependencies(self.arvrunner,
kwargs.get("name", ""),
@@ -218,7 +226,7 @@ class ArvadosWorkflow(Workflow):
"inputs": self.tool["inputs"],
"outputs": self.tool["outputs"],
"stdout": "cwl.output.json",
- "requirements": new_spec["requirements"]+[
+ "requirements": self.requirements+[
{
"class": "InitialWorkDirRequirement",
"listing": [{
@@ -232,7 +240,7 @@ class ArvadosWorkflow(Workflow):
"entry": json.dumps(joborder_keepmount, indent=2, sort_keys=True, separators=(',',': ')).replace("\\", "\\\\").replace('$(', '\$(').replace('${', '\${')
}]
}],
- "hints": new_spec["hints"],
+ "hints": self.hints,
"arguments": ["--no-container", "--move-outputs", "--preserve-entire-environment", "workflow.cwl#main", "cwl.input.yml"],
"id": "#"
})
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index 8518986..5a6e6dc 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -397,7 +397,7 @@ class TestWorkflow(unittest.TestCase):
@mock.patch("arvados.collection.CollectionReader")
@mock.patch("arvados.collection.Collection")
@mock.patch('arvados.commands.keepdocker.list_images_in_arv')
- def test_max_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
+ def test_overall_resource_singlecontainer(self, list_images_in_arv, mockcollection, mockcollectionreader):
arvados_cwl.add_arv_hints()
api = mock.MagicMock()
@@ -455,7 +455,7 @@ class TestWorkflow(unittest.TestCase):
'command': [u'cwltool', u'--no-container', u'--move-outputs', u'--preserve-entire-environment', u'workflow.cwl#main', u'cwl.input.yml'],
'task.stdout': 'cwl.output.json'}]},
'runtime_constraints': {
- 'min_scratch_mb_per_node': 2048,
+ 'min_scratch_mb_per_node': 4096,
'min_cores_per_node': 3,
'docker_image': 'arvados/jobs',
'min_ram_mb_per_node': 1024
diff --git a/sdk/cwl/tests/wf/echo_a.cwl b/sdk/cwl/tests/wf/echo_a.cwl
index f16bb09..b7893e2 100644
--- a/sdk/cwl/tests/wf/echo_a.cwl
+++ b/sdk/cwl/tests/wf/echo_a.cwl
@@ -3,6 +3,7 @@ class: CommandLineTool
requirements:
ResourceRequirement:
coresMin: 2
+ outdirMin: 1024
inputs: []
outputs: []
baseCommand: echo
diff --git a/sdk/cwl/tests/wf/echo_b.cwl b/sdk/cwl/tests/wf/echo_b.cwl
index 1b22157..4db11cc 100644
--- a/sdk/cwl/tests/wf/echo_b.cwl
+++ b/sdk/cwl/tests/wf/echo_b.cwl
@@ -3,6 +3,7 @@ class: CommandLineTool
requirements:
ResourceRequirement:
coresMin: 3
+ outdirMin: 2048
inputs: []
outputs: []
baseCommand: echo
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list