[ARVADOS] created: cbc620e6517c169fb265153eb6c995eebe959721

[Git user]

        at  cbc620e6517c169fb265153eb6c995eebe959721 (commit)


commit cbc620e6517c169fb265153eb6c995eebe959721
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Sep 22 12:29:33 2016 -0400

    9988: Add arvados-specfic hints RuntimeConstraints and APIRequirement.

diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
index 2e5044d..2a81ed4 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema.yml
@@ -12,3 +12,36 @@ $graph:
       jsonldPredicate:
         _id: "@type"
         _type: "@vocab"
+
+- name: RuntimeConstraints
+  type: record
+  doc: |
+    Set Arvados-specific runtime hints.
+  fields:
+    - name: class
+      type: string
+      doc: "Always 'arv:RuntimeConstraints'"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    - name: keep_cache
+      type: int?
+      doc: |
+        Size of file data buffer for Keep mount in MiB. Default is 256
+        MiB. Increase this to reduce cache thrashing in situtations such as
+        accessing multiple large (64+ MiB) files at the same time, or
+        performing random access on a large file.
+
+- name: APIRequirement
+  type: record
+  doc: |
+    Indicates that process wants to access to the Arvados API.  Will be granted
+    limited network access and have ARVADOS_API_HOST and ARVADOS_API_TOKEN set
+    in the environment.
+  fields:
+    - name: class
+      type: string
+      doc: "Always 'arv:APIRequirement'"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 413435d..aaae7d9 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -90,6 +90,14 @@ class ArvadosContainer(object):
             runtime_constraints["vcpus"] = resources.get("cores", 1)
             runtime_constraints["ram"] = resources.get("ram") * 2**20
 
+        api_req, _ = get_feature(self, "http://arvados.org/cwl#APIRequirement")
+        if api_req:
+            runtime_constraints["API"] = True
+
+        runtime_req, _ = get_feature(self, "http://arvados.org/cwl#RuntimeConstraints")
+        if runtime_req:
+            logger.warn("RuntimeConstraints not yet supported by container API")
+
         container_request["mounts"] = mounts
         container_request["runtime_constraints"] = runtime_constraints
 
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 78bca6e..1098002 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -85,6 +85,10 @@ class ArvadosJob(object):
             runtime_constraints["min_ram_mb_per_node"] = resources.get("ram")
             runtime_constraints["min_scratch_mb_per_node"] = resources.get("tmpdirSize", 0) + resources.get("outdirSize", 0)
 
+        runtime_req, _ = get_feature(self, "http://arvados.org/cwl#RuntimeConstraints")
+        if runtime_req:
+            runtime_constraints["keep_cache_mb_per_task"] = runtime_req["keep_cache"]
+
         filters = [["repository", "=", "arvados"],
                    ["script", "=", "crunchrunner"],
                    ["script_version", "in git", "9e5b98e8f5f4727856b53447191f9c06e3da2ba6"]]
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 7d941cf..822a213 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -86,6 +86,11 @@ class TestContainer(unittest.TestCase):
                 "coresMin": 3,
                 "ramMin": 3000,
                 "tmpdirMin": 4000
+            }, {
+                "class": "http://arvados.org/cwl#RuntimeConstraints",
+                "keep_cache": 512
+            }, {
+                "class": "http://arvados.org/cwl#APIRequirement",
             }],
             "baseCommand": "ls"
         }
@@ -107,7 +112,8 @@ class TestContainer(unittest.TestCase):
                 'name': 'test_resource_requirements',
                 'runtime_constraints': {
                     'vcpus': 3,
-                    'ram': 3145728000
+                    'ram': 3145728000,
+                    'API': True
                 }, 'priority': 1,
                 'mounts': {
                     '/var/spool/cwl': {'kind': 'tmp'}
diff --git a/sdk/cwl/tests/test_job.py b/sdk/cwl/tests/test_job.py
index fdb4520..e36a607 100644
--- a/sdk/cwl/tests/test_job.py
+++ b/sdk/cwl/tests/test_job.py
@@ -80,6 +80,11 @@ class TestJob(unittest.TestCase):
                 "coresMin": 3,
                 "ramMin": 3000,
                 "tmpdirMin": 4000
+            }, {
+                "class": "http://arvados.org/cwl#RuntimeConstraints",
+                "keep_cache": 512
+            }, {
+                "class": "http://arvados.org/cwl#APIRequirement",
             }],
             "baseCommand": "ls"
         }
@@ -107,7 +112,8 @@ class TestJob(unittest.TestCase):
                     'docker_image': 'arvados/jobs',
                     'min_cores_per_node': 3,
                     'min_ram_mb_per_node': 3000,
-                    'min_scratch_mb_per_node': 5024 # tmpdirSize + outdirSize
+                    'min_scratch_mb_per_node': 5024, # tmpdirSize + outdirSize
+                    'keep_cache_mb_per_task': 512
                 }
             },
             find_or_create=True,

-----------------------------------------------------------------------


hooks/post-receive
--