[ARVADOS] created: 162766a3f8e91024f0f40fbc3d5955b899df73a9
Git user
git at public.curoverse.com
Mon Oct 17 17:06:24 EDT 2016
at 162766a3f8e91024f0f40fbc3d5955b899df73a9 (commit)
commit 162766a3f8e91024f0f40fbc3d5955b899df73a9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Oct 17 17:06:17 2016 -0400
10194: Create job with pinned cwl version.
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 03e74ef..e4889b3 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -17,6 +17,7 @@ from .runner import Runner, arvados_jobs_image
from .pathmapper import InitialWorkDirPathMapper
from .perf import Perf
from . import done
+from ._version import __version__
logger = logging.getLogger('arvados.cwl-runner')
metrics = logging.getLogger('arvados.cwl-runner.metrics')
@@ -245,7 +246,7 @@ class RunnerJob(Runner):
self.job_order["arv:output_name"] = self.output_name
return {
"script": "cwl-runner",
- "script_version": "master",
+ "script_version": __version__,
"repository": "arvados",
"script_parameters": self.job_order,
"runtime_constraints": {
@@ -256,10 +257,18 @@ class RunnerJob(Runner):
def run(self, *args, **kwargs):
job_spec = self.arvados_job_spec(*args, **kwargs)
+ job_spec.setdefault("owner_uuid", self.arvrunner.project_uuid)
+
+ job = self.arvrunner.api.jobs().create(
+ body=job_spec,
+ find_or_create=self.enable_reuse
+ ).execute(num_retries=self.arvrunner.num_retries)
+
for k,v in job_spec["script_parameters"].items():
if isinstance(v, dict):
job_spec["script_parameters"][k] = {"value": v}
+ job_spec["job"] = job
self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().create(
body={
"owner_uuid": self.arvrunner.project_uuid,
@@ -272,16 +281,6 @@ class RunnerJob(Runner):
self.uuid = self.arvrunner.pipeline["uuid"]
return
- job = None
- while not job:
- time.sleep(2)
- self.arvrunner.pipeline = self.arvrunner.api.pipeline_instances().get(
- uuid=self.arvrunner.pipeline["uuid"]).execute(
- num_retries=self.arvrunner.num_retries)
- job = self.arvrunner.pipeline["components"]["cwl-runner"].get("job")
- if not job and self.arvrunner.pipeline["state"] != "RunningOnServer":
- raise WorkflowException("Submitted pipeline is %s" % (self.arvrunner.pipeline["state"]))
-
self.uuid = job["uuid"]
self.arvrunner.processes[self.uuid] = self
commit f99aa5b85a1aa3bff1f1140f6a109fd308b03d30
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Oct 17 15:02:31 2016 -0400
10194: arvados/jobs a-c-r pinning WIP
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 1a87217..faafe4d 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -383,7 +383,7 @@ def versionstring():
arvpkg = pkg_resources.require("arvados-python-client")
cwlpkg = pkg_resources.require("cwltool")
- return "%s %s, %s %s, %s %s" % (sys.argv[0], arvcwlpkg[0].version,
+ return "%s %s %s, %s %s, %s %s" % (sys.argv[0], __version__, arvcwlpkg[0].version,
"arvados-python-client", arvpkg[0].version,
"cwltool", cwlpkg[0].version)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index a3220f9..58b011f 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -11,7 +11,7 @@ import arvados.collection
from .arvdocker import arv_docker_get_image
from . import done
-from .runner import Runner
+from .runner import Runner, arvados_jobs_image
logger = logging.getLogger('arvados.cwl-runner')
@@ -79,7 +79,7 @@ class ArvadosContainer(object):
(docker_req, docker_is_req) = get_feature(self, "DockerRequirement")
if not docker_req:
- docker_req = {"dockerImageId": "arvados/jobs"}
+ docker_req = {"dockerImageId": arvados_jobs_image(self.arvrunner)}
container_request["container_image"] = arv_docker_get_image(self.arvrunner.api,
docker_req,
@@ -180,7 +180,7 @@ class RunnerContainer(Runner):
jobpath = "/var/lib/cwl/job/cwl.input.json"
container_image = arv_docker_get_image(self.arvrunner.api,
- {"dockerImageId": "arvados/jobs"},
+ {"dockerImageId": arvados_jobs_image(self.arvrunner)},
pull_image,
self.arvrunner.project_uuid)
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 8269eee..03e74ef 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -13,7 +13,7 @@ from cwltool.builder import Builder
import arvados.collection
from .arvdocker import arv_docker_get_image
-from .runner import Runner
+from .runner import Runner, arvados_jobs_image
from .pathmapper import InitialWorkDirPathMapper
from .perf import Perf
from . import done
@@ -88,7 +88,7 @@ class ArvadosJob(object):
if docker_req and kwargs.get("use_container") is not False:
runtime_constraints["docker_image"] = arv_docker_get_image(self.arvrunner.api, docker_req, pull_image, self.arvrunner.project_uuid)
else:
- runtime_constraints["docker_image"] = "arvados/jobs"
+ runtime_constraints["docker_image"] = arvados_jobs_image(self.arvrunner)
resources = self.builder.resources
if resources is not None:
@@ -249,7 +249,7 @@ class RunnerJob(Runner):
"repository": "arvados",
"script_parameters": self.job_order,
"runtime_constraints": {
- "docker_image": "arvados/jobs"
+ "docker_image": arvados_jobs_image(self.arvrunner)
}
}
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 054d353..da9f07b 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -18,6 +18,7 @@ import ruamel.yaml as yaml
from .arvdocker import arv_docker_get_image
from .pathmapper import ArvPathMapper
+from ._version import __version__
logger = logging.getLogger('arvados.cwl-runner')
@@ -136,6 +137,13 @@ def upload_instance(arvrunner, name, tool, job_order):
return workflowmapper
+def arvados_jobs_image(arvrunner):
+ img = "arvados/jobs:"+__version__
+ try:
+ arv_docker_get_image(arvrunner.api, {"dockerPull": img}, True, arvrunner.project_uuid)
+ except Exception as e:
+ raise Exception("Docker image %s is not available\n%s" % (img, e) )
+ return img
class Runner(object):
def __init__(self, runner, tool, job_order, enable_reuse, output_name):
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index f9cf608..c9875a4 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -2,6 +2,7 @@
import os
import sys
+import subprocess
import setuptools.command.egg_info as egg_info_cmd
from setuptools import setup, find_packages
@@ -21,7 +22,8 @@ try:
'--format=format:%H']).strip()
with open("arvados_cwl/_version.py", "w") as f:
f.write("__version__ = '%s'\n" % gitinfo)
-except:
+except Exception as e:
+ print e
pass
setup(name='arvados-cwl-runner',
commit e144534106f2031b4a9bec0d80eebf094e33a285
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Mon Oct 17 13:48:39 2016 -0400
10194: Record git version in package
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7ebb13f..1a87217 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -30,6 +30,7 @@ from .arvworkflow import ArvadosWorkflow, upload_workflow
from .fsaccess import CollectionFsAccess
from .perf import Perf
from .pathmapper import FinalOutputPathMapper
+from ._version import __version__
from cwltool.pack import pack
from cwltool.process import shortname, UnsupportedRequirement, getListing
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 43cae78..f9cf608 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -15,6 +15,15 @@ try:
except ImportError:
tagger = egg_info_cmd.egg_info
+try:
+ gitinfo = subprocess.check_output(
+ ['git', 'log', '--first-parent', '--max-count=1',
+ '--format=format:%H']).strip()
+ with open("arvados_cwl/_version.py", "w") as f:
+ f.write("__version__ = '%s'\n" % gitinfo)
+except:
+ pass
+
setup(name='arvados-cwl-runner',
version='1.0',
description='Arvados Common Workflow Language runner',
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list