[ARVADOS] created: 1f2fa77dfbcf03c5af8c504994bff2799acd922a
Git user
git at public.curoverse.com
Tue Feb 23 11:21:01 EST 2016
at 1f2fa77dfbcf03c5af8c504994bff2799acd922a (commit)
commit 1f2fa77dfbcf03c5af8c504994bff2799acd922a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Feb 23 11:20:47 2016 -0500
8488: crunchrunner brings certificates with it to ensure that it can
communicate with API server from inside the container.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index e33cc64..9a792cb 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -23,8 +23,9 @@ from cwltool.process import get_feature
logger = logging.getLogger('arvados.cwl-runner')
logger.setLevel(logging.INFO)
-crunchrunner_pdh = "e9b79ec72c692982d59f3a438fb49df2+66"
+crunchrunner_pdh = "721abe848fd8e6e6d1c99b920e6b7a2c+140"
crunchrunner_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner"
+certs_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/ca-certificates.crt"
def arv_docker_get_image(api_client, dockerRequirement, pull_image):
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
@@ -311,12 +312,16 @@ class ArvCwlRunner(object):
self.api.collections().get(uuid=crunchrunner_pdh).execute()
except arvados.errors.ApiError as e:
import httplib2
- h = httplib2.Http('ca_certs': arvados.util.ca_certs_path())
+ h = httplib2.Http(ca_certs=arvados.util.ca_certs_path())
resp, content = h.request(crunchrunner_download, "GET")
+ resp2, content2 = h.request(certs_download, "GET")
with arvados.collection.Collection() as col:
with col.open("crunchrunner", "w") as f:
f.write(content)
- col.save_new("crunchrunner binary")
+ with col.open("ca-certificates.crt", "w") as f:
+ f.write(content2)
+
+ col.save_new("crunchrunner binary", ensure_unique_name=True)
self.pipeline = self.api.pipeline_instances().create(body={"name": shortname(tool.tool["id"]),
"components": {},
diff --git a/sdk/go/crunchrunner/crunchrunner.go b/sdk/go/crunchrunner/crunchrunner.go
index 8e24e18..081ba50 100644
--- a/sdk/go/crunchrunner/crunchrunner.go
+++ b/sdk/go/crunchrunner/crunchrunner.go
@@ -1,13 +1,17 @@
package main
import (
+ "crypto/x509"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "io/ioutil"
"log"
+ "net/http"
"os"
"os/exec"
"os/signal"
+ "path"
"strings"
"syscall"
)
@@ -317,6 +321,15 @@ func main() {
log.Fatal(err)
}
+ certpath := path.Join(path.Dir(os.Args[0]), "ca-certificates.crt")
+ certdata, err := ioutil.ReadFile(certpath)
+ if err == nil {
+ log.Printf("Using TLS certificates at %v", certpath)
+ certs := x509.NewCertPool()
+ certs.AppendCertsFromPEM(certdata)
+ api.Client.Transport.(*http.Transport).TLSClientConfig.RootCAs = certs
+ }
+
jobUuid := os.Getenv("JOB_UUID")
taskUuid := os.Getenv("TASK_UUID")
tmpdir := os.Getenv("TASK_WORK")
commit decbc26fa2ba5141542ede084f58e7ebff870059
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Feb 23 09:44:51 2016 -0500
8488: Set certificate path.
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 671f897..e33cc64 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -6,6 +6,7 @@ import arvados.events
import arvados.commands.keepdocker
import arvados.commands.run
import arvados.collection
+import arvados.util
import cwltool.draft2tool
import cwltool.workflow
import cwltool.main
@@ -23,6 +24,7 @@ logger = logging.getLogger('arvados.cwl-runner')
logger.setLevel(logging.INFO)
crunchrunner_pdh = "e9b79ec72c692982d59f3a438fb49df2+66"
+crunchrunner_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner"
def arv_docker_get_image(api_client, dockerRequirement, pull_image):
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
@@ -307,10 +309,10 @@ class ArvCwlRunner(object):
try:
self.api.collections().get(uuid=crunchrunner_pdh).execute()
- except arvados.errors.ApiError:
+ except arvados.errors.ApiError as e:
import httplib2
- h = httplib2.Http()
- resp, content = h.request("https://cloud.curoverse.com/collections/download/%s/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner" % (crunchrunner_pdh), "GET")
+ h = httplib2.Http('ca_certs': arvados.util.ca_certs_path())
+ resp, content = h.request(crunchrunner_download, "GET")
with arvados.collection.Collection() as col:
with col.open("crunchrunner", "w") as f:
f.write(content)
@@ -364,7 +366,6 @@ class ArvCwlRunner(object):
def main(args, stdout, stderr, api_client=None):
- runner = ArvCwlRunner(api_client=arvados.api('v1'))
args.insert(0, "--leave-outputs")
parser = cwltool.main.arg_parser()
exgroup = parser.add_mutually_exclusive_group()
@@ -375,4 +376,10 @@ def main(args, stdout, stderr, api_client=None):
default=False, dest="enable_reuse",
help="")
+ try:
+ runner = ArvCwlRunner(api_client=arvados.api('v1'))
+ except Exception as e:
+ logger.error(e)
+ return 1
+
return cwltool.main.main(args, executor=runner.arvExecutor, makeTool=runner.arvMakeTool, parser=parser)
commit 8b72e31d7cf0131d672bbb7ef6a5e1cb5ca834ec
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Fri Feb 19 12:01:56 2016 -0500
8488: Use crunchrunner binary instead of git repo
diff --git a/crunch_scripts/crunchrunner b/crunch_scripts/crunchrunner
new file mode 100755
index 0000000..71c10c9
--- /dev/null
+++ b/crunch_scripts/crunchrunner
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec $TASK_KEEPMOUNT/$JOB_PARAMETER_CRUNCHRUNNER
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8370e3d..671f897 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -5,6 +5,7 @@ import arvados
import arvados.events
import arvados.commands.keepdocker
import arvados.commands.run
+import arvados.collection
import cwltool.draft2tool
import cwltool.workflow
import cwltool.main
@@ -21,6 +22,8 @@ from cwltool.process import get_feature
logger = logging.getLogger('arvados.cwl-runner')
logger.setLevel(logging.INFO)
+crunchrunner_pdh = "e9b79ec72c692982d59f3a438fb49df2+66"
+
def arv_docker_get_image(api_client, dockerRequirement, pull_image):
if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
@@ -142,9 +145,9 @@ class ArvadosJob(object):
try:
response = self.arvrunner.api.jobs().create(body={
"script": "crunchrunner",
- "repository": kwargs["repository"],
- "script_version": "master",
- "script_parameters": {"tasks": [script_parameters]},
+ "repository": "arvados",
+ "script_version": "8488-cwl-crunchrunner-collection", #"master",
+ "script_parameters": {"tasks": [script_parameters], "crunchrunner": crunchrunner_pdh+"/crunchrunner"},
"runtime_constraints": runtime_constraints
}, find_or_create=kwargs.get("enable_reuse", True)).execute(num_retries=self.arvrunner.num_retries)
@@ -302,6 +305,17 @@ class ArvCwlRunner(object):
def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
+ try:
+ self.api.collections().get(uuid=crunchrunner_pdh).execute()
+ except arvados.errors.ApiError:
+ import httplib2
+ h = httplib2.Http()
+ resp, content = h.request("https://cloud.curoverse.com/collections/download/%s/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner" % (crunchrunner_pdh), "GET")
+ with arvados.collection.Collection() as col:
+ with col.open("crunchrunner", "w") as f:
+ f.write(content)
+ col.save_new("crunchrunner binary")
+
self.pipeline = self.api.pipeline_instances().create(body={"name": shortname(tool.tool["id"]),
"components": {},
"state": "RunningOnClient"}).execute(num_retries=self.num_retries)
@@ -310,7 +324,6 @@ class ArvCwlRunner(object):
kwargs["fs_access"] = self.fs_access
kwargs["enable_reuse"] = args.enable_reuse
- kwargs["repository"] = args.repository
if kwargs.get("conformance_test"):
return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
@@ -362,6 +375,4 @@ def main(args, stdout, stderr, api_client=None):
default=False, dest="enable_reuse",
help="")
- parser.add_argument('--repository', type=str, default="peter/crunchrunner", help="Repository containing the 'crunchrunner' program.")
-
return cwltool.main.main(args, executor=runner.arvExecutor, makeTool=runner.arvMakeTool, parser=parser)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list