[ARVADOS] created: 1f2fa77dfbcf03c5af8c504994bff2799acd922a

Git user git at public.curoverse.com
Tue Feb 23 11:21:01 EST 2016


        at  1f2fa77dfbcf03c5af8c504994bff2799acd922a (commit)


commit 1f2fa77dfbcf03c5af8c504994bff2799acd922a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Feb 23 11:20:47 2016 -0500

    8488: crunchrunner brings certificates with it to ensure that it can
    communicate with API server from inside the container.

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index e33cc64..9a792cb 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -23,8 +23,9 @@ from cwltool.process import get_feature
 logger = logging.getLogger('arvados.cwl-runner')
 logger.setLevel(logging.INFO)
 
-crunchrunner_pdh = "e9b79ec72c692982d59f3a438fb49df2+66"
+crunchrunner_pdh = "721abe848fd8e6e6d1c99b920e6b7a2c+140"
 crunchrunner_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner"
+certs_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/ca-certificates.crt"
 
 def arv_docker_get_image(api_client, dockerRequirement, pull_image):
     if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
@@ -311,12 +312,16 @@ class ArvCwlRunner(object):
             self.api.collections().get(uuid=crunchrunner_pdh).execute()
         except arvados.errors.ApiError as e:
             import httplib2
-            h = httplib2.Http('ca_certs': arvados.util.ca_certs_path())
+            h = httplib2.Http(ca_certs=arvados.util.ca_certs_path())
             resp, content = h.request(crunchrunner_download, "GET")
+            resp2, content2 = h.request(certs_download, "GET")
             with arvados.collection.Collection() as col:
                 with col.open("crunchrunner", "w") as f:
                     f.write(content)
-                col.save_new("crunchrunner binary")
+                with col.open("ca-certificates.crt", "w") as f:
+                    f.write(content2)
+
+                col.save_new("crunchrunner binary", ensure_unique_name=True)
 
         self.pipeline = self.api.pipeline_instances().create(body={"name": shortname(tool.tool["id"]),
                                                                    "components": {},
diff --git a/sdk/go/crunchrunner/crunchrunner.go b/sdk/go/crunchrunner/crunchrunner.go
index 8e24e18..081ba50 100644
--- a/sdk/go/crunchrunner/crunchrunner.go
+++ b/sdk/go/crunchrunner/crunchrunner.go
@@ -1,13 +1,17 @@
 package main
 
 import (
+	"crypto/x509"
 	"fmt"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"git.curoverse.com/arvados.git/sdk/go/keepclient"
+	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"os/exec"
 	"os/signal"
+	"path"
 	"strings"
 	"syscall"
 )
@@ -317,6 +321,15 @@ func main() {
 		log.Fatal(err)
 	}
 
+	certpath := path.Join(path.Dir(os.Args[0]), "ca-certificates.crt")
+	certdata, err := ioutil.ReadFile(certpath)
+	if err == nil {
+		log.Printf("Using TLS certificates at %v", certpath)
+		certs := x509.NewCertPool()
+		certs.AppendCertsFromPEM(certdata)
+		api.Client.Transport.(*http.Transport).TLSClientConfig.RootCAs = certs
+	}
+
 	jobUuid := os.Getenv("JOB_UUID")
 	taskUuid := os.Getenv("TASK_UUID")
 	tmpdir := os.Getenv("TASK_WORK")

commit decbc26fa2ba5141542ede084f58e7ebff870059
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Feb 23 09:44:51 2016 -0500

    8488: Set certificate path.

diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 671f897..e33cc64 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -6,6 +6,7 @@ import arvados.events
 import arvados.commands.keepdocker
 import arvados.commands.run
 import arvados.collection
+import arvados.util
 import cwltool.draft2tool
 import cwltool.workflow
 import cwltool.main
@@ -23,6 +24,7 @@ logger = logging.getLogger('arvados.cwl-runner')
 logger.setLevel(logging.INFO)
 
 crunchrunner_pdh = "e9b79ec72c692982d59f3a438fb49df2+66"
+crunchrunner_download = "https://cloud.curoverse.com/collections/download/qr1hi-4zz18-n3m1yxd0vx78jic/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner"
 
 def arv_docker_get_image(api_client, dockerRequirement, pull_image):
     if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
@@ -307,10 +309,10 @@ class ArvCwlRunner(object):
 
         try:
             self.api.collections().get(uuid=crunchrunner_pdh).execute()
-        except arvados.errors.ApiError:
+        except arvados.errors.ApiError as e:
             import httplib2
-            h = httplib2.Http()
-            resp, content = h.request("https://cloud.curoverse.com/collections/download/%s/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner" % (crunchrunner_pdh), "GET")
+            h = httplib2.Http('ca_certs': arvados.util.ca_certs_path())
+            resp, content = h.request(crunchrunner_download, "GET")
             with arvados.collection.Collection() as col:
                 with col.open("crunchrunner", "w") as f:
                     f.write(content)
@@ -364,7 +366,6 @@ class ArvCwlRunner(object):
 
 
 def main(args, stdout, stderr, api_client=None):
-    runner = ArvCwlRunner(api_client=arvados.api('v1'))
     args.insert(0, "--leave-outputs")
     parser = cwltool.main.arg_parser()
     exgroup = parser.add_mutually_exclusive_group()
@@ -375,4 +376,10 @@ def main(args, stdout, stderr, api_client=None):
                         default=False, dest="enable_reuse",
                         help="")
 
+    try:
+        runner = ArvCwlRunner(api_client=arvados.api('v1'))
+    except Exception as e:
+        logger.error(e)
+        return 1
+
     return cwltool.main.main(args, executor=runner.arvExecutor, makeTool=runner.arvMakeTool, parser=parser)

commit 8b72e31d7cf0131d672bbb7ef6a5e1cb5ca834ec
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Feb 19 12:01:56 2016 -0500

    8488: Use crunchrunner binary instead of git repo

diff --git a/crunch_scripts/crunchrunner b/crunch_scripts/crunchrunner
new file mode 100755
index 0000000..71c10c9
--- /dev/null
+++ b/crunch_scripts/crunchrunner
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec $TASK_KEEPMOUNT/$JOB_PARAMETER_CRUNCHRUNNER
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8370e3d..671f897 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -5,6 +5,7 @@ import arvados
 import arvados.events
 import arvados.commands.keepdocker
 import arvados.commands.run
+import arvados.collection
 import cwltool.draft2tool
 import cwltool.workflow
 import cwltool.main
@@ -21,6 +22,8 @@ from cwltool.process import get_feature
 logger = logging.getLogger('arvados.cwl-runner')
 logger.setLevel(logging.INFO)
 
+crunchrunner_pdh = "e9b79ec72c692982d59f3a438fb49df2+66"
+
 def arv_docker_get_image(api_client, dockerRequirement, pull_image):
     if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement:
         dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
@@ -142,9 +145,9 @@ class ArvadosJob(object):
         try:
             response = self.arvrunner.api.jobs().create(body={
                 "script": "crunchrunner",
-                "repository": kwargs["repository"],
-                "script_version": "master",
-                "script_parameters": {"tasks": [script_parameters]},
+                "repository": "arvados",
+                "script_version": "8488-cwl-crunchrunner-collection", #"master",
+                "script_parameters": {"tasks": [script_parameters], "crunchrunner": crunchrunner_pdh+"/crunchrunner"},
                 "runtime_constraints": runtime_constraints
             }, find_or_create=kwargs.get("enable_reuse", True)).execute(num_retries=self.arvrunner.num_retries)
 
@@ -302,6 +305,17 @@ class ArvCwlRunner(object):
     def arvExecutor(self, tool, job_order, input_basedir, args, **kwargs):
         events = arvados.events.subscribe(arvados.api('v1'), [["object_uuid", "is_a", "arvados#job"]], self.on_message)
 
+        try:
+            self.api.collections().get(uuid=crunchrunner_pdh).execute()
+        except arvados.errors.ApiError:
+            import httplib2
+            h = httplib2.Http()
+            resp, content = h.request("https://cloud.curoverse.com/collections/download/%s/1i1u2qtq66k1atziv4ocfgsg5nu5tj11n4r6e0bhvjg03rix4m/crunchrunner" % (crunchrunner_pdh), "GET")
+            with arvados.collection.Collection() as col:
+                with col.open("crunchrunner", "w") as f:
+                    f.write(content)
+                col.save_new("crunchrunner binary")
+
         self.pipeline = self.api.pipeline_instances().create(body={"name": shortname(tool.tool["id"]),
                                                                    "components": {},
                                                                    "state": "RunningOnClient"}).execute(num_retries=self.num_retries)
@@ -310,7 +324,6 @@ class ArvCwlRunner(object):
 
         kwargs["fs_access"] = self.fs_access
         kwargs["enable_reuse"] = args.enable_reuse
-        kwargs["repository"] = args.repository
 
         if kwargs.get("conformance_test"):
             return cwltool.main.single_job_executor(tool, job_order, input_basedir, args, **kwargs)
@@ -362,6 +375,4 @@ def main(args, stdout, stderr, api_client=None):
                         default=False, dest="enable_reuse",
                         help="")
 
-    parser.add_argument('--repository', type=str, default="peter/crunchrunner", help="Repository containing the 'crunchrunner' program.")
-
     return cwltool.main.main(args, executor=runner.arvExecutor, makeTool=runner.arvMakeTool, parser=parser)

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list