[ARVADOS] updated: 5976c751048100e813edd638632927794df072f2
Git user
git at public.curoverse.com
Wed Mar 22 14:03:24 EDT 2017
Summary of changes:
.../app/controllers/application_controller.rb | 13 +-
.../workbench/app/controllers/search_controller.rb | 3 +
.../app/views/projects/_show_dashboard.html.erb | 12 +-
.../test/controllers/disabled_api_test.rb | 11 +
doc/_config.yml | 5 +
doc/_includes/_compute_ping_rb.liquid | 285 +++++++++++
.../install-compute-node.html.textile.liquid | 8 +
.../install-slurm.html.textile.liquid} | 96 +---
.../install-compute-ping.html.textile.liquid | 9 +
.../install-nodemanager.html.textile.liquid | 562 +++++++++++++++++++++
doc/install/migrate-docker19.html.textile.liquid | 31 ++
docker/migrate-docker19/Dockerfile | 31 ++
docker/migrate-docker19/build.sh | 2 +
.../docker/run => docker/migrate-docker19/dnd.sh | 1 -
docker/migrate-docker19/migrate.sh | 41 ++
sdk/cwl/arvados_cwl/arvdocker.py | 8 +-
sdk/cwl/tests/test_container.py | 6 +-
sdk/python/arvados/api.py | 2 +-
sdk/python/arvados/collection.py | 1 +
sdk/python/arvados/commands/keepdocker.py | 90 +---
sdk/python/arvados/commands/migrate19.py | 204 ++++++++
sdk/python/bin/arv-migrate-docker19 | 4 +-
sdk/python/tests/test_arv_keepdocker.py | 38 +-
.../controllers/arvados/v1/groups_controller.rb | 16 +
services/api/app/models/collection.rb | 101 ++--
services/api/app/models/container_request.rb | 2 +-
services/api/app/models/job.rb | 11 +-
services/api/config/application.default.yml | 1 +
.../api/test/functional/arvados/v1/filters_test.rb | 52 ++
services/api/test/unit/container_request_test.rb | 54 +-
services/api/test/unit/job_test.rb | 29 +-
services/crunch-run/crunchrun.go | 30 +-
services/crunch-run/crunchrun_test.go | 20 +
services/fuse/arvados_fuse/command.py | 4 +
services/fuse/tests/test_mount_type.py | 23 +
.../arvnodeman/computenode/dispatch/slurm.py | 2 +-
services/nodemanager/arvnodeman/daemon.py | 17 +-
tools/arvbox/bin/arvbox | 78 +--
.../arvbox/docker/service/websockets/run-service | 39 +-
39 files changed, 1621 insertions(+), 321 deletions(-)
create mode 100644 doc/_includes/_compute_ping_rb.liquid
copy doc/install/{install-crunch-dispatch.html.textile.liquid => crunch2-slurm/install-slurm.html.textile.liquid} (53%)
create mode 100644 doc/install/install-compute-ping.html.textile.liquid
create mode 100644 doc/install/install-nodemanager.html.textile.liquid
create mode 100644 doc/install/migrate-docker19.html.textile.liquid
create mode 100644 docker/migrate-docker19/Dockerfile
create mode 100755 docker/migrate-docker19/build.sh
copy tools/arvbox/lib/arvbox/docker/service/docker/run => docker/migrate-docker19/dnd.sh (98%)
create mode 100755 docker/migrate-docker19/migrate.sh
create mode 100644 sdk/python/arvados/commands/migrate19.py
create mode 100644 services/fuse/tests/test_mount_type.py
via 5976c751048100e813edd638632927794df072f2 (commit)
via b77893f2a8ae755f22615054f2c267d990995e1c (commit)
via db7c2ba7523fbce1dc47ea1a715a738a28b3bb3b (commit)
via 433cd703220880784c8dab316501d1c74b09e37b (commit)
via 03589784d8bc566305e1ea3473a8b37941147517 (commit)
via fd42d69ffa558918ec40f9aeaceb2cbf93de1e8a (commit)
via 2accec1f7ca3e2dd1e7fe9ee28bb6a8a6cb74abc (commit)
via de283bd8dd383a4177bb714212cf7c59f38a23d8 (commit)
via da8c9048bf4f3a36f5691aa52e56e2db672dff3a (commit)
via 47178c71032b476e34fd0c6b10065e1b96a568f1 (commit)
via bf03dd4f523463b896428527fa0685631e9ac649 (commit)
via 996b635700d7270229200a56d2c2b9f7c96a84fb (commit)
via b60a21fe3482e1f7ca1815e2a90e276a8c377636 (commit)
via 2c69d491fbbd1461bb5a6dd9c0f0dd493081b45e (commit)
via 73dbf5bf94b162e5e9ed74cb60f17ee2c7018059 (commit)
via 54837bcae94d30f37f19e4ee436c2bb96bd8f21a (commit)
via b1aa6c85c3db947f0963b1bddb2784c115f5c97d (commit)
via 98329b75276412e605d54b3a0f907a5cf204384f (commit)
via 91118e3af51c7fa655f14f2da99216f8ff0e11e0 (commit)
via bdaa9de9882fee122fd2274d92ea500113df8195 (commit)
via d363323bfea33aee9182e71fd93387b0c4f944a2 (commit)
via 449d780f0a470f67d1a28fe1459a8bab79562069 (commit)
via eabc13f27dfb178069546202bca772f893bac81c (commit)
via dee634b21e48e467b0d7b5ef1b2965c1df5e41d1 (commit)
via f074ef8181c2b93cbfcd870eb67c8c95cb0e442a (commit)
via 2ed5325b37efa7aa9d38a60c1d5c8b4980df8489 (commit)
via 55c719bff1b34d037506639fd4cf7f0a74f4c3cb (commit)
via 6058f336a4562b6d6780c4fcfc4d28b6b2a51948 (commit)
via 1d7577d08b6ca646fcca0e941ce614c37051264e (commit)
via 205b07766565f3ce7b80bcdfee88f83af451fb01 (commit)
via 0eb521827f6a64e567722527f36f0b4d130af504 (commit)
via 5fffad607e39e331f1051c35c5b70ecd5a30939a (commit)
via 355fe24bf5a4ac9e45372333e8bbc8e39b5b86a1 (commit)
via 25296def9590c8a29bdcf75c78a03444d39ca6b6 (commit)
via c04608e40b971b2fc342db8e5e44817b568a3a4c (commit)
via a13833894cda6bc1c872b3eb4215dec5c67aad62 (commit)
via 7eb2eff1864696bb51b2f4913499da5b62fb74ec (commit)
via 6ea807b2caf6c934f170b2e4d89c23c4a08ca69c (commit)
via c7515954731c6b7c0d8fb241ec4316a5e6d62d0c (commit)
via dbb0d22a96fa2de606430d78c05c254598f501df (commit)
via 918e16de43e993894dcaffbe0cafa0fc1d2f16b3 (commit)
via a72205728f94f5261b657766e01f5767dc15d4b5 (commit)
via e1e0bec5d9828fff1b9269322d415789675e6fab (commit)
via f4a7c5e6b1e6e30ad8a31a704642fcc360239e03 (commit)
via f63c160fd1cec8dddc673900f5cda824bed30d88 (commit)
via 605d11f27e73ce60794fa9858aeed8ed961261e8 (commit)
via fa8c5cdb36fa89af92087d6465b207d65ffc9618 (commit)
via 82697fea93b1c87cdee27d2b9a76c1b7ac07497e (commit)
via a15fc58f280171660afd2b9f1938f0a6bc5ede4a (commit)
via c7143dc92806d7b0c6f434e91744a04e5283284d (commit)
via 005595fb1a7da851a863118fba7e3d00026fb3a3 (commit)
via 2ab1b540a710fd454f33e9ba22db601e67f3d217 (commit)
via cad4856b84ff09f8d66faf16cc02ce68ef87995f (commit)
from b30e81ee46c196476a03bc244fbc4419573ba34e (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5976c751048100e813edd638632927794df072f2
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Mar 22 15:01:48 2017 -0300
10218: Get the container record directly from the API server, instead of Marshaling runner.Container.
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index a3338ec..63b63a1 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -33,6 +33,7 @@ type IArvadosClient interface {
Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error
Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error
Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error
+ CallRaw(method string, resourceType string, uuid string, action string, parameters arvadosclient.Dict) (reader io.ReadCloser, err error)
Discovery(key string) (interface{}, error)
}
@@ -128,7 +129,6 @@ func (runner *ContainerRunner) SetupSignals() {
go func(sig chan os.Signal) {
<-sig
- log.Print("signal handler calling runner.stop()")
runner.stop()
signal.Stop(sig)
}(runner.SigChan)
@@ -566,19 +566,27 @@ func (runner *ContainerRunner) LogContainerRecord() (err error) {
"container",
runner.LogCollection.Open("container.json"),
}
- logger := log.New(w, "container", 0)
-
- // Convert container record to pretty-printed JSON []byte
- rec, err := json.MarshalIndent(runner.Container, "", " ")
+ // Get Container record JSON from the API Server
+ reader, err := runner.ArvClient.CallRaw("GET", "containers", runner.Container.UUID, "", nil)
if err != nil {
- return fmt.Errorf("While converting container record to JSON: %v", err)
+ return fmt.Errorf("While retrieving container record from the API server: %v", err)
}
-
- // Write JSON record line-by-line
- for _, line := range strings.Split(string(rec), "\n") {
- logger.Println(line)
+ // Read the API server response as []byte
+ json_bytes, err := ioutil.ReadAll(reader)
+ if err != nil {
+ return fmt.Errorf("While reading container record API server response: %v", err)
+ }
+ // Decode the JSON []byte
+ var cr map[string]interface{}
+ if err = json.Unmarshal(json_bytes, &cr); err != nil {
+ return fmt.Errorf("While decoding the container record JSON response: %v", err)
+ }
+ // Re-encode it using indentation to improve readability
+ enc := json.NewEncoder(w)
+ enc.SetIndent("", " ")
+ if err = enc.Encode(cr); err != nil {
+ return fmt.Errorf("While logging the JSON container record: %v", err)
}
-
err = w.Close()
if err != nil {
return fmt.Errorf("While closing container.json log: %v", err)
diff --git a/services/crunch-run/crunchrun_test.go b/services/crunch-run/crunchrun_test.go
index c26830d..9b12dd9 100644
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@ -184,6 +184,21 @@ func (client *ArvTestClient) Call(method, resourceType, uuid, action string, par
}
}
+func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
+ parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
+ j := []byte(`{
+ "command": ["sleep", "1"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": ".",
+ "environment": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
+ }`)
+ return ioutil.NopCloser(bytes.NewReader(j)), nil
+}
+
func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
if resourceType == "collections" {
if uuid == hwPDH {
@@ -322,6 +337,11 @@ func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parame
return errors.New("ArvError")
}
+func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
+ parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
+ return nil, errors.New("ArvError")
+}
+
func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
return errors.New("ArvError")
}
commit b77893f2a8ae755f22615054f2c267d990995e1c
Merge: b30e81e db7c2ba
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Mon Mar 20 16:07:40 2017 -0300
10218: Merge branch 'master' into 10218-record-node-info
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list