[ARVADOS] updated: e468c36bcc610e9b5f4537c64de325e88660d89c
Git user
git at public.curoverse.com
Wed Mar 22 19:42:10 EDT 2017
Summary of changes:
.../app/controllers/application_controller.rb | 13 +-
.../workbench/app/controllers/search_controller.rb | 3 +
.../app/views/projects/_show_dashboard.html.erb | 12 +-
.../app/views/work_units/_show_child.html.erb | 4 +-
.../test/controllers/disabled_api_test.rb | 11 +
doc/_config.yml | 5 +
doc/_includes/_compute_ping_rb.liquid | 285 +++++++++++
.../install-compute-node.html.textile.liquid | 6 +
.../install-slurm.html.textile.liquid} | 96 +---
.../install-compute-ping.html.textile.liquid | 9 +
.../install-nodemanager.html.textile.liquid | 562 +++++++++++++++++++++
doc/install/migrate-docker19.html.textile.liquid | 31 ++
docker/migrate-docker19/Dockerfile | 31 ++
docker/migrate-docker19/build.sh | 2 +
.../docker/run => docker/migrate-docker19/dnd.sh | 1 -
docker/migrate-docker19/migrate.sh | 41 ++
sdk/cli/arvados-cli.gemspec | 2 +-
sdk/cli/test/test_arv-collection-create.rb | 1 +
sdk/cwl/arvados_cwl/arvdocker.py | 8 +-
sdk/cwl/tests/test_container.py | 6 +-
sdk/python/arvados/api.py | 2 +-
sdk/python/arvados/collection.py | 1 +
sdk/python/arvados/commands/keepdocker.py | 92 +---
sdk/python/arvados/commands/migrate19.py | 204 ++++++++
sdk/python/bin/arv-migrate-docker19 | 4 +-
sdk/python/tests/run_test_server.py | 14 +-
sdk/python/tests/test_arv_keepdocker.py | 44 +-
.../controllers/arvados/v1/groups_controller.rb | 16 +
.../arvados/v1/keep_disks_controller.rb | 32 +-
services/api/app/models/collection.rb | 101 ++--
services/api/app/models/container_request.rb | 2 +-
services/api/app/models/job.rb | 11 +-
services/api/config/application.default.yml | 1 +
.../api/test/functional/arvados/v1/filters_test.rb | 52 ++
services/api/test/unit/container_request_test.rb | 54 +-
services/api/test/unit/job_test.rb | 29 +-
services/crunch-run/crunchrun.go | 141 +++++-
services/crunch-run/crunchrun_test.go | 76 ++-
services/fuse/arvados_fuse/command.py | 4 +
services/fuse/tests/test_mount_type.py | 23 +
.../arvnodeman/computenode/dispatch/slurm.py | 2 +-
services/nodemanager/arvnodeman/config.py | 3 +-
services/nodemanager/arvnodeman/daemon.py | 17 +-
services/nodemanager/arvnodeman/jobqueue.py | 8 +-
services/nodemanager/arvnodeman/launcher.py | 3 +-
services/nodemanager/doc/azure.example.cfg | 6 +
services/nodemanager/doc/ec2.example.cfg | 6 +
services/nodemanager/doc/gce.example.cfg | 6 +
services/nodemanager/tests/test_config.py | 4 +
services/nodemanager/tests/test_jobqueue.py | 16 +
tools/arvbox/bin/arvbox | 78 +--
.../arvbox/docker/service/websockets/run-service | 39 +-
52 files changed, 1862 insertions(+), 358 deletions(-)
create mode 100644 doc/_includes/_compute_ping_rb.liquid
copy doc/install/{install-crunch-dispatch.html.textile.liquid => crunch2-slurm/install-slurm.html.textile.liquid} (53%)
create mode 100644 doc/install/install-compute-ping.html.textile.liquid
create mode 100644 doc/install/install-nodemanager.html.textile.liquid
create mode 100644 doc/install/migrate-docker19.html.textile.liquid
create mode 100644 docker/migrate-docker19/Dockerfile
create mode 100755 docker/migrate-docker19/build.sh
copy tools/arvbox/lib/arvbox/docker/service/docker/run => docker/migrate-docker19/dnd.sh (98%)
create mode 100755 docker/migrate-docker19/migrate.sh
create mode 100644 sdk/python/arvados/commands/migrate19.py
create mode 100644 services/fuse/tests/test_mount_type.py
via e468c36bcc610e9b5f4537c64de325e88660d89c (commit)
via dc6c3fccb583ae98eee808addb526c45ebdbf2c6 (commit)
via bc8a4a8863f147e4c0b95d0aa3e2f3e549637ee4 (commit)
via a54e88868ac259443e2cd8d5f6fddb4b8154acb9 (commit)
via 432e71aea50074b1674e96fb1a03cf512952ab75 (commit)
via 6b2005a2fa3e7c040f8db222010fa870f87336bf (commit)
via 5976c751048100e813edd638632927794df072f2 (commit)
via b50e323ec94e9c7355c6f1c5b93f488540d08ec3 (commit)
via 49510014c88646ef4320b137240966ba8f6d5108 (commit)
via 5689f1de6e1a263237303b73013dcebb4ad77e1f (commit)
via 2fd606b328508babad9af6c0a30c159568b525c2 (commit)
via f8084a54742def1915c9f4be290f9076033af152 (commit)
via b77893f2a8ae755f22615054f2c267d990995e1c (commit)
via db7c2ba7523fbce1dc47ea1a715a738a28b3bb3b (commit)
via 433cd703220880784c8dab316501d1c74b09e37b (commit)
via 9c99316ba4a62ab15b383a8e3b2ec6f4d7165a3e (commit)
via e20dfe63dea30fc45bd4fa50aa9d6b5ab1040c99 (commit)
via 03589784d8bc566305e1ea3473a8b37941147517 (commit)
via fd42d69ffa558918ec40f9aeaceb2cbf93de1e8a (commit)
via 99ee47018fcb7cff2adc9184e2063eeba9d55798 (commit)
via 2accec1f7ca3e2dd1e7fe9ee28bb6a8a6cb74abc (commit)
via de283bd8dd383a4177bb714212cf7c59f38a23d8 (commit)
via b30e81ee46c196476a03bc244fbc4419573ba34e (commit)
via da8c9048bf4f3a36f5691aa52e56e2db672dff3a (commit)
via 47178c71032b476e34fd0c6b10065e1b96a568f1 (commit)
via bf03dd4f523463b896428527fa0685631e9ac649 (commit)
via 996b635700d7270229200a56d2c2b9f7c96a84fb (commit)
via b60a21fe3482e1f7ca1815e2a90e276a8c377636 (commit)
via 2c69d491fbbd1461bb5a6dd9c0f0dd493081b45e (commit)
via 9f34ff6f6a6ea1d384528a093bad1c72bd2cea89 (commit)
via 73dbf5bf94b162e5e9ed74cb60f17ee2c7018059 (commit)
via 386faadf691e444b71d6c96e7c00792d9a0ba2c7 (commit)
via 8513e042b0033599146546bd3a2ad903c67c9ff5 (commit)
via 1230d8a106c5c62edcbb9fcf6d1b94585e5596b2 (commit)
via 54837bcae94d30f37f19e4ee436c2bb96bd8f21a (commit)
via b1aa6c85c3db947f0963b1bddb2784c115f5c97d (commit)
via 98329b75276412e605d54b3a0f907a5cf204384f (commit)
via 91118e3af51c7fa655f14f2da99216f8ff0e11e0 (commit)
via bdaa9de9882fee122fd2274d92ea500113df8195 (commit)
via d363323bfea33aee9182e71fd93387b0c4f944a2 (commit)
via 449d780f0a470f67d1a28fe1459a8bab79562069 (commit)
via eabc13f27dfb178069546202bca772f893bac81c (commit)
via dee634b21e48e467b0d7b5ef1b2965c1df5e41d1 (commit)
via f074ef8181c2b93cbfcd870eb67c8c95cb0e442a (commit)
via 2ed5325b37efa7aa9d38a60c1d5c8b4980df8489 (commit)
via 55c719bff1b34d037506639fd4cf7f0a74f4c3cb (commit)
via 6058f336a4562b6d6780c4fcfc4d28b6b2a51948 (commit)
via 1d7577d08b6ca646fcca0e941ce614c37051264e (commit)
via 205b07766565f3ce7b80bcdfee88f83af451fb01 (commit)
via 0eb521827f6a64e567722527f36f0b4d130af504 (commit)
via 5fffad607e39e331f1051c35c5b70ecd5a30939a (commit)
via 355fe24bf5a4ac9e45372333e8bbc8e39b5b86a1 (commit)
via 25296def9590c8a29bdcf75c78a03444d39ca6b6 (commit)
via c04608e40b971b2fc342db8e5e44817b568a3a4c (commit)
via a13833894cda6bc1c872b3eb4215dec5c67aad62 (commit)
via 7eb2eff1864696bb51b2f4913499da5b62fb74ec (commit)
via 6ea807b2caf6c934f170b2e4d89c23c4a08ca69c (commit)
via c7515954731c6b7c0d8fb241ec4316a5e6d62d0c (commit)
via dbb0d22a96fa2de606430d78c05c254598f501df (commit)
via 918e16de43e993894dcaffbe0cafa0fc1d2f16b3 (commit)
via a72205728f94f5261b657766e01f5767dc15d4b5 (commit)
via e1e0bec5d9828fff1b9269322d415789675e6fab (commit)
via f4a7c5e6b1e6e30ad8a31a704642fcc360239e03 (commit)
via f63c160fd1cec8dddc673900f5cda824bed30d88 (commit)
via 605d11f27e73ce60794fa9858aeed8ed961261e8 (commit)
via fa8c5cdb36fa89af92087d6465b207d65ffc9618 (commit)
via 82697fea93b1c87cdee27d2b9a76c1b7ac07497e (commit)
via a15fc58f280171660afd2b9f1938f0a6bc5ede4a (commit)
via c7143dc92806d7b0c6f434e91744a04e5283284d (commit)
via 005595fb1a7da851a863118fba7e3d00026fb3a3 (commit)
via 533a50bbcee4ece451a65e87066563a5b33ef150 (commit)
via c97408a51dab3e968c76cf8c1d873fbe0f99a0dd (commit)
via 4812b5639cfaf724540786fcd331aaa227635c77 (commit)
via 101e3227b25c16874fa73660bfd7e338fbfe0da2 (commit)
via 9dae71315a83504493952f9039de958594e45f31 (commit)
via a1b0149879593df786807afbefb26d4687ebe161 (commit)
via b6423b5e5261f1ddaa88db2548e2190f82de21e8 (commit)
via cb0b666f86ce87d35667e8e23e3775b703a37841 (commit)
via 2ab1b540a710fd454f33e9ba22db601e67f3d217 (commit)
via cad4856b84ff09f8d66faf16cc02ce68ef87995f (commit)
from c071433ab8d50a2fa92d1f75babf5a4ec9f0f5a8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e468c36bcc610e9b5f4537c64de325e88660d89c
Merge: c071433 dc6c3fc
Author: radhika <radhika at curoverse.com>
Date: Wed Mar 22 19:41:48 2017 -0400
Merge branch 'master' into 9132-dockerclient
Conflicts:
services/crunch-run/crunchrun.go
diff --cc services/crunch-run/crunchrun.go
index 49f34d3,26c8a7a..0fb1030
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@@ -186,15 -136,14 +189,15 @@@ func (runner *ContainerRunner) SetupSig
// stop the underlying Docker container.
func (runner *ContainerRunner) stop() {
- runner.CancelLock.Lock()
- defer runner.CancelLock.Unlock()
- if runner.Cancelled {
+ runner.cStateLock.Lock()
+ defer runner.cStateLock.Unlock()
+ if runner.cCancelled {
return
}
- runner.Cancelled = true
- if runner.ContainerID != "" {
+ runner.cCancelled = true
+ if runner.cStarted {
- err := runner.Docker.StopContainer(runner.ContainerID, 10)
+ timeout := time.Duration(10)
+ err := runner.Docker.ContainerStop(context.TODO(), runner.ContainerID, &(timeout))
if err != nil {
log.Printf("StopContainer failed: %s", err)
}
@@@ -651,8 -693,12 +746,13 @@@ func (runner *ContainerRunner) CreateCo
// StartContainer starts the docker container created by CreateContainer.
func (runner *ContainerRunner) StartContainer() error {
runner.CrunchLog.Printf("Starting Docker container id '%s'", runner.ContainerID)
+ runner.cStateLock.Lock()
+ defer runner.cStateLock.Unlock()
+ if runner.cCancelled {
+ return ErrCancelled
+ }
- err := runner.Docker.StartContainer(runner.ContainerID, &runner.HostConfig)
+ err := runner.Docker.ContainerStart(context.TODO(), runner.ContainerID,
+ dockertypes.ContainerStartOptions{})
if err != nil {
return fmt.Errorf("could not start container: %v", err)
}
@@@ -664,22 -711,21 +765,21 @@@
func (runner *ContainerRunner) WaitFinish() error {
runner.CrunchLog.Print("Waiting for container to finish")
- waitDocker := runner.Docker.Wait(runner.ContainerID)
+ waitDocker, err := runner.Docker.ContainerWait(context.TODO(), runner.ContainerID)
+ if err != nil {
+ return fmt.Errorf("container wait: %v", err)
+ }
+
- runner.CrunchLog.Printf("container wait API status code: %v", waitDocker)
+ code := int(waitDocker)
+ runner.ExitCode = &code
+
waitMount := runner.ArvMountExit
- for waitDocker != nil {
- select {
- case err := <-waitMount:
- runner.CrunchLog.Printf("arv-mount exited before container finished: %v", err)
- waitMount = nil
- runner.stop()
- case wr := <-waitDocker:
- if wr.Error != nil {
- return fmt.Errorf("While waiting for container to finish: %v", wr.Error)
- }
- runner.ExitCode = &wr.ExitCode
- waitDocker = nil
- }
+ select {
+ case err := <-waitMount:
+ runner.CrunchLog.Printf("arv-mount exited before container finished: %v", err)
+ waitMount = nil
+ runner.stop()
+ default:
}
// wait for stdout/stderr to complete
diff --cc services/crunch-run/crunchrun_test.go
index b1f2228,7224c4f..b6df1a3
--- a/services/crunch-run/crunchrun_test.go
+++ b/services/crunch-run/crunchrun_test.go
@@@ -651,6 -671,56 +673,54 @@@ func (s *TestSuite) TestCrunchstat(c *C
c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`)
}
+ func (s *TestSuite) TestNodeInfoLog(c *C) {
+ api, _, _ := FullRunHelper(c, `{
+ "command": ["sleep", "1"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": ".",
+ "environment": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
- }`, nil, func(t *TestDockerClient) {
++ }`, nil, 0, func(t *TestDockerClient) {
+ time.Sleep(time.Second)
+ t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{}
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+
+ c.Assert(api.Logs["node-info"], NotNil)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
+ c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk INodes.*`)
+ }
+
+ func (s *TestSuite) TestContainerRecordLog(c *C) {
+ api, _, _ := FullRunHelper(c, `{
+ "command": ["sleep", "1"],
+ "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
+ "cwd": ".",
+ "environment": {},
+ "mounts": {"/tmp": {"kind": "tmp"} },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {}
- }`, nil, func(t *TestDockerClient) {
++ }`, nil, 0, func(t *TestDockerClient) {
+ time.Sleep(time.Second)
+ t.logWriter.Close()
- t.finish <- dockerclient.WaitResult{}
+ })
+
+ c.Check(api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(api.CalledWith("container.state", "Complete"), NotNil)
+
+ c.Assert(api.Logs["container"], NotNil)
+ c.Check(api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
+ }
+
func (s *TestSuite) TestFullRunStderr(c *C) {
api, _, _ := FullRunHelper(c, `{
"command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list