[ARVADOS] updated: 28cd3486a00927c6efbf2f73f9e267b3c4746cd0
Git user
git at public.curoverse.com
Mon Aug 15 21:23:08 EDT 2016
Summary of changes:
apps/workbench/app/models/container_work_unit.rb | 6 +-
apps/workbench/app/models/proxy_work_unit.rb | 4 +
apps/workbench/app/models/work_unit.rb | 4 +
.../app/views/work_units/_show_log.html.erb | 2 +-
apps/workbench/test/integration/websockets_test.rb | 9 +-
build/go-package-scripts/postinst | 26 ++++--
build/go-package-scripts/prerm | 19 +++++
build/run-build-packages.sh | 4 +-
build/run-library.sh | 6 +-
crunch_scripts/cwl-runner | 8 +-
.../create-standard-objects.html.textile.liquid | 10 +++
lib/crunchstat/crunchstat.go | 7 +-
sdk/cwl/arvados_cwl/__init__.py | 28 ++++++-
sdk/cwl/arvados_cwl/arvcontainer.py | 4 +-
sdk/cwl/arvados_cwl/arvjob.py | 4 +-
sdk/cwl/arvados_cwl/arvtool.py | 2 +
sdk/cwl/arvados_cwl/fsaccess.py | 21 +++--
sdk/cwl/arvados_cwl/pathmapper.py | 5 +-
sdk/cwl/setup.py | 2 +-
sdk/cwl/tests/arvados-tests.sh | 5 ++
sdk/cwl/tests/arvados-tests.yml | 10 +++
sdk/cwl/tests/dir-job.yml | 3 +
sdk/cwl/tests/keep-dir-test-input.cwl | 21 +++++
sdk/cwl/tests/runner.sh | 2 +
sdk/cwl/tests/test_container.py | 17 +++-
sdk/cwl/tests/test_job.py | 15 ++--
.../tests/__init__.py => sdk/cwl/tests/testdir/a | 0
.../tests/__init__.py => sdk/cwl/tests/testdir/b | 0
.../tests/__init__.py => sdk/cwl/tests/testdir/c/d | 0
sdk/go/arvados/container.go | 13 +--
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 64 ++++++++++----
.../crunch-dispatch-slurm.service | 12 +++
.../crunch-dispatch-slurm_test.go | 5 +-
services/crunch-dispatch-slurm/script.go | 15 ++++
services/crunch-dispatch-slurm/script_test.go | 24 ++++++
services/crunch-dispatch-slurm/usage.go | 33 ++++++++
services/crunch-run/cgroup.go | 29 +++++++
services/crunch-run/cgroup_test.go | 17 ++++
services/crunch-run/crunchrun.go | 97 ++++++++++++++++------
services/crunch-run/crunchrun_test.go | 89 +++++++++++++++-----
services/crunch-run/logging.go | 52 +++++-------
41 files changed, 543 insertions(+), 151 deletions(-)
create mode 100755 build/go-package-scripts/prerm
create mode 100755 sdk/cwl/tests/arvados-tests.sh
create mode 100644 sdk/cwl/tests/arvados-tests.yml
create mode 100644 sdk/cwl/tests/dir-job.yml
create mode 100644 sdk/cwl/tests/keep-dir-test-input.cwl
create mode 100755 sdk/cwl/tests/runner.sh
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/testdir/a (100%)
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/testdir/b (100%)
copy tools/crunchstat-summary/tests/__init__.py => sdk/cwl/tests/testdir/c/d (100%)
create mode 100644 services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
create mode 100644 services/crunch-dispatch-slurm/script.go
create mode 100644 services/crunch-dispatch-slurm/script_test.go
create mode 100644 services/crunch-dispatch-slurm/usage.go
create mode 100644 services/crunch-run/cgroup.go
create mode 100644 services/crunch-run/cgroup_test.go
discards 26e2fafd86c2b3241ff98747fa1795dbf00e72d1 (commit)
discards 902e38e8489642d581b6157837fd24fcb57a14f0 (commit)
via 28cd3486a00927c6efbf2f73f9e267b3c4746cd0 (commit)
via 17703ec8e1acca41ca5bd48e38651879996816b6 (commit)
via 70bbddf187a5f2ac1e7c205965c782c94b4efc0d (commit)
via 542afd1ca6f77368165f39d84f4bee6c9e2472bd (commit)
via cdd7a8615c9cb7fb292a1394ccc0468f6dd07b63 (commit)
via 30591f2ddcd3cf72ecf5bfa9a84aaa932d0096d1 (commit)
via b526df9ab5269735bce42af3ed9fbe777e513093 (commit)
via b9f912472b42ffffb26da11eaf9b19a825acec47 (commit)
via c5ac66520889330a417b1dbc03da2fbc5ef5cfd0 (commit)
via 61f5363e5bb0660a3a9f0de5fca20985990911ca (commit)
via aeaef97c33e37f73dd133dcc098ca6f42a9adde1 (commit)
via 5c6bcd52032b2c974bc04e5e41282ae46835c08d (commit)
via d84cefc8e4c34f36232e11c58af44d1afbbf89b3 (commit)
via d4a9835195f1c87d6d3436050d55bc60ba031259 (commit)
via fd22b368fb9d3bdfa17d5b6a65279fde8664700e (commit)
via a4c37c079728f2013f5a299acc97f1b24c1e2884 (commit)
via bf43afc52fce505edbe3fad9daa6b0c729d02eeb (commit)
via 6e0abcdabb14bcd21695420d3cfab0d57362a2be (commit)
via 08698803d8b324ba9a889a3aa41559d782bcc4bf (commit)
via c961ea106e3528a2690f7ab1e0032fa68f650bfe (commit)
via 46df7c2b292e5f12da365b918b0f28757eb4c4ce (commit)
via 9dda943e37fce96c5debe1410a11cfa9c88daf48 (commit)
via 001ded3c64c9061126d0328079b9b3b2d13a4855 (commit)
via 7715771293c856a55fdce2a7ffbeea583d8982a7 (commit)
via 8a7a54ed6d3c57f1142ae22ccb082c6cb9f4dd4d (commit)
via 80eb5efc773680493edefdb8dda08ecbb21db824 (commit)
via ac4ee0eb5a2d581b7bb4d26b475f0fbcf64b0630 (commit)
via 608f6a788a392246ad0747d94448cb07e0ff6d8f (commit)
via 3c5183b3a2b895e238b0e824a684787a56c155b4 (commit)
via e5e9ae2751cf691c7eeee60eac9d70e4f1e57a3b (commit)
via c1307df1b44856448b9dc64000cd5cd1befb2b14 (commit)
via ddb96e86a98113ea313dd01e8b7957f07345b6a7 (commit)
via 13c821b6dd20ed658db9c6c3db0b8d10d9175fbb (commit)
via 5951500ed7a4eaf07c4d2bad3b87d5a0ed1f60e8 (commit)
via c3cbac378fb45e3bf996a5d691cd3f205dfb3f90 (commit)
via 8094a4b4914d892461b2a6fbcbeb10b938a6733b (commit)
via e2fc01e97c473de174c39c328dc3116ecc699225 (commit)
via ffff84bc7c76e3c6581a10559dcfc193768e7b58 (commit)
via 47833e79d907d5a5c269f0a2924db535a9c42c05 (commit)
via e03666a151bc6cd371d371ef36d787e462f4f51d (commit)
via 2f27547b2ec374fddbaed63b2a72362d7bf81157 (commit)
via 6add44717dcbc817152f606dd1329c9b2768b562 (commit)
via 3c101af94da9d7cbbbaa8941f5510292c5de60f7 (commit)
via 5df1299bef79cbae96aecfd710a8d2cc088e4ecd (commit)
via 483ca35ac4348924cfbc187dab4f1b88a272eea7 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (26e2fafd86c2b3241ff98747fa1795dbf00e72d1)
\
N -- N -- N (28cd3486a00927c6efbf2f73f9e267b3c4746cd0)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 28cd3486a00927c6efbf2f73f9e267b3c4746cd0
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Aug 15 21:17:30 2016 -0400
9706: Fix config struct to match example.
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 8686fa4..ef08f45 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -22,7 +22,7 @@ import (
// Config used by crunch-dispatch-slurm
type Config struct {
- arvados.Client
+ Client arvados.Client
SbatchArguments []string
PollPeriod arvados.Duration
commit 17703ec8e1acca41ca5bd48e38651879996816b6
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Aug 15 16:58:02 2016 -0400
9706: Notify systemd when crunch-dispatch-slurm is ready.
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 1d9015d..8686fa4 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -9,6 +9,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/dispatch"
+ "github.com/coreos/go-systemd/daemon"
"io"
"io/ioutil"
"log"
@@ -105,6 +106,10 @@ func doMain() error {
PollInterval: time.Duration(config.PollPeriod),
DoneProcessing: make(chan struct{})}
+ if _, err := daemon.SdNotify("READY=1"); err != nil {
+ log.Printf("Error notifying init daemon: %v", err)
+ }
+
err = dispatcher.RunDispatcher()
if err != nil {
return err
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
index eac8d89..5b37e3c 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
@@ -4,7 +4,7 @@ Documentation=https://doc.arvados.org/
After=network.target
[Service]
-Type=simple
+Type=notify
ExecStart=/usr/bin/crunch-dispatch-slurm
Restart=always
commit 70bbddf187a5f2ac1e7c205965c782c94b4efc0d
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Aug 15 16:10:58 2016 -0400
9706: Add systemd unit file for crunch-dispatch-slurm.
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
new file mode 100644
index 0000000..eac8d89
--- /dev/null
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.service
@@ -0,0 +1,12 @@
+[Unit]
+Description=Arvados Crunch Dispatcher for SLURM
+Documentation=https://doc.arvados.org/
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=/usr/bin/crunch-dispatch-slurm
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
commit 542afd1ca6f77368165f39d84f4bee6c9e2472bd
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Aug 15 16:10:29 2016 -0400
9706: Stop systemd services when their packages are uninstalled.
diff --git a/build/go-package-scripts/postinst b/build/go-package-scripts/postinst
index 3adb0fc..8559a97 100755
--- a/build/go-package-scripts/postinst
+++ b/build/go-package-scripts/postinst
@@ -15,12 +15,14 @@ case "${1}" in
eval "$(systemctl -p UnitFileState show "$systemd_unit")"
case "$UnitFileState" in
disabled)
- systemctl enable "$systemd_unit"
- # Failing to start the service is not a package error,
- # so don't let an error here propagate up.
+ # Failing to enable or start the service is not a
+ # package error, so don't let errors here
+ # propagate up.
+ systemctl enable "$systemd_unit" || true
systemctl start "$systemd_unit" || true
;;
enabled)
+ systemctl daemon-reload || true
systemctl reload-or-try-restart "$systemd_unit" || true
;;
esac
diff --git a/build/go-package-scripts/prerm b/build/go-package-scripts/prerm
new file mode 100755
index 0000000..4e1399d
--- /dev/null
+++ b/build/go-package-scripts/prerm
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+set -e
+
+# NOTE: This package name detection will only work on Debian.
+# If this prerm script ever starts doing work on Red Hat,
+# we'll need to adapt this code accordingly.
+script="$(basename "$0")"
+pkg="${script%.prerm}"
+systemd_unit="${pkg}.service"
+
+case "${1}" in
+ remove)
+ if [ -e /run/systemd/system ]; then
+ systemctl stop "$systemd_unit" || true
+ systemctl disable "$systemd_unit" || true
+ fi
+ ;;
+esac
diff --git a/build/run-library.sh b/build/run-library.sh
index 9140ac1..485e0d2 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -104,8 +104,10 @@ package_go_binary() {
declare -a switches=()
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
- switches+=("${systemd_unit}=/lib/systemd/system/${prog}.service")
- switches+=(--after-install "$WORKSPACE/build/go-package-scripts/postinst")
+ switches+=(
+ --after-install "$WORKSPACE/build/go-package-scripts/postinst"
+ --before-remove "$WORKSPACE/build/go-package-scripts/prerm"
+ "${systemd_unit}=/lib/systemd/system/${prog}.service")
fi
switches+=("$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file")
commit cdd7a8615c9cb7fb292a1394ccc0468f6dd07b63
Author: Brett Smith <brett at curoverse.com>
Date: Wed Aug 10 13:22:02 2016 -0400
9706: Go service postinst improvements.
* Use /bin/sh, per Debian policy.
* Prefer basename for readability.
* Check for systemd init with [ -e /run/systemd/system ].
* Make configuration changes based on checking the unit file state.
diff --git a/build/go-package-scripts/postinst b/build/go-package-scripts/postinst
index 137f898..3adb0fc 100755
--- a/build/go-package-scripts/postinst
+++ b/build/go-package-scripts/postinst
@@ -1,15 +1,29 @@
-#!/bin/bash
+#!/bin/sh
set -e
-script="${0##*/}"
+# NOTE: This package name detection will only work on Debian.
+# If this postinst script ever starts doing work on Red Hat,
+# we'll need to adapt this code accordingly.
+script="$(basename "$0")"
pkg="${script%.postinst}"
+systemd_unit="${pkg}.service"
case "${1}" in
configure)
- if which systemctl >/dev/null && [[ -e "/lib/systemd/system/${pkg}.service" ]]; then
- systemctl enable "${pkg}"
- systemctl start "${pkg}"
+ if [ -e /run/systemd/system ]; then
+ eval "$(systemctl -p UnitFileState show "$systemd_unit")"
+ case "$UnitFileState" in
+ disabled)
+ systemctl enable "$systemd_unit"
+ # Failing to start the service is not a package error,
+ # so don't let an error here propagate up.
+ systemctl start "$systemd_unit" || true
+ ;;
+ enabled)
+ systemctl reload-or-try-restart "$systemd_unit" || true
+ ;;
+ esac
fi
;;
esac
commit 30591f2ddcd3cf72ecf5bfa9a84aaa932d0096d1
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Aug 4 18:43:17 2016 -0400
9706: Add postinst script to Go packages that have systemd unit files, so (on Debian) the service starts up without manual intervention.
diff --git a/build/go-package-scripts/postinst b/build/go-package-scripts/postinst
new file mode 100755
index 0000000..137f898
--- /dev/null
+++ b/build/go-package-scripts/postinst
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+
+script="${0##*/}"
+pkg="${script%.postinst}"
+
+case "${1}" in
+ configure)
+ if which systemctl >/dev/null && [[ -e "/lib/systemd/system/${pkg}.service" ]]; then
+ systemctl enable "${pkg}"
+ systemctl start "${pkg}"
+ fi
+ ;;
+esac
diff --git a/build/run-library.sh b/build/run-library.sh
index aee579c..9140ac1 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -101,14 +101,15 @@ package_go_binary() {
cd $WORKSPACE/packages/$TARGET
go get "git.curoverse.com/arvados.git/$src_path"
- declare -a addfiles=()
+ declare -a switches=()
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
- addfiles+=("${systemd_unit}=/lib/systemd/system/${prog}.service")
+ switches+=("${systemd_unit}=/lib/systemd/system/${prog}.service")
+ switches+=(--after-install "$WORKSPACE/build/go-package-scripts/postinst")
fi
- addfiles+=("$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file")
+ switches+=("$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file")
- fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "${addfiles[@]}"
+ fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "${switches[@]}"
}
default_iteration() {
commit b526df9ab5269735bce42af3ed9fbe777e513093
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Aug 3 17:06:55 2016 -0400
9706: If "foo.service" exists in the source dir for a Go program, package it so it installs to /lib/systemd/system/.
diff --git a/build/run-library.sh b/build/run-library.sh
index 32cf205..aee579c 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -100,7 +100,15 @@ package_go_binary() {
cd $WORKSPACE/packages/$TARGET
go get "git.curoverse.com/arvados.git/$src_path"
- fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file"
+
+ declare -a addfiles=()
+ systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
+ if [[ -e "${systemd_unit}" ]]; then
+ addfiles+=("${systemd_unit}=/lib/systemd/system/${prog}.service")
+ fi
+ addfiles+=("$WORKSPACE/$license_file=/usr/share/doc/$prog/$license_file")
+
+ fpm_build "$GOPATH/bin/$basename=/usr/bin/$prog" "$prog" 'Curoverse, Inc.' dir "$version" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=$description" "${addfiles[@]}"
}
default_iteration() {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list