[ARVADOS] updated: 1.3.0-1809-gf5f4c5f60
Git user
git at public.curoverse.com
Tue Oct 29 15:35:17 UTC 2019
Summary of changes:
apps/workbench/Gemfile | 4 +
apps/workbench/Gemfile.lock | 7 +-
build/run-tests.sh | 4 +
doc/admin/upgrading.html.textile.liquid | 9 -
lib/config/config.default.yml | 9 +-
lib/config/export.go | 1 +
lib/config/generated_config.go | 9 +-
lib/dispatchcloud/scheduler/sync.go | 7 +
lib/dispatchcloud/scheduler/sync_test.go | 56 ++++
lib/dispatchcloud/worker/runner.go | 7 +-
sdk/cwl/arvados_cwl/__init__.py | 2 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 2 +-
sdk/cwl/tests/test_submit.py | 8 +-
sdk/go/arvados/config.go | 1 +
sdk/go/arvados/fs_base.go | 25 ++
sdk/go/arvados/fs_collection.go | 324 +++++++++++++++------
sdk/go/arvados/fs_collection_test.go | 189 +++++++++++-
sdk/go/arvados/fs_site.go | 2 +
sdk/go/httpserver/logger.go | 8 +-
sdk/go/httpserver/logger_test.go | 68 ++++-
sdk/go/httpserver/responsewriter.go | 29 +-
.../arvados/client/logic/collection/FileToken.java | 6 +-
.../arvados/client/logic/keep/FileDownloader.java | 8 +-
.../client/logic/collection/FileTokenTest.java | 2 +-
services/api/Gemfile | 4 +
services/api/Gemfile.lock | 7 +-
services/crunch-run/copier.go | 31 +-
services/crunch-run/crunchrun.go | 42 +--
28 files changed, 695 insertions(+), 176 deletions(-)
create mode 100644 lib/dispatchcloud/scheduler/sync_test.go
via f5f4c5f6088707c86c43ca2bd70a70c14699e450 (commit)
via d8f53190b772815880da5c4dd9198d65b950a237 (commit)
via 916ec66d0caeeb37983043810bea22e0bc41751f (commit)
via e51103a789f4b246b2dcdac2b2fa215c707500b7 (commit)
via 5d6531e7ed927ce8059caa61515efc3d4d539cfe (commit)
via 053c19a77e69cd17d52ec7f8a27c223ee24e1998 (commit)
via aa4b9b1d650f2174eb19bbf2ba9787f3ace59b04 (commit)
via f4e2a0e7fa1b248665fd3fb3998ac0fcb98e8eb6 (commit)
via 868833828599674d566e1c9550a035d5d8002076 (commit)
via 857533188539ebbcb9cf3874342784b811251b3a (commit)
via e701875382821088123769823f6b66a5a4ae2907 (commit)
via 4cadc4832ed4dd58e841628232c472e99b6f7d1b (commit)
via fd689ff480c77b2b5ba60c02f5522e794f190661 (commit)
via de6d121eac429a1dc65e3c3e646f45d54bcfcbf3 (commit)
via 36994ebb9f4b0c3149b671e0229677c39a37cba8 (commit)
via 6980748d61d4dff0ab95be4c65f55be47d13bb57 (commit)
via 7db3ceda16742b65d73ebbc05d02351a5e0496bd (commit)
via bd8028e0b754dc36c4118ce50eea930fba9748d3 (commit)
via faf146c21364533f95d937fe75cc2a46344eed83 (commit)
via 62d79fb469256caa6e287d0787d7d58f6142ed69 (commit)
via ab3284a88a806268f680dd7fc7651a85ad87db01 (commit)
via cc8174aca759773fa661bd9172f877b2d639576b (commit)
via bf08477c7e766c7692731c08212c8d1c3c5628ea (commit)
via a6cba47c47dcd6a7f0bce99de9bbed6a87ef3102 (commit)
via 0daa251cdddbef3db6a69b388170fdb2901964c6 (commit)
via c8919a8e3a27406d707b4491b5858a34525bdeba (commit)
via 97e783760423f6f0200b057629aa9cd593a5c9c7 (commit)
via 3e5a4f9f0c4ece9af648a84a5274114ebe922e06 (commit)
via d0e4ca7b44ab1ed8c0eb1f4df3a02127d070afb1 (commit)
via 76d28579bd98ef8edd2cba30c1ac019795b9ea1e (commit)
via 84347b3dbe7d7140e34e3a67b58e06d3c6b5fa9c (commit)
via fed5a79b5852ebf78d660ec8cb169ce512ad6532 (commit)
via aea1693cfc6359a7f290797724ee989bc47b09e0 (commit)
via 5e109ae2b94f5dcaddb19da360507f73078e2d1c (commit)
via 15917b2b3902994ca4f6b004577b1130ba0fdaf0 (commit)
via 78724c7c38805402c442629d4d678f8145e31d76 (commit)
via b714ab7401074991afe2fdc239c89107b3af6ca1 (commit)
from 9481ff4a22314c0d5acffe78fbc7595278414e6f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f5f4c5f6088707c86c43ca2bd70a70c14699e450
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Oct 29 11:34:49 2019 -0400
15734: Fix InstanceType logging: propagate to detached child.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/dispatchcloud/worker/runner.go b/lib/dispatchcloud/worker/runner.go
index 91db82460..e819a6036 100644
--- a/lib/dispatchcloud/worker/runner.go
+++ b/lib/dispatchcloud/worker/runner.go
@@ -35,11 +35,8 @@ type remoteRunner struct {
// newRemoteRunner returns a new remoteRunner. Caller should ensure
// Close() is called to release resources.
func newRemoteRunner(uuid string, wkr *worker) *remoteRunner {
- // Early (<1.5) versions of crunch-run error out if they see
- // non-string values in the env map -- so here we send the
- // instance type record as a JSON doc. Once worker images are
- // updated, we can skip the extra encoding, and just include
- // {"InstanceType": wkr.instType} in the env map.
+ // Send the instance type record as a JSON doc so crunch-run
+ // can log it.
var instJSON bytes.Buffer
enc := json.NewEncoder(&instJSON)
enc.SetIndent("", " ")
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 082ca0ce9..d1092e98e 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -116,9 +116,6 @@ type ContainerRunner struct {
ContainerArvClient IArvadosClient
ContainerKeepClient IKeepClient
- // environment provided by arvados-dispatch-cloud
- dispatchEnv map[string]interface{}
-
Container arvados.Container
ContainerConfig dockercontainer.Config
HostConfig dockercontainer.HostConfig
@@ -856,7 +853,7 @@ func (runner *ContainerRunner) LogContainerRecord() error {
// LogNodeRecord logs the current host's InstanceType config entry (or
// the arvados#node record, if running via crunch-dispatch-slurm).
func (runner *ContainerRunner) LogNodeRecord() error {
- if it, ok := runner.dispatchEnv["InstanceType"]; ok {
+ if it := os.Getenv("InstanceType"); it != "" {
// Dispatched via arvados-dispatch-cloud. Save
// InstanceType config fragment received from
// dispatcher on stdin.
@@ -865,22 +862,9 @@ func (runner *ContainerRunner) LogNodeRecord() error {
return err
}
defer w.Close()
- if it, ok := it.(string); ok {
- // dispatcher supplied JSON data (in order to
- // stay compatible with old crunch-run
- // versions)
- _, err = io.WriteString(w, it)
- if err != nil {
- return err
- }
- } else {
- // dispatcher supplied struct
- enc := json.NewEncoder(w)
- enc.SetIndent("", " ")
- err = enc.Encode(it)
- if err != nil {
- return err
- }
+ _, err = io.WriteString(w, it)
+ if err != nil {
+ return err
}
return w.Close()
} else {
@@ -1811,12 +1795,11 @@ func main() {
flag.Parse()
- var env map[string]interface{}
if *stdinEnv && !ignoreDetachFlag {
// Load env vars on stdin if asked (but not in a
// detached child process, in which case stdin is
// /dev/null).
- env = loadEnv(os.Stdin)
+ loadEnv(os.Stdin)
}
switch {
@@ -1871,8 +1854,6 @@ func main() {
os.Exit(1)
}
- cr.dispatchEnv = env
-
parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerId+".")
if tmperr != nil {
log.Fatalf("%s: %v", containerId, tmperr)
@@ -1912,23 +1893,20 @@ func main() {
}
}
-func loadEnv(rdr io.Reader) map[string]interface{} {
+func loadEnv(rdr io.Reader) {
buf, err := ioutil.ReadAll(rdr)
if err != nil {
log.Fatalf("read stdin: %s", err)
}
- var env map[string]interface{}
+ var env map[string]string
err = json.Unmarshal(buf, &env)
if err != nil {
log.Fatalf("decode stdin: %s", err)
}
for k, v := range env {
- if v, ok := v.(string); ok {
- err = os.Setenv(k, v)
- if err != nil {
- log.Fatalf("setenv(%q): %s", k, err)
- }
+ err = os.Setenv(k, v)
+ if err != nil {
+ log.Fatalf("setenv(%q): %s", k, err)
}
}
- return env
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list