[ARVADOS] updated: 1.1.2-168-g8843744
Git user
git at public.curoverse.com
Wed Jan 31 16:59:37 EST 2018
Summary of changes:
.../app/assets/javascripts/application.js | 1 +
.../app/assets/javascripts/components/edit_tags.js | 265 +++++++++++++++++++
.../app/assets/javascripts/edit_collection_tags.js | 72 ------
.../app/assets/javascripts/mithril_mount.js | 3 +-
.../app/assets/javascripts/models/session_db.js | 16 +-
.../app/assets/stylesheets/application.css.scss | 11 +
.../app/controllers/collections_controller.rb | 24 --
.../app/views/collections/_show_tag_rows.html.erb | 35 ---
.../app/views/collections/_show_tags.html.erb | 50 +---
.../app/views/collections/save_tags.js.erb | 7 -
apps/workbench/app/views/collections/tags.js.erb | 5 -
apps/workbench/npm_packages | 2 +
apps/workbench/public/vocabulary-example.json | 32 +++
.../controllers/collections_controller_test.rb | 60 -----
.../workbench/test/integration/collections_test.rb | 84 ------
build/build-dev-docker-jobs-image.sh | 13 +-
doc/admin/merge-remote-account.html.textile.liquid | 2 +
lib/mount/command.go | 2 +-
sdk/cwl/arvados_cwl/__init__.py | 24 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 3 +-
sdk/cwl/arvados_cwl/arvjob.py | 7 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 7 +-
sdk/cwl/arvados_cwl/fsaccess.py | 9 +-
sdk/cwl/arvados_cwl/runner.py | 42 ++-
sdk/cwl/setup.py | 4 +-
sdk/cwl/tests/arvados-tests.sh | 3 +
sdk/cwl/tests/{secondary/dir => hg19}/hg19.fa | 0
sdk/cwl/tests/{secondary/dir => hg19}/hg19.fa.amb | 0
sdk/cwl/tests/{secondary/dir => hg19}/hg19.fa.ann | 0
sdk/cwl/tests/{secondary/dir => hg19}/hg19.fa.fai | 0
sdk/cwl/tests/test_container.py | 15 +-
sdk/cwl/tests/test_job.py | 6 +-
sdk/cwl/tests/test_submit.py | 4 +
sdk/go/arvados/fs_deferred.go | 103 ++++++++
sdk/go/arvados/fs_project.go | 113 +++++++++
sdk/go/arvados/fs_site.go | 23 +-
sdk/go/arvadostest/fixtures.go | 1 +
.../controllers/arvados/v1/schema_controller.rb | 1 +
.../api/app/models/api_client_authorization.rb | 11 +-
services/api/test/fixtures/users.yml | 16 ++
.../arvados/v1/schema_controller_test.rb | 1 +
services/api/test/integration/remote_user_test.rb | 7 +
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 91 ++-----
.../crunch-dispatch-slurm_test.go | 195 ++++++--------
services/crunch-dispatch-slurm/slurm.go | 73 ++++++
services/crunch-dispatch-slurm/squeue.go | 9 +-
services/crunch-run/crunchrun.go | 282 ++++++++++-----------
services/crunch-run/crunchrun_test.go | 143 ++++++-----
services/fuse/arvados_fuse/fusedir.py | 105 ++++----
services/keepstore/azure_blob_volume.go | 16 +-
services/keepstore/azure_blob_volume_test.go | 5 +
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 1 +
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 5 +-
tools/arvbox/lib/arvbox/docker/crunch-setup.sh | 9 +-
tools/arvbox/lib/arvbox/docker/go-setup.sh | 16 ++
tools/arvbox/lib/arvbox/docker/keep-setup.sh | 9 +-
.../docker/service/arv-git-httpd/run-service | 9 +-
.../lib/arvbox/docker/service/composer/run-service | 12 +-
.../service/crunch-dispatch-local/run-service | 15 +-
.../lib/arvbox/docker/service/keep-web/run-service | 9 +-
.../arvbox/docker/service/keepproxy/run-service | 9 +-
.../arvbox/lib/arvbox/docker/service/postgres/run | 2 +
.../arvbox/docker/service/websockets/run-service | 8 +-
tools/sync-groups/sync-groups.go | 21 +-
tools/sync-groups/sync-groups_test.go | 33 ++-
65 files changed, 1224 insertions(+), 937 deletions(-)
create mode 100644 apps/workbench/app/assets/javascripts/components/edit_tags.js
delete mode 100644 apps/workbench/app/assets/javascripts/edit_collection_tags.js
delete mode 100644 apps/workbench/app/views/collections/_show_tag_rows.html.erb
delete mode 100644 apps/workbench/app/views/collections/save_tags.js.erb
delete mode 100644 apps/workbench/app/views/collections/tags.js.erb
create mode 100644 apps/workbench/public/vocabulary-example.json
copy sdk/cwl/tests/{secondary/dir => hg19}/hg19.fa (100%)
copy sdk/cwl/tests/{secondary/dir => hg19}/hg19.fa.amb (100%)
copy sdk/cwl/tests/{secondary/dir => hg19}/hg19.fa.ann (100%)
copy sdk/cwl/tests/{secondary/dir => hg19}/hg19.fa.fai (100%)
create mode 100644 sdk/go/arvados/fs_deferred.go
create mode 100644 sdk/go/arvados/fs_project.go
create mode 100644 services/crunch-dispatch-slurm/slurm.go
create mode 100644 tools/arvbox/lib/arvbox/docker/go-setup.sh
via 8843744ac700dd080333e8e1bcf6273e7fe273c3 (commit)
via dbcb7c65b6324273fabc7dc2507a1978d8a538b8 (commit)
via ceca47732d7c6f768e8e6f54e6fa96a49f07a553 (commit)
via cd3bab6f91b89073c9b5646a69110a8b1a5e9e66 (commit)
via 913baf3c2d96000ea564821a9ae9163e991105a3 (commit)
via 9482a917eef77ef64fc9fee27b0d107727f765ad (commit)
via b4351d725f01431856cbe1ea0e8faf8d4f91dd4e (commit)
via 3a4434f8b767e05456eea248f645422b1ed670d0 (commit)
via 5bd7108fed4a87eb7cc5a25844bb8c100977c968 (commit)
via 1b7a6c0ca4fa348c313a0862cfca597319cfe08f (commit)
via 90bffaaf5e537608c3a8d2520544bfdd9b98d286 (commit)
via 8137793fcd26eed30004f62a0c9ed0f839a9bd5e (commit)
via b50ea3465244e66ecf2a852f598f5b576e705017 (commit)
via e6fa5ac95cff84a7b00dd8ab32349d88473400da (commit)
via e8de8d362df44459ecbdff44ed27a12b7652762c (commit)
via 03ee5e57fab4ec34b5830e8a93d329a084adf42c (commit)
via ae35f42f7026faa85894b0a8c90caec87c6af239 (commit)
via 1809dbb21b4cb65e0e141dd41d41a3516ef33ec5 (commit)
via 0dcf9daff8fce376f20f125c3ef867333976c18c (commit)
via 8758f3ddc81b9ef9aaab111eb331e452c4ec7de5 (commit)
via a63829d71b904ead17aeac3c68bf01e9b020a3a1 (commit)
via bb72c328306667a0429c10caa90a79b2b9ff0bd6 (commit)
via 2edd084814bd450c0bfb14915bb9bc3ce96498eb (commit)
via 3194c1b24ffe6fff5fcb2f620ca6ee43741e3462 (commit)
via d14421423f2bd72b2c3eb95bdebe85a210972a12 (commit)
via 31007a99c336423f2b34a306028ec0aa41b1dd3a (commit)
via 3a3fc5ac54a80b62b05f78cf2da0a7b43f4a4380 (commit)
via 6b3cfe60beb46ba087de5093926363d8b03ab889 (commit)
via db0e7f5c4848c5c79d9fe43d4c9317c4d5ce482a (commit)
via ba15fa5da21f4bafd3f90a8d259ea2aae764c77e (commit)
via 0cae6411490ad7a4a6d611605fb04de6db2f1190 (commit)
via 8a798254e07ef391c535c919cba448da8ed6db8a (commit)
via d179241c734c2c533f4453beaecd53d27ced9a98 (commit)
via 0fd0dbc7b828503258fa071d4cd712da01dfe11e (commit)
via f1e1a1ae570c8f09130e5de2a9fc15c0a6067542 (commit)
via 8f982486f5ed7b49d250fafdf3840a929a824ef7 (commit)
via 37860134053cda88c7ee3a3f4300e949cad016f5 (commit)
via b017ae29973838673d66d9f9da4a07f9852d9476 (commit)
via 18cf19a3131b19ffcf2181ad31005fa641d46f6d (commit)
via 824680f81d5c5d243f49096be975568b786fac2a (commit)
via d8a7800b5ca3d50bcd62545711585681e2b9154b (commit)
via 2702b79d8981e562aa9848f41d96bd0a37a278c6 (commit)
via 7ec6b571c54af1ddd404488b2922c41dfdffdde1 (commit)
via 87b24df10b354794904d6c3dea6258dd0f45adc3 (commit)
via f44a15adce692614ecb816dbe2d0205704d9a4ab (commit)
via 5a82b37499c717e62d899a633f548713c872307b (commit)
via 3fc4aaee466529bd8ed7fad664d2eb61b37d6864 (commit)
via 75cb3149396f6385d6af30deaed74ee87ace696d (commit)
via ce7505914ba4c53a7aa1d987aaf92de81af1fb49 (commit)
via aa04aa117da273ac363453d2297e3474407ac573 (commit)
via 790688e4bcd6f1a92ac409c0c354acba752aaa9c (commit)
via 6fe66955fb53dde27d9677d31fdb137913b2b850 (commit)
via 0dd6cd18de9b20a85c0bbbcd0df4e497743a132d (commit)
via 479aa841b36246aa16e4cf77f78cdb8b93bc8de1 (commit)
via 8cca38ff5e4d05c85dfb137b29c419e5df8299e0 (commit)
via e5b7fb12522636f24d9b23ea587b8d12e13c50c2 (commit)
via 467722fb20db289fa1eb0833e2d87ee56ed91ba7 (commit)
via 7ed41dc0191d366c914850a350c3b30f769365af (commit)
via 90dd1c90fb310834234163711019e2b932c6e396 (commit)
via f61d8bca07c8792dfd6216119ee63c573810963a (commit)
via 7c7dae7f8f4ec4e33acf9302152907547200023d (commit)
via 7d0d61f43be20aec7ff643001fcf940e485d465b (commit)
via 2ada2ce8390db1e22c5f4fc236518640339e9ce5 (commit)
via 3416d60e2b1a51a6a02498e15ec9a9c225e1b4c2 (commit)
via 5cd85a21e20d191f7fb67dd20023196001930928 (commit)
via 5dc1784a06a53777891df5fdea5f1c29da92939b (commit)
via 8abda7cdf322d6b023b15894509ccfd057819c4f (commit)
via 4070335144818d3b797c5ba0e8bfa4fb2f6cffe3 (commit)
via 47e59a35d5ed9b2cdb052894d741972324058505 (commit)
via b18af1fdb7474f525938f66650d2d267b2784ec2 (commit)
via 4f83103fc5cb227d23b40c8a28c497855b2d5f61 (commit)
via 11f2d977836ec3737b23abb1d716cef58b1314f2 (commit)
via 01cea11ce0cb0e8df2907e66c71540658ed02d45 (commit)
via 4fed183482e37ad80e97e841d2f0b825ef0d2570 (commit)
via 4c6b92aa9648eb091a8ed229bcdbeb3f2d75feeb (commit)
via e54da634282d90d4def79ea95802fbf3da68072a (commit)
via 36da431dd8b1f360f60d06165ab640e10d695d18 (commit)
via 57cbcbe7f4403ec4b17450441c74a3a0f23a1380 (commit)
via 71023ae5387402ed6c76948b4886d8d4a606f095 (commit)
via 97b13b759d46c5e8f74150add53b76daa5861120 (commit)
via b8dccb1fa84d699e1220e1a3d57b13085349b5ce (commit)
from 25e9311a9f958e3e0e42929a90f54b15997b22c0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8843744ac700dd080333e8e1bcf6273e7fe273c3
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 31 16:57:30 2018 -0500
12308: Add projects hierarchy under "home".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs_deferred.go
new file mode 100644
index 0000000..e638838
--- /dev/null
+++ b/sdk/go/arvados/fs_deferred.go
@@ -0,0 +1,103 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "log"
+ "os"
+ "sync"
+ "time"
+)
+
+func deferredCollectionFS(fs FileSystem, parent inode, coll Collection) inode {
+ var modTime time.Time
+ if coll.ModifiedAt != nil {
+ modTime = *coll.ModifiedAt
+ } else {
+ modTime = time.Now()
+ }
+ placeholder := &treenode{
+ fs: fs,
+ parent: parent,
+ inodes: nil,
+ fileinfo: fileinfo{
+ name: coll.Name,
+ modTime: modTime,
+ mode: 0755 | os.ModeDir,
+ },
+ }
+ return &deferrednode{wrapped: placeholder, create: func() inode {
+ err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+coll.UUID, nil, nil)
+ if err != nil {
+ log.Printf("BUG: unhandled error: %s", err)
+ return placeholder
+ }
+ cfs, err := coll.FileSystem(fs, fs)
+ if err != nil {
+ log.Printf("BUG: unhandled error: %s", err)
+ return placeholder
+ }
+ root := cfs.rootnode()
+ root.SetParent(parent, coll.Name)
+ return root
+ }}
+}
+
+// A deferrednode wraps an inode that's expensive to build. Initially,
+// it responds to basic directory functions by proxying to the given
+// placeholder. If a caller uses a read/write/lock operation,
+// deferrednode calls the create() func to create the real inode, and
+// proxies to the real inode from then on.
+//
+// In practice, this means a deferrednode's parent's directory listing
+// can be generated using only the placeholder, instead of waiting for
+// create().
+type deferrednode struct {
+ wrapped inode
+ create func() inode
+ mtx sync.Mutex
+ created bool
+}
+
+func (dn *deferrednode) realinode() inode {
+ dn.mtx.Lock()
+ defer dn.mtx.Unlock()
+ if !dn.created {
+ dn.wrapped = dn.create()
+ dn.created = true
+ }
+ return dn.wrapped
+}
+
+func (dn *deferrednode) currentinode() inode {
+ dn.mtx.Lock()
+ defer dn.mtx.Unlock()
+ return dn.wrapped
+}
+
+func (dn *deferrednode) Read(p []byte, pos filenodePtr) (int, filenodePtr, error) {
+ return dn.realinode().Read(p, pos)
+}
+
+func (dn *deferrednode) Write(p []byte, pos filenodePtr) (int, filenodePtr, error) {
+ return dn.realinode().Write(p, pos)
+}
+
+func (dn *deferrednode) Child(name string, replace func(inode) inode) inode {
+ return dn.realinode().Child(name, replace)
+}
+
+func (dn *deferrednode) Truncate(size int64) error { return dn.realinode().Truncate(size) }
+func (dn *deferrednode) SetParent(p inode, name string) { dn.realinode().SetParent(p, name) }
+func (dn *deferrednode) IsDir() bool { return dn.currentinode().IsDir() }
+func (dn *deferrednode) Readdir() []os.FileInfo { return dn.realinode().Readdir() }
+func (dn *deferrednode) Size() int64 { return dn.currentinode().Size() }
+func (dn *deferrednode) FileInfo() os.FileInfo { return dn.currentinode().FileInfo() }
+func (dn *deferrednode) Lock() { dn.realinode().Lock() }
+func (dn *deferrednode) Unlock() { dn.realinode().Unlock() }
+func (dn *deferrednode) RLock() { dn.realinode().RLock() }
+func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
+func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
+func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
diff --git a/sdk/go/arvados/fs_project.go b/sdk/go/arvados/fs_project.go
new file mode 100644
index 0000000..4dd8699
--- /dev/null
+++ b/sdk/go/arvados/fs_project.go
@@ -0,0 +1,113 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "log"
+ "os"
+ "sync"
+)
+
+// projectnode exposes an Arvados project as a filesystem directory.
+type projectnode struct {
+ inode
+ uuid string
+ setupOnce sync.Once
+ err error
+}
+
+func (pn *projectnode) setup() {
+ fs := pn.FS().(*siteFileSystem)
+ if pn.uuid == "" {
+ var resp User
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/users/current", nil, nil)
+ if pn.err != nil {
+ return
+ }
+ pn.uuid = resp.UUID
+ }
+ filters := []Filter{{"owner_uuid", "=", pn.uuid}}
+ params := ResourceListParams{
+ Filters: filters,
+ Order: "uuid",
+ }
+ for {
+ var resp CollectionList
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/collections", nil, params)
+ if pn.err != nil {
+ // TODO: retry on next access, instead of returning the same error forever
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, i := range resp.Items {
+ coll := i
+ if coll.Name == "" {
+ continue
+ }
+ pn.inode.Child(coll.Name, func(inode) inode {
+ return deferredCollectionFS(fs, pn, coll)
+ })
+ }
+ params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ }
+
+ filters = append(filters, Filter{"group_class", "=", "project"})
+ params.Filters = filters
+ for {
+ var resp GroupList
+ pn.err = fs.RequestAndDecode(&resp, "GET", "arvados/v1/groups", nil, params)
+ if pn.err != nil {
+ // TODO: retry on next access, instead of returning the same error forever
+ return
+ }
+ if len(resp.Items) == 0 {
+ break
+ }
+ for _, group := range resp.Items {
+ if group.Name == "" || group.Name == "." || group.Name == ".." {
+ continue
+ }
+ pn.inode.Child(group.Name, func(inode) inode {
+ return fs.newProjectNode(pn, group.Name, group.UUID)
+ })
+ }
+ params.Filters = append(filters, Filter{"uuid", ">", resp.Items[len(resp.Items)-1].UUID})
+ }
+}
+
+func (pn *projectnode) Readdir() []os.FileInfo {
+ pn.setupOnce.Do(pn.setup)
+ return pn.inode.Readdir()
+}
+
+func (pn *projectnode) Child(name string, replace func(inode) inode) inode {
+ pn.setupOnce.Do(pn.setup)
+ if pn.err != nil {
+ log.Printf("BUG: not propagating error setting up %T %v: %s", pn, pn, pn.err)
+ // TODO: propagate error, instead of just being empty
+ return nil
+ }
+ if replace == nil {
+ // lookup
+ return pn.inode.Child(name, nil)
+ }
+ return pn.inode.Child(name, func(existing inode) inode {
+ if repl := replace(existing); repl == nil {
+ // delete
+ // (TODO)
+ return pn.Child(name, nil) // not implemented
+ } else if repl.FileInfo().IsDir() {
+ // mkdir
+ // TODO: repl.SetParent(pn, name), etc.
+ return pn.Child(name, nil) // not implemented
+ } else {
+ // create file
+ // TODO: repl.SetParent(pn, name), etc.
+ return pn.Child(name, nil) // not implemented
+ }
+ })
+}
diff --git a/sdk/go/arvados/fs_site.go b/sdk/go/arvados/fs_site.go
index 37ec8a3..c8d7360 100644
--- a/sdk/go/arvados/fs_site.go
+++ b/sdk/go/arvados/fs_site.go
@@ -38,8 +38,7 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
inodes: make(map[string]inode),
}
root.inode.Child("by_id", func(inode) inode {
- var vn inode
- vn = &vdirnode{
+ return &vdirnode{
inode: &treenode{
fs: fs,
parent: fs.root,
@@ -52,7 +51,9 @@ func (c *Client) SiteFileSystem(kc keepClient) FileSystem {
},
create: fs.mountCollection,
}
- return vn
+ })
+ root.inode.Child("home", func(inode) inode {
+ return fs.newProjectNode(fs.root, "home", "")
})
return fs
}
@@ -76,6 +77,22 @@ func (fs *siteFileSystem) mountCollection(parent inode, id string) inode {
return root
}
+func (fs *siteFileSystem) newProjectNode(root inode, name, uuid string) inode {
+ return &projectnode{
+ uuid: uuid,
+ inode: &treenode{
+ fs: fs,
+ parent: root,
+ inodes: make(map[string]inode),
+ fileinfo: fileinfo{
+ name: name,
+ modTime: time.Now(),
+ mode: 0755 | os.ModeDir,
+ },
+ },
+ }
+}
+
// vdirnode wraps an inode by ignoring any requests to add/replace
// children, and calling a create() func when a non-existing child is
// looked up.
commit dbcb7c65b6324273fabc7dc2507a1978d8a538b8
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 31 13:16:00 2018 -0500
12308: Fix usage message.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/mount/command.go b/lib/mount/command.go
index 1159fbc..4acaeae 100644
--- a/lib/mount/command.go
+++ b/lib/mount/command.go
@@ -35,7 +35,7 @@ type cmd struct {
// stderr argument and prints to os.Stderr instead.
func (c *cmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
logger := log.New(stderr, prog+" ", 0)
- flags := flag.NewFlagSet(args[0], flag.ContinueOnError)
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
ro := flags.Bool("ro", false, "read-only")
experimental := flags.Bool("experimental", false, "acknowledge this is an experimental command, and should not be used in production (required)")
err := flags.Parse(args)
commit ceca47732d7c6f768e8e6f54e6fa96a49f07a553
Merge: 25e9311 cd3bab6
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Jan 31 13:13:36 2018 -0500
12308: Merge branch 'master' into 12308-go-fuse
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list