[ARVADOS] updated: 1.1.4-94-gc5633c8
Git user
git at public.curoverse.com
Fri Apr 13 22:22:26 EDT 2018
Summary of changes:
.../app/assets/javascripts/models/session_db.js | 23 +-
build/build-dev-docker-jobs-image.sh | 6 +-
build/build.list | 2 +-
.../install-nodemanager.html.textile.liquid | 8 +-
sdk/cli/arvados-cli.gemspec | 4 +-
sdk/cwl/arvados_cwl/__init__.py | 175 +++++---
sdk/cwl/arvados_cwl/arvcontainer.py | 36 +-
sdk/cwl/arvados_cwl/arvdocker.py | 2 +-
sdk/cwl/arvados_cwl/arvjob.py | 54 +--
sdk/cwl/arvados_cwl/crunch_script.py | 3 +-
sdk/cwl/arvados_cwl/runner.py | 3 +-
sdk/cwl/arvados_cwl/task_queue.py | 60 +++
sdk/cwl/tests/test_submit.py | 76 +++-
sdk/cwl/tests/test_tq.py | 50 +++
sdk/python/arvados/commands/_util.py | 19 +
sdk/python/arvados/commands/keepdocker.py | 236 ++++++-----
sdk/python/arvados/commands/put.py | 17 +-
sdk/python/arvados/safeapi.py | 6 +-
sdk/ruby/arvados.gemspec | 4 +-
services/api/script/get_anonymous_user_token.rb | 2 +-
.../crunch-dispatch-slurm.service | 1 +
services/crunch-run/copier.go | 357 ++++++++++++++++
services/crunch-run/copier_test.go | 222 ++++++++++
services/crunch-run/crunchrun.go | 470 +++++----------------
services/crunch-run/crunchrun_test.go | 269 +++---------
services/crunch-run/logging_test.go | 45 +-
services/crunch-run/upload.go | 342 ---------------
services/crunch-run/upload_test.go | 189 ---------
services/fuse/arvados_fuse/fusedir.py | 25 +-
services/fuse/tests/test_mount.py | 17 +-
services/login-sync/arvados-login-sync.gemspec | 4 +-
31 files changed, 1349 insertions(+), 1378 deletions(-)
create mode 100644 sdk/cwl/arvados_cwl/task_queue.py
create mode 100644 sdk/cwl/tests/test_tq.py
create mode 100644 services/crunch-run/copier.go
create mode 100644 services/crunch-run/copier_test.go
delete mode 100644 services/crunch-run/upload.go
delete mode 100644 services/crunch-run/upload_test.go
discards 6b784b924cdea0ac49bd0d5535ba299d4258e7fb (commit)
discards a9916f5648ad8486812c3ec8ef6f627e2f0542e4 (commit)
discards 973d9760a020b5a1f7b421e620d5143e5c4010cf (commit)
discards e618db07e97783e6ab588332d0f2872842677f03 (commit)
discards 1f397d05c069e75ebbd74b59ab2133df4f7389c7 (commit)
discards a78cdf7dfd18c32431e9d2a26f08b4a35cd4d444 (commit)
discards 3b06a75d2756e3c7b2997ba38cfd54668096ee7e (commit)
discards 309e6cb3d00a2b731ffe66d89102caacf2cd81aa (commit)
discards e7fe3ce516f71bc14c5ff0e22dcd19772ab8c72a (commit)
discards 335ee474af3cf03865bc107478da6a51e362c2ab (commit)
discards aadbd550d97c2cbf47563ceb1a9004932fa5e28f (commit)
discards a6c49634025095e9b4d29f5971fe5ecb720f9539 (commit)
discards cd0945bb8591fd97fe37a77dc9057c29dc5d7558 (commit)
discards 04d11c9cf24f358a0780edc87c910c50130fd71b (commit)
discards c5d6ae2fc5f1c382f2011a128230d893a5b2103c (commit)
discards 3e2aed249f9ae1d99fb654b19d3ae49788f8fdaa (commit)
discards 8f0b7529cda14a10cd953819af9c1b76201ae4f5 (commit)
discards 282394bf9076af3387755979e4fd9eb360291861 (commit)
discards cedd5efb0f3813125931c64b754dddabd875d537 (commit)
discards 4c46038d3726e1ff7006cc30bbd6b71b957d0215 (commit)
discards ab1abbac7ec08c7a7599670459f04b8a35dcbe68 (commit)
discards 0c7a458fc70791a5f395dad171a272b92b6ffb7f (commit)
discards efd536fbbfca8473e7df444dd1d97ce916bfe345 (commit)
discards 541e6b47d90dbdadca287bea0cc303026f20facb (commit)
discards f09c8facc9a607f3b5e9171e66170536f1ba83c7 (commit)
discards 959335d561a3882b391c88c4b2106f263e827b51 (commit)
discards 9424d875b67ddda957b2ef705a1619394b57db70 (commit)
discards 14fadeaed3dd4f01778eae8342ca0b5e190e3429 (commit)
discards 3c230a24be3e99f89805199311ec8b36665c2b74 (commit)
discards f67ede4c778f5ac6f3bd0e04d1f47993603b3375 (commit)
discards e1bdb7bb963f3754f3aebced8810b062ee4fecf8 (commit)
discards 0ff4662fab347279897b285e3ef3320497ced8c9 (commit)
discards 649c27a98f398984b6aa1f806992cfdd9831c904 (commit)
discards c2e623cbcf93e996b4fb1d29fcf99d954f7f2ad3 (commit)
discards 183cf741009a42fa3dd2af911c416755e4c5cb0e (commit)
discards 4668cf881aef0b21245a419b661b010720035f3c (commit)
via c5633c850d664d2f78e0efccf9ec9734b4e32de5 (commit)
via 98911cfe4792b20798858cefb353c451460e1a80 (commit)
via fd95e7a933b1534b76e4820838e278595b4e1220 (commit)
via 1c19ed2dd05a023be541e369130910eb277b2816 (commit)
via b1160af572bfcaeb6f0c1bab5b08987312a92814 (commit)
via 8e3adbcf390deaffed7f2449056959252e1a49f4 (commit)
via 1d1c8c8f1eda09f76a9a9730d64ae357840ffda9 (commit)
via ec0d70da440802b2859d3199e559f27cf74c035d (commit)
via 19da21ab8e56154d7db15c2643524cb8348a7a8a (commit)
via e4c5f98f696c354638bbba22ee4a1db20a52837c (commit)
via 27fac89679511949fba6d5fb29eb905c579d2d97 (commit)
via 5c4842c33069b4b908ecfe4b25185424d052a197 (commit)
via 2e950c2085596513edc9b2ccfe8134eee743516e (commit)
via a8bcd28990c6cea999b51685bac971f608d3cc39 (commit)
via 31a3b4f34aa56ade25def7cdd59b026f5a59ee13 (commit)
via f22f1e3c92e9a69e9e7f82fa6226ad100a8025da (commit)
via 01b7da36d017bd061c6f104a02471faced68c28b (commit)
via 6522329120f0e2a73bf9c55f4f937fa3d109803d (commit)
via 76cc598a22d58cd889ad41b47ef061bde90c9f52 (commit)
via a840e658554cccdf5cda70c71666a62b862f9f48 (commit)
via 179f22930a1db2734e5279075ff528fcf8d916c1 (commit)
via 78eedab0a2feadc7877529ac4ce65ccaaf4db768 (commit)
via 4dda5d7b0ea74103c07617cc2ab9e5c97682c85d (commit)
via 605b7ae8ab75664a2ff01386eeb8c0d20e3aca2a (commit)
via e0aff8f82a3ab460cb32aa6436c317ae7634dbe0 (commit)
via 20846db140dbba8a688718716e1e0f99ccfb3b51 (commit)
via 138fef8ee97f3cbd335434ad6acd26771fd0b762 (commit)
via fcdea9d131e4ba823ac8b69224d90b8eb5f4ae2e (commit)
via 86a14d03a87200728f0e8c55515b3976b94f6a62 (commit)
via 2b7dbbdec2963498b19d9df1a8a9273201ae61fb (commit)
via ff932544fa3204e12cb9f1beb13683a2c6d9e3c1 (commit)
via c816094d4f796309391c269783cb3454a4efb87d (commit)
via 1f15c5b6060b0d82c9b17fb724b7128374f65747 (commit)
via 95be914af0ab0a82c4fa92b3f9c29ebec88e8595 (commit)
via 6171961498800d18e826da1dcb1b908600eec0ac (commit)
via 312137098ee5c5384db59e49d69163cbeb8a48b8 (commit)
via 4e7de9eae07a34115cbebb074880f10b74c077d6 (commit)
via 163dac6272b8b75e7c1b73076ab922df8b03728e (commit)
via 7466d0c1af1f7d9a0c1b4ae54e9f1bbc951f2711 (commit)
via bcb6f55e2edfddd9edfa08c6463013b89ed0b78d (commit)
via cd9626d625f72b31054204c68cb2bb32e5dd3111 (commit)
via b9e031258dc079151d815167d1d6665bc63e2318 (commit)
via c415c7ce2c5e95bebd7b8649ed5863a07b208dd3 (commit)
via 9a8b51c6c2468162ee9748514141a94d24e5f663 (commit)
via fa645932e008bc03bd4906b4e4b795f22ed78fd3 (commit)
via 091c2bc58ff1a8d1c43abf0e334837d8872b914d (commit)
via 6aef698e8f1f99aa0511afd21db86d4c7cf8b5da (commit)
via 1a3de5eab1062485e6985d43bc7c3e036254efb6 (commit)
via a82b39e1849a67854cb3399a4660b77548edd580 (commit)
via 5f179cb06b6e26c3359fed97d48d13408150f6f1 (commit)
via 8c81e6c09228a9d7a3e8036624c60367615ddfc6 (commit)
via db30db0c7a441eec9d5dd2ca2b74cfde9966d6ce (commit)
via 85736e92085b10d58141ef66c1354fb1cf16c562 (commit)
via b29442ebd118d522f9be9508f5111f2a68eae1a4 (commit)
via 3cf6c6d8dcd3616d0075f9af1732d5ed4231b861 (commit)
via 540ecd0ae604df1cf02a63515e6e9e8e04e6e64a (commit)
via 69c8df415d721461135331a50e98255a625b12d1 (commit)
via 418c57bce3aac1a22548e53e1018a1547d9efee4 (commit)
via ff353df24233b40612eb228a8024ea30902397e9 (commit)
via 826bc4316d3afb0013b9951c37137323d0e00b08 (commit)
via 4647e6d7fb4a33c8896caf7f60f27df52040d45d (commit)
via 25be7a865d12707f5d2afe9300124fe4ef75145d (commit)
via 89c42b3d5d978de79797be9c9c142330d23e28c4 (commit)
via c69d271455a0550cda8e56ab6a4685fe3d7d91ea (commit)
via b5367444a51cc91e7b589bfdb68eb12e8a1f4937 (commit)
via 1b7ed029c3d9d50b275573b65e8fbf4943e76bcb (commit)
via 6b17b8e5aaf08b3124d0d7536b3c73581894d70a (commit)
via 11f2a3542e8bef42edcb413405efb43bd2e30be7 (commit)
via df591042778408d03d410d5c22a669d85652d1ea (commit)
via 29a6622585581b5e4f519968bbd291939bf49392 (commit)
via 8296c0784c70660e0a7247ba000741d37bda38fd (commit)
via d1ae12cad34862d063a1235bfe53459eef7ae589 (commit)
via 1012f12d29be01b56f2bbbe2e9bd5969d69f7b89 (commit)
via caf56e922b9caaa9a65332dc6bf3a36bd8fea48f (commit)
via 2f03d4d926870a93cb880b389519a05c97de73b3 (commit)
via f5d919c7e7bcf46e245a4459f3393022ff471db0 (commit)
via 47eb67e4c084abde49d5463d4ced8b4436a59dfd (commit)
via 7c32daf9b5b1dcb8a003ac30bfc0ed2a9ef0eb74 (commit)
via 07cb2b1d22be82abb87fd2a5f95ae86e760c87e6 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (6b784b924cdea0ac49bd0d5535ba299d4258e7fb)
\
N -- N -- N (c5633c850d664d2f78e0efccf9ec9734b4e32de5)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit c5633c850d664d2f78e0efccf9ec9734b4e32de5
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 13 16:40:56 2018 -0400
12308: Add "arvados-client mount" command via cgofuse.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index b89c8d9..2f4c491 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -75,6 +75,7 @@ lib/cli
lib/cmd
lib/crunchstat
lib/dispatchcloud
+lib/mount
services/api
services/arv-git-httpd
services/crunchstat
@@ -900,6 +901,7 @@ gostuff=(
lib/cmd
lib/crunchstat
lib/dispatchcloud
+ lib/mount
sdk/go/arvados
sdk/go/arvadosclient
sdk/go/blockdigest
diff --git a/cmd/arvados-client/Makefile b/cmd/arvados-client/Makefile
new file mode 100644
index 0000000..33fbc40
--- /dev/null
+++ b/cmd/arvados-client/Makefile
@@ -0,0 +1,11 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+all:
+ go get .
+ docker build --tag=cgofuse --build-arg=http_proxy="$(http_proxy)" --build-arg=https_proxy="$(https_proxy)" "$(GOPATH)"/src/github.com/curoverse/cgofuse
+ go get github.com/karalabe/xgo
+ xgo --image=cgofuse --targets=linux/amd64,linux/386,darwin/amd64,darwin/386,windows/amd64,windows/386 .
+ install arvados-* "$(GOPATH)"/bin/
+ rm --interactive=never arvados-*
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index b616b54..c9068fb 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -13,6 +13,7 @@ import (
"git.curoverse.com/arvados.git/lib/cli"
"git.curoverse.com/arvados.git/lib/cmd"
+ "git.curoverse.com/arvados.git/lib/mount"
)
var (
@@ -58,6 +59,8 @@ var (
"user": cli.APICall,
"virtual_machine": cli.APICall,
"workflow": cli.APICall,
+
+ "mount": mount.Command,
})
)
diff --git a/lib/mount/command.go b/lib/mount/command.go
new file mode 100644
index 0000000..4acaeae
--- /dev/null
+++ b/lib/mount/command.go
@@ -0,0 +1,76 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mount
+
+import (
+ "flag"
+ "io"
+ "log"
+ "os"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/curoverse/cgofuse/fuse"
+)
+
+var Command = &cmd{}
+
+type cmd struct {
+ // ready, if non-nil, will be closed when the mount is
+ // initialized. If ready is non-nil, it RunCommand() should
+ // not be called more than once, or when ready is already
+ // closed.
+ ready chan struct{}
+ // It is safe to call Unmount ounly after ready has been
+ // closed.
+ Unmount func() (ok bool)
+}
+
+// RunCommand implements the subcommand "mount <path> [fuse options]".
+//
+// The "-d" fuse option (and perhaps other features) ignores the
+// stderr argument and prints to os.Stderr instead.
+func (c *cmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := log.New(stderr, prog+" ", 0)
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ ro := flags.Bool("ro", false, "read-only")
+ experimental := flags.Bool("experimental", false, "acknowledge this is an experimental command, and should not be used in production (required)")
+ err := flags.Parse(args)
+ if err != nil {
+ logger.Print(err)
+ return 2
+ }
+ if !*experimental {
+ logger.Printf("error: experimental command %q used without --experimental flag", prog)
+ return 2
+ }
+
+ client := arvados.NewClientFromEnv()
+ ac, err := arvadosclient.New(client)
+ if err != nil {
+ logger.Print(err)
+ return 1
+ }
+ kc, err := keepclient.MakeKeepClient(ac)
+ if err != nil {
+ logger.Print(err)
+ return 1
+ }
+ host := fuse.NewFileSystemHost(&keepFS{
+ Client: client,
+ KeepClient: kc,
+ ReadOnly: *ro,
+ Uid: os.Getuid(),
+ Gid: os.Getgid(),
+ ready: c.ready,
+ })
+ c.Unmount = host.Unmount
+ ok := host.Mount("", flags.Args())
+ if !ok {
+ return 1
+ }
+ return 0
+}
diff --git a/lib/mount/command_test.go b/lib/mount/command_test.go
new file mode 100644
index 0000000..9cd4139
--- /dev/null
+++ b/lib/mount/command_test.go
@@ -0,0 +1,59 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mount
+
+import (
+ "bytes"
+ "io/ioutil"
+ "os"
+ "time"
+
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CmdSuite{})
+
+type CmdSuite struct {
+ mnt string
+}
+
+func (s *CmdSuite) SetUpTest(c *check.C) {
+ tmpdir, err := ioutil.TempDir("", "")
+ c.Assert(err, check.IsNil)
+ s.mnt = tmpdir
+}
+
+func (s *CmdSuite) TearDownTest(c *check.C) {
+ c.Check(os.RemoveAll(s.mnt), check.IsNil)
+}
+
+func (s *CmdSuite) TestMount(c *check.C) {
+ exited := make(chan int)
+ stdin := bytes.NewBufferString("stdin")
+ stdout := bytes.NewBuffer(nil)
+ stderr := bytes.NewBuffer(nil)
+ mountCmd := cmd{ready: make(chan struct{})}
+ ready := false
+ go func() {
+ exited <- mountCmd.RunCommand("test mount", []string{"--experimental", s.mnt}, stdin, stdout, stderr)
+ }()
+ go func() {
+ <-mountCmd.ready
+ ready = true
+ ok := mountCmd.Unmount()
+ c.Check(ok, check.Equals, true)
+ }()
+ select {
+ case <-time.After(5 * time.Second):
+ c.Fatal("timed out")
+ case errCode, ok := <-exited:
+ c.Check(ok, check.Equals, true)
+ c.Check(errCode, check.Equals, 0)
+ }
+ c.Check(ready, check.Equals, true)
+ c.Check(stdout.String(), check.Equals, "")
+ // stdin should not have been read
+ c.Check(stdin.String(), check.Equals, "stdin")
+}
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
new file mode 100644
index 0000000..68ad0b4
--- /dev/null
+++ b/lib/mount/fs.go
@@ -0,0 +1,375 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mount
+
+import (
+ "io"
+ "log"
+ "os"
+ "runtime/debug"
+ "sync"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/curoverse/cgofuse/fuse"
+)
+
+// sharedFile wraps arvados.File with a sync.Mutex, so fuse can safely
+// use a single filehandle concurrently on behalf of multiple
+// threads/processes.
+type sharedFile struct {
+ arvados.File
+ sync.Mutex
+}
+
+// keepFS implements cgofuse's FileSystemInterface.
+type keepFS struct {
+ fuse.FileSystemBase
+ Client *arvados.Client
+ KeepClient *keepclient.KeepClient
+ ReadOnly bool
+ Uid int
+ Gid int
+
+ root arvados.CustomFileSystem
+ open map[uint64]*sharedFile
+ lastFH uint64
+ sync.Mutex
+
+ // If non-nil, this channel will be closed by Init() to notify
+ // other goroutines that the mount is ready.
+ ready chan struct{}
+}
+
+var (
+ invalidFH = ^uint64(0)
+)
+
+// newFH wraps f in a sharedFile, adds it to fs's lookup table using a
+// new handle number, and returns the handle number.
+func (fs *keepFS) newFH(f arvados.File) uint64 {
+ fs.Lock()
+ defer fs.Unlock()
+ if fs.open == nil {
+ fs.open = make(map[uint64]*sharedFile)
+ }
+ fs.lastFH++
+ fh := fs.lastFH
+ fs.open[fh] = &sharedFile{File: f}
+ return fh
+}
+
+func (fs *keepFS) lookupFH(fh uint64) *sharedFile {
+ fs.Lock()
+ defer fs.Unlock()
+ return fs.open[fh]
+}
+
+func (fs *keepFS) Init() {
+ defer fs.debugPanics()
+ fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
+ fs.root.MountProject("home", "")
+ if fs.ready != nil {
+ close(fs.ready)
+ }
+}
+
+func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
+ defer fs.debugPanics()
+ if fs.ReadOnly {
+ return -fuse.EROFS, invalidFH
+ }
+ f, err := fs.root.OpenFile(path, flags|os.O_CREATE, os.FileMode(mode))
+ if err == os.ErrExist {
+ return -fuse.EEXIST, invalidFH
+ } else if err != nil {
+ return -fuse.EINVAL, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
+ defer fs.debugPanics()
+ if fs.ReadOnly && flags&(os.O_RDWR|os.O_WRONLY|os.O_CREATE) != 0 {
+ return -fuse.EROFS, invalidFH
+ }
+ f, err := fs.root.OpenFile(path, flags, 0)
+ if err != nil {
+ return -fuse.ENOENT, invalidFH
+ } else if fi, err := f.Stat(); err != nil {
+ return -fuse.EIO, invalidFH
+ } else if fi.IsDir() {
+ f.Close()
+ return -fuse.EISDIR, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Utimens(path string, tmsp []fuse.Timespec) int {
+ defer fs.debugPanics()
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ f, err := fs.root.OpenFile(path, 0, 0)
+ if err != nil {
+ return fs.errCode(err)
+ }
+ f.Close()
+ return 0
+}
+
+func (fs *keepFS) errCode(err error) int {
+ if os.IsNotExist(err) {
+ return -fuse.ENOENT
+ }
+ switch err {
+ case os.ErrExist:
+ return -fuse.EEXIST
+ case arvados.ErrInvalidArgument:
+ return -fuse.EINVAL
+ case arvados.ErrInvalidOperation:
+ return -fuse.ENOSYS
+ case arvados.ErrDirectoryNotEmpty:
+ return -fuse.ENOTEMPTY
+ case nil:
+ return 0
+ default:
+ return -fuse.EIO
+ }
+}
+
+func (fs *keepFS) Mkdir(path string, mode uint32) int {
+ defer fs.debugPanics()
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ f, err := fs.root.OpenFile(path, os.O_CREATE|os.O_EXCL, os.FileMode(mode)|os.ModeDir)
+ if err != nil {
+ return fs.errCode(err)
+ }
+ f.Close()
+ return 0
+}
+
+func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
+ defer fs.debugPanics()
+ f, err := fs.root.OpenFile(path, 0, 0)
+ if err != nil {
+ return fs.errCode(err), invalidFH
+ } else if fi, err := f.Stat(); err != nil {
+ return fs.errCode(err), invalidFH
+ } else if !fi.IsDir() {
+ f.Close()
+ return -fuse.ENOTDIR, invalidFH
+ }
+ return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Releasedir(path string, fh uint64) (errc int) {
+ defer fs.debugPanics()
+ return fs.Release(path, fh)
+}
+
+func (fs *keepFS) Rmdir(path string) int {
+ defer fs.debugPanics()
+ return fs.errCode(fs.root.Remove(path))
+}
+
+func (fs *keepFS) Release(path string, fh uint64) (errc int) {
+ defer fs.debugPanics()
+ fs.Lock()
+ defer fs.Unlock()
+ defer delete(fs.open, fh)
+ if f := fs.open[fh]; f != nil {
+ err := f.Close()
+ if err != nil {
+ return -fuse.EIO
+ }
+ }
+ return 0
+}
+
+func (fs *keepFS) Rename(oldname, newname string) (errc int) {
+ defer fs.debugPanics()
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ return fs.errCode(fs.root.Rename(oldname, newname))
+}
+
+func (fs *keepFS) Unlink(path string) (errc int) {
+ defer fs.debugPanics()
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ return fs.errCode(fs.root.Remove(path))
+}
+
+func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
+ defer fs.debugPanics()
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+
+ // Sometimes fh is a valid filehandle and we don't need to
+ // waste a name lookup.
+ if f := fs.lookupFH(fh); f != nil {
+ return fs.errCode(f.Truncate(size))
+ }
+
+ // Other times, fh is invalid and we need to lookup path.
+ f, err := fs.root.OpenFile(path, os.O_RDWR, 0)
+ if err != nil {
+ return fs.errCode(err)
+ }
+ defer f.Close()
+ return fs.errCode(f.Truncate(size))
+}
+
+func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
+ defer fs.debugPanics()
+ var fi os.FileInfo
+ var err error
+ if f := fs.lookupFH(fh); f != nil {
+ // Valid filehandle -- ignore path.
+ fi, err = f.Stat()
+ } else {
+ // Invalid filehandle -- lookup path.
+ fi, err = fs.root.Stat(path)
+ }
+ if err != nil {
+ return fs.errCode(err)
+ }
+ fs.fillStat(stat, fi)
+ return 0
+}
+
+func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ if fi, err := fs.root.Stat(path); err != nil {
+ return fs.errCode(err)
+ } else if mode & ^uint32(fuse.S_IFREG|fuse.S_IFDIR|0777) != 0 || (fi.Mode()&os.ModeDir != 0) != (mode&fuse.S_IFDIR != 0) {
+ return -fuse.ENOSYS
+ } else {
+ return 0
+ }
+}
+
+func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
+ defer fs.debugPanics()
+ var m uint32
+ if fi.IsDir() {
+ m = m | fuse.S_IFDIR
+ } else {
+ m = m | fuse.S_IFREG
+ }
+ m = m | uint32(fi.Mode()&os.ModePerm)
+ stat.Mode = m
+ stat.Nlink = 1
+ stat.Size = fi.Size()
+ t := fuse.NewTimespec(fi.ModTime())
+ stat.Mtim = t
+ stat.Ctim = t
+ stat.Atim = t
+ stat.Birthtim = t
+ stat.Blksize = 1024
+ stat.Blocks = (stat.Size + stat.Blksize - 1) / stat.Blksize
+ if fs.Uid > 0 && int64(fs.Uid) < 1<<31 {
+ stat.Uid = uint32(fs.Uid)
+ }
+ if fs.Gid > 0 && int64(fs.Gid) < 1<<31 {
+ stat.Gid = uint32(fs.Gid)
+ }
+}
+
+func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ defer fs.debugPanics()
+ if fs.ReadOnly {
+ return -fuse.EROFS
+ }
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ f.Lock()
+ defer f.Unlock()
+ if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+ return fs.errCode(err)
+ }
+ n, err := f.Write(buf)
+ if err != nil {
+ log.Printf("error writing %q: %s", path, err)
+ return fs.errCode(err)
+ }
+ return n
+}
+
+func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
+ defer fs.debugPanics()
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ f.Lock()
+ defer f.Unlock()
+ if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+ return fs.errCode(err)
+ }
+ n, err := f.Read(buf)
+ if err != nil && err != io.EOF {
+ log.Printf("error reading %q: %s", path, err)
+ return fs.errCode(err)
+ }
+ return n
+}
+
+func (fs *keepFS) Readdir(path string,
+ fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
+ ofst int64,
+ fh uint64) (errc int) {
+ defer fs.debugPanics()
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ fill(".", nil, 0)
+ fill("..", nil, 0)
+ var stat fuse.Stat_t
+ fis, err := f.Readdir(-1)
+ if err != nil {
+ return fs.errCode(err)
+ }
+ for _, fi := range fis {
+ fs.fillStat(&stat, fi)
+ fill(fi.Name(), &stat, 0)
+ }
+ return 0
+}
+
+func (fs *keepFS) Fsync(path string, datasync bool, fh uint64) int {
+ defer fs.debugPanics()
+ f := fs.lookupFH(fh)
+ if f == nil {
+ return -fuse.EBADF
+ }
+ return fs.errCode(f.Sync())
+}
+
+func (fs *keepFS) Fsyncdir(path string, datasync bool, fh uint64) int {
+ return fs.Fsync(path, datasync, fh)
+}
+
+// debugPanics (when deferred by keepFS handlers) prints an error and
+// stack trace on stderr when a handler crashes. (Without this,
+// cgofuse recovers from panics silently and returns EIO.)
+func (fs *keepFS) debugPanics() {
+ if err := recover(); err != nil {
+ log.Printf("(%T) %v", err, err)
+ debug.PrintStack()
+ panic(err)
+ }
+}
diff --git a/lib/mount/fs_test.go b/lib/mount/fs_test.go
new file mode 100644
index 0000000..1e63b76
--- /dev/null
+++ b/lib/mount/fs_test.go
@@ -0,0 +1,49 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mount
+
+import (
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
+ "github.com/curoverse/cgofuse/fuse"
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&FSSuite{})
+
+type FSSuite struct{}
+
+func (*FSSuite) TestFuseInterface(c *check.C) {
+ var _ fuse.FileSystemInterface = &keepFS{}
+}
+
+func (*FSSuite) TestOpendir(c *check.C) {
+ client := arvados.NewClientFromEnv()
+ ac, err := arvadosclient.New(client)
+ c.Assert(err, check.IsNil)
+ kc, err := keepclient.MakeKeepClient(ac)
+ c.Assert(err, check.IsNil)
+
+ var fs fuse.FileSystemInterface = &keepFS{
+ Client: client,
+ KeepClient: kc,
+ }
+ fs.Init()
+ errc, fh := fs.Opendir("/by_id")
+ c.Check(errc, check.Equals, 0)
+ c.Check(fh, check.Not(check.Equals), uint64(0))
+ c.Check(fh, check.Not(check.Equals), invalidFH)
+ errc, fh = fs.Opendir("/bogus")
+ c.Check(errc, check.Equals, -fuse.ENOENT)
+ c.Check(fh, check.Equals, invalidFH)
+}
diff --git a/vendor/vendor.json b/vendor/vendor.json
index a4f750b..7af81ba 100644
--- a/vendor/vendor.json
+++ b/vendor/vendor.json
@@ -84,6 +84,12 @@
"revisionTime": "2018-01-08T08:51:32Z"
},
{
+ "checksumSHA1": "gMBls0ytB5wHvZizUQE8Eivv9WQ=",
+ "path": "github.com/curoverse/cgofuse/fuse",
+ "revision": "d08d9e36b4ca1364eb7a4eb9db0b7fa76c9250a2",
+ "revisionTime": "2017-12-17T05:18:50Z"
+ },
+ {
"checksumSHA1": "+TKtBzv23ywvmmqRiGEjUba4YmI=",
"path": "github.com/dgrijalva/jwt-go",
"revision": "dbeaa9332f19a944acb5736b4456cfcc02140e29",
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list