[ARVADOS] updated: 1.3.0-2122-g8635f0060

Git user git at public.arvados.org
Fri Jan 31 21:38:46 UTC 2020


Summary of changes:
 apps/workbench/Gemfile.lock                        |  2 +-
 apps/workbench/app/models/container_work_unit.rb   |  2 +-
 build/run-build-packages.sh                        |  3 +
 build/run-build-test-packages-one-target.sh        |  8 +--
 build/run-library.sh                               |  4 +-
 build/version-at-commit.sh                         | 39 ++++++++--
 cmd/arvados-client/Makefile                        |  7 +-
 doc/_includes/_install_ruby_and_bundler.liquid     | 10 ++-
 doc/_layouts/default.html.liquid                   |  1 +
 .../install-dispatch-cloud.html.textile.liquid     | 10 ++-
 ...nstall-manual-prerequisites.html.textile.liquid |  4 +-
 doc/sdk/python/cookbook.html.textile.liquid        | 21 ++++++
 docker/jobs/Dockerfile                             |  7 +-
 go.mod                                             |  1 +
 go.sum                                             |  6 +-
 lib/config/config.default.yml                      |  2 +-
 lib/config/generated_config.go                     |  2 +-
 lib/mount/command.go                               |  8 +++
 lib/mount/fs.go                                    |  5 ++
 sdk/cli/Gemfile                                    |  1 +
 sdk/cwl/arvados_cwl/arvcontainer.py                |  3 +-
 sdk/cwl/arvados_cwl/fsaccess.py                    |  5 +-
 sdk/go/arvados/config.go                           |  2 +-
 sdk/python/arvados/commands/keepdocker.py          |  7 +-
 sdk/ruby/Gemfile                                   |  1 +
 services/api/Gemfile.lock                          |  2 +-
 .../app/controllers/arvados/v1/users_controller.rb |  6 +-
 services/api/app/mailers/user_notifier.rb          |  2 +-
 .../functional/arvados/v1/users_controller_test.rb |  2 +-
 services/api/test/unit/user_notifier_test.rb       |  2 +-
 services/fuse/arvados_version.py                   | 17 ++++-
 services/login-sync/Gemfile.lock                   |  4 +-
 tools/arvbox/bin/arvbox                            |  6 +-
 tools/arvbox/lib/arvbox/docker/Dockerfile.base     | 19 ++---
 tools/arvbox/lib/arvbox/docker/common.sh           | 19 ++---
 tools/arvbox/lib/arvbox/docker/service/nginx/run   | 82 +++++++++++-----------
 36 files changed, 212 insertions(+), 110 deletions(-)

  discards  8f8d7bb073b80a349525a936fdbfaec275472135 (commit)
  discards  79bf5481b1866c17b4228e0639df25b84960d731 (commit)
  discards  a4661dd0b88cccd7fc0f9e78dc73425a85c2b8ed (commit)
  discards  bf5309a0b67b14e37d78f3ae9ce33807714a47db (commit)
       via  8635f0060002ca66168e3a3f435e7200837a7dca (commit)
       via  4ea1be700f86adf823856e8d14d2224b90e73c58 (commit)
       via  315e9b521f97a408e3bc079d6f7e5f110cbf70a9 (commit)
       via  44ddbde1cd9945778ed7e4ae1d96c65e24f17742 (commit)
       via  34571842b78a039ebf931307daf9b5d87f88645a (commit)
       via  b2b6c8a9ac127ba76dc0421782503e4574837f4d (commit)
       via  7d4e2e418a987a454d18902a937fc91bed69ce75 (commit)
       via  405017353d0a74ef28288744957fd362df98c0e2 (commit)
       via  9fc2ff5583fc31c9f75f80dcf7258bed63084568 (commit)
       via  c88bd2141126b4d912ddb7f97774570ca81e6688 (commit)
       via  805883d4570e13e2760fa371e1a7714db3d8a0c9 (commit)
       via  bc4f35573b38ccde0756209057c8bce6947a38bc (commit)
       via  6e89a1027ef15b39b5d8b694939f68ff22e10c86 (commit)
       via  3c8aecebabdfeea3fd607789b48e38fba9ef5ce7 (commit)
       via  6637c4e3656d3e2334a46a7c30eaed91905a0603 (commit)
       via  28e50cc9480fdad416404542511a172cdc7253c7 (commit)
       via  6275b9ec9584e82f33688674a19225361fefa36b (commit)
       via  8b7de058f2d71698513ddbedfd5870a196c579bf (commit)
       via  16c01c88acf2a11d2e764278dcb1fb9aaa582559 (commit)
       via  18522b9cfaa23f0003094cccebfe127bf82a35f4 (commit)
       via  1b275b7390ba414f82241b23098b99654a227bf4 (commit)
       via  b871183da9ae071b277bfbd362f1cd3b0bde7076 (commit)
       via  ee38970362aa72e8aeb875ce15028fffd7834a3b (commit)
       via  a1a1e1b9e28c470c8efb27ea290ba00d48c520f2 (commit)
       via  eaebd5b26a34d1fcefb7f830b53c5eea6490dbf3 (commit)
       via  f60d2f4abd0f01bdfa2b0eac3a1f1e659d70b8ff (commit)
       via  a74bc6fadbf83e8d0555adecfd3fa776e536d6c2 (commit)
       via  1123c446a167b531b0683004ff3283d50aa037d0 (commit)
       via  92cb9d33fc73ef2b435a91631a82bc39b2542c2c (commit)
       via  2c07af6c654eaaeab46ab565d0f36f95bf430992 (commit)
       via  c8828ded72f562811dfe29ca20809dd5641f7a1d (commit)
       via  24c356c045c01884c0c37b16168d98ba16f0e1a6 (commit)
       via  d90dd5662fd356d38768b16e0f2d04fb10c19e87 (commit)
       via  6a6e6ad952cb39d62eea2b4e814ebbcc3da5cce4 (commit)
       via  b9969745321abe73dd8d2a04dc60c55fe9434ae6 (commit)

This update added new revisions after undoing existing revisions.  That is
to say, the old revision is not a strict subset of the new revision.  This
situation occurs when you --force push a change and generate a repository
containing something like this:

 * -- * -- B -- O -- O -- O (8f8d7bb073b80a349525a936fdbfaec275472135)
            \
             N -- N -- N (8635f0060002ca66168e3a3f435e7200837a7dca)

When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 8635f0060002ca66168e3a3f435e7200837a7dca
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Jan 31 16:38:32 2020 -0500

    12308: Update xgo recipe.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/cmd/arvados-client/Makefile b/cmd/arvados-client/Makefile
index 701814e38..765ab9f51 100644
--- a/cmd/arvados-client/Makefile
+++ b/cmd/arvados-client/Makefile
@@ -3,9 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 all:
-	go get .
-	docker build --tag=cgofuse --build-arg=http_proxy="$(http_proxy)" --build-arg=https_proxy="$(https_proxy)" "$(GOPATH)"/src/github.com/arvados/cgofuse
-	go get github.com/karalabe/xgo
-	xgo --image=cgofuse --targets=linux/amd64,linux/386,darwin/amd64,darwin/386,windows/amd64,windows/386 .
+	go mod download
+	docker build --tag=cgofuse --build-arg=http_proxy="$(http_proxy)" --build-arg=https_proxy="$(https_proxy)" https://github.com/arvados/cgofuse.git
+	go run github.com/karalabe/xgo --image=cgofuse --targets=linux/amd64,linux/386,darwin/amd64,darwin/386,windows/amd64,windows/386 --branch=$(shell git log -n1 --format=format:%H) --pkg=cmd/arvados-client git.arvados.org/arvados.git
 	install arvados-* "$(GOPATH)"/bin/
 	rm --interactive=never arvados-*
diff --git a/go.mod b/go.mod
index a82b9e7bc..0a4c18939 100644
--- a/go.mod
+++ b/go.mod
@@ -31,6 +31,7 @@ require (
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff
 	github.com/julienschmidt/httprouter v1.2.0
+	github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 // indirect
 	github.com/lib/pq v0.0.0-20171126050459-83612a56d3dd
 	github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c // indirect
diff --git a/go.sum b/go.sum
index b8f15cf7d..152e35d60 100644
--- a/go.sum
+++ b/go.sum
@@ -104,6 +104,8 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7 h1:AYzjK/SHz6m6mg5iuFwkrAhCc14jvCpW9d6frC9iDPE=
+github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7/go.mod h1:iYGcTYIPUvEWhFo6aKUuLchs+AV4ssYdyuBbQJZGcBk=
 github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 h1:xXn0nBttYwok7DhU4RxqaADEpQn7fEMt5kKc3yoj/n0=
 github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=

commit 4ea1be700f86adf823856e8d14d2224b90e73c58
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Jan 31 16:25:27 2020 -0500

    12308: go mod tidy
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/go.sum b/go.sum
index 1136fbf4b..b8f15cf7d 100644
--- a/go.sum
+++ b/go.sum
@@ -17,8 +17,6 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/arvados/cgofuse v1.0.4 h1:FgxrUZMI4PQO7rZg/l8MXpKKr+1w4TEJaY3L9OGGbF8=
-github.com/arvados/cgofuse v1.0.4/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
 github.com/arvados/cgofuse v1.2.1-0.20200117191602-0a6595d3faa1 h1:rJ20nQahPEFL/f6v0a9dLVlkWs71qN7xGkmhzgGTCys=
 github.com/arvados/cgofuse v1.2.1-0.20200117191602-0a6595d3faa1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
 github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
@@ -36,8 +34,6 @@ github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7 h1:e3u8KWFMR3irlDo1Z/tL8Hsz1MJmCLkSoX5AZRMKZkg=
 github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/curoverse/goamz v0.0.0-20190905141525-1bba09f407ef h1:k3Q9m06dbTShrR4phl/QNi15ZSPkIwgyQmNvJRcXR3Y=
-github.com/curoverse/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:NUkr+hZ9k+l0cEXg9S7EW8+UIfPkP/hNy2Ga0QVPZ88=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=

commit 315e9b521f97a408e3bc079d6f7e5f110cbf70a9
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Thu Jan 30 10:00:17 2020 -0500

    12308: Add pprof flag.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/mount/command.go b/lib/mount/command.go
index da3f97b46..5b331f0ce 100644
--- a/lib/mount/command.go
+++ b/lib/mount/command.go
@@ -8,6 +8,8 @@ import (
 	"flag"
 	"io"
 	"log"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
@@ -39,6 +41,7 @@ func (c *cmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, st
 	ro := flags.Bool("ro", false, "read-only")
 	experimental := flags.Bool("experimental", false, "acknowledge this is an experimental command, and should not be used in production (required)")
 	blockCache := flags.Int("block-cache", 4, "read cache size (number of 64MiB blocks)")
+	pprof := flags.String("pprof", "", "serve Go profile data at `[addr]:port`")
 	err := flags.Parse(args)
 	if err != nil {
 		logger.Print(err)
@@ -48,6 +51,11 @@ func (c *cmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, st
 		logger.Printf("error: experimental command %q used without --experimental flag", prog)
 		return 2
 	}
+	if *pprof != "" {
+		go func() {
+			log.Println(http.ListenAndServe(*pprof, nil))
+		}()
+	}
 
 	client := arvados.NewClientFromEnv()
 	ac, err := arvadosclient.New(client)

commit 44ddbde1cd9945778ed7e4ae1d96c65e24f17742
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Tue Jan 21 11:34:57 2020 -0500

    12308: Avoid returning partial reads before EOF.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/mount/fs.go b/lib/mount/fs.go
index 5b6fc0012..774948f57 100644
--- a/lib/mount/fs.go
+++ b/lib/mount/fs.go
@@ -320,6 +320,16 @@ func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
 		return fs.errCode(err)
 	}
 	n, err := f.Read(buf)
+	for err == nil && n < len(buf) {
+		// f is an io.Reader ("If some data is available but
+		// not len(p) bytes, Read conventionally returns what
+		// is available instead of waiting for more") -- but
+		// our caller requires us to either fill buf or reach
+		// EOF.
+		done := n
+		n, err = f.Read(buf[done:])
+		n += done
+	}
 	if err != nil && err != io.EOF {
 		log.Printf("error reading %q: %s", path, err)
 		return fs.errCode(err)

commit 34571842b78a039ebf931307daf9b5d87f88645a
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Jan 17 15:10:41 2020 -0500

    12308: Update cgofuse.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/go.mod b/go.mod
index 1bf5430e0..a82b9e7bc 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/Microsoft/go-winio v0.4.5 // indirect
 	github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
-	github.com/arvados/cgofuse v1.0.4
+	github.com/arvados/cgofuse v1.2.1-0.20200117191602-0a6595d3faa1
 	github.com/aws/aws-sdk-go v1.25.30
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7
diff --git a/go.sum b/go.sum
index 7c20dc6ba..1136fbf4b 100644
--- a/go.sum
+++ b/go.sum
@@ -19,6 +19,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/arvados/cgofuse v1.0.4 h1:FgxrUZMI4PQO7rZg/l8MXpKKr+1w4TEJaY3L9OGGbF8=
 github.com/arvados/cgofuse v1.0.4/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
+github.com/arvados/cgofuse v1.2.1-0.20200117191602-0a6595d3faa1 h1:rJ20nQahPEFL/f6v0a9dLVlkWs71qN7xGkmhzgGTCys=
+github.com/arvados/cgofuse v1.2.1-0.20200117191602-0a6595d3faa1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
 github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
 github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:rCtgyMmBGEbjTm37fCuBYbNL0IhztiALzo3OB9HyiOM=
 github.com/aws/aws-sdk-go v1.25.30 h1:I9qj6zW3mMfsg91e+GMSN/INcaX9tTFvr/l/BAHKaIY=

commit b2b6c8a9ac127ba76dc0421782503e4574837f4d
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Jan 17 13:51:10 2020 -0500

    12308: Configurable read cache size.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/mount/command.go b/lib/mount/command.go
index f99e6da23..da3f97b46 100644
--- a/lib/mount/command.go
+++ b/lib/mount/command.go
@@ -38,6 +38,7 @@ func (c *cmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, st
 	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
 	ro := flags.Bool("ro", false, "read-only")
 	experimental := flags.Bool("experimental", false, "acknowledge this is an experimental command, and should not be used in production (required)")
+	blockCache := flags.Int("block-cache", 4, "read cache size (number of 64MiB blocks)")
 	err := flags.Parse(args)
 	if err != nil {
 		logger.Print(err)
@@ -59,6 +60,7 @@ func (c *cmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, st
 		logger.Print(err)
 		return 1
 	}
+	kc.BlockCache = &keepclient.BlockCache{MaxBlocks: *blockCache}
 	host := fuse.NewFileSystemHost(&keepFS{
 		Client:     client,
 		KeepClient: kc,

commit 7d4e2e418a987a454d18902a937fc91bed69ce75
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Jan 17 13:51:05 2020 -0500

    12308: Add "arvados-client mount" command via cgofuse.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index f21861762..8fe51e479 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -90,6 +90,7 @@ lib/dispatchcloud/container
 lib/dispatchcloud/scheduler
 lib/dispatchcloud/ssh_executor
 lib/dispatchcloud/worker
+lib/mount
 lib/service
 services/api
 services/arv-git-httpd
diff --git a/cmd/arvados-client/Makefile b/cmd/arvados-client/Makefile
new file mode 100644
index 000000000..701814e38
--- /dev/null
+++ b/cmd/arvados-client/Makefile
@@ -0,0 +1,11 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+all:
+	go get .
+	docker build --tag=cgofuse --build-arg=http_proxy="$(http_proxy)" --build-arg=https_proxy="$(https_proxy)" "$(GOPATH)"/src/github.com/arvados/cgofuse
+	go get github.com/karalabe/xgo
+	xgo --image=cgofuse --targets=linux/amd64,linux/386,darwin/amd64,darwin/386,windows/amd64,windows/386 .
+	install arvados-* "$(GOPATH)"/bin/
+	rm --interactive=never arvados-*
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index bc6c7f002..887bc62bb 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -9,6 +9,7 @@ import (
 
 	"git.arvados.org/arvados.git/lib/cli"
 	"git.arvados.org/arvados.git/lib/cmd"
+	"git.arvados.org/arvados.git/lib/mount"
 )
 
 var (
@@ -50,6 +51,8 @@ var (
 		"user":                     cli.APICall,
 		"virtual_machine":          cli.APICall,
 		"workflow":                 cli.APICall,
+
+		"mount": mount.Command,
 	})
 )
 
diff --git a/go.mod b/go.mod
index 033723d23..1bf5430e0 100644
--- a/go.mod
+++ b/go.mod
@@ -9,6 +9,7 @@ require (
 	github.com/Microsoft/go-winio v0.4.5 // indirect
 	github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 // indirect
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
+	github.com/arvados/cgofuse v1.0.4
 	github.com/aws/aws-sdk-go v1.25.30
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7
diff --git a/go.sum b/go.sum
index d7a022dda..7c20dc6ba 100644
--- a/go.sum
+++ b/go.sum
@@ -17,6 +17,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/arvados/cgofuse v1.0.4 h1:FgxrUZMI4PQO7rZg/l8MXpKKr+1w4TEJaY3L9OGGbF8=
+github.com/arvados/cgofuse v1.0.4/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
 github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
 github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef/go.mod h1:rCtgyMmBGEbjTm37fCuBYbNL0IhztiALzo3OB9HyiOM=
 github.com/aws/aws-sdk-go v1.25.30 h1:I9qj6zW3mMfsg91e+GMSN/INcaX9tTFvr/l/BAHKaIY=
diff --git a/lib/mount/command.go b/lib/mount/command.go
new file mode 100644
index 000000000..f99e6da23
--- /dev/null
+++ b/lib/mount/command.go
@@ -0,0 +1,76 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mount
+
+import (
+	"flag"
+	"io"
+	"log"
+	"os"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+	"git.arvados.org/arvados.git/sdk/go/keepclient"
+	"github.com/arvados/cgofuse/fuse"
+)
+
+var Command = &cmd{}
+
+type cmd struct {
+	// ready, if non-nil, will be closed when the mount is
+	// initialized.  If ready is non-nil, it RunCommand() should
+	// not be called more than once, or when ready is already
+	// closed.
+	ready chan struct{}
+	// It is safe to call Unmount ounly after ready has been
+	// closed.
+	Unmount func() (ok bool)
+}
+
+// RunCommand implements the subcommand "mount <path> [fuse options]".
+//
+// The "-d" fuse option (and perhaps other features) ignores the
+// stderr argument and prints to os.Stderr instead.
+func (c *cmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	logger := log.New(stderr, prog+" ", 0)
+	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+	ro := flags.Bool("ro", false, "read-only")
+	experimental := flags.Bool("experimental", false, "acknowledge this is an experimental command, and should not be used in production (required)")
+	err := flags.Parse(args)
+	if err != nil {
+		logger.Print(err)
+		return 2
+	}
+	if !*experimental {
+		logger.Printf("error: experimental command %q used without --experimental flag", prog)
+		return 2
+	}
+
+	client := arvados.NewClientFromEnv()
+	ac, err := arvadosclient.New(client)
+	if err != nil {
+		logger.Print(err)
+		return 1
+	}
+	kc, err := keepclient.MakeKeepClient(ac)
+	if err != nil {
+		logger.Print(err)
+		return 1
+	}
+	host := fuse.NewFileSystemHost(&keepFS{
+		Client:     client,
+		KeepClient: kc,
+		ReadOnly:   *ro,
+		Uid:        os.Getuid(),
+		Gid:        os.Getgid(),
+		ready:      c.ready,
+	})
+	c.Unmount = host.Unmount
+	ok := host.Mount("", flags.Args())
+	if !ok {
+		return 1
+	}
+	return 0
+}
diff --git a/lib/mount/command_test.go b/lib/mount/command_test.go
new file mode 100644
index 000000000..9cd413979
--- /dev/null
+++ b/lib/mount/command_test.go
@@ -0,0 +1,59 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mount
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"time"
+
+	check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&CmdSuite{})
+
+type CmdSuite struct {
+	mnt string
+}
+
+func (s *CmdSuite) SetUpTest(c *check.C) {
+	tmpdir, err := ioutil.TempDir("", "")
+	c.Assert(err, check.IsNil)
+	s.mnt = tmpdir
+}
+
+func (s *CmdSuite) TearDownTest(c *check.C) {
+	c.Check(os.RemoveAll(s.mnt), check.IsNil)
+}
+
+func (s *CmdSuite) TestMount(c *check.C) {
+	exited := make(chan int)
+	stdin := bytes.NewBufferString("stdin")
+	stdout := bytes.NewBuffer(nil)
+	stderr := bytes.NewBuffer(nil)
+	mountCmd := cmd{ready: make(chan struct{})}
+	ready := false
+	go func() {
+		exited <- mountCmd.RunCommand("test mount", []string{"--experimental", s.mnt}, stdin, stdout, stderr)
+	}()
+	go func() {
+		<-mountCmd.ready
+		ready = true
+		ok := mountCmd.Unmount()
+		c.Check(ok, check.Equals, true)
+	}()
+	select {
+	case <-time.After(5 * time.Second):
+		c.Fatal("timed out")
+	case errCode, ok := <-exited:
+		c.Check(ok, check.Equals, true)
+		c.Check(errCode, check.Equals, 0)
+	}
+	c.Check(ready, check.Equals, true)
+	c.Check(stdout.String(), check.Equals, "")
+	// stdin should not have been read
+	c.Check(stdin.String(), check.Equals, "stdin")
+}
diff --git a/lib/mount/fs.go b/lib/mount/fs.go
new file mode 100644
index 000000000..5b6fc0012
--- /dev/null
+++ b/lib/mount/fs.go
@@ -0,0 +1,375 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mount
+
+import (
+	"io"
+	"log"
+	"os"
+	"runtime/debug"
+	"sync"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/keepclient"
+	"github.com/arvados/cgofuse/fuse"
+)
+
+// sharedFile wraps arvados.File with a sync.Mutex, so fuse can safely
+// use a single filehandle concurrently on behalf of multiple
+// threads/processes.
+type sharedFile struct {
+	arvados.File
+	sync.Mutex
+}
+
+// keepFS implements cgofuse's FileSystemInterface.
+type keepFS struct {
+	fuse.FileSystemBase
+	Client     *arvados.Client
+	KeepClient *keepclient.KeepClient
+	ReadOnly   bool
+	Uid        int
+	Gid        int
+
+	root   arvados.CustomFileSystem
+	open   map[uint64]*sharedFile
+	lastFH uint64
+	sync.Mutex
+
+	// If non-nil, this channel will be closed by Init() to notify
+	// other goroutines that the mount is ready.
+	ready chan struct{}
+}
+
+var (
+	invalidFH = ^uint64(0)
+)
+
+// newFH wraps f in a sharedFile, adds it to fs's lookup table using a
+// new handle number, and returns the handle number.
+func (fs *keepFS) newFH(f arvados.File) uint64 {
+	fs.Lock()
+	defer fs.Unlock()
+	if fs.open == nil {
+		fs.open = make(map[uint64]*sharedFile)
+	}
+	fs.lastFH++
+	fh := fs.lastFH
+	fs.open[fh] = &sharedFile{File: f}
+	return fh
+}
+
+func (fs *keepFS) lookupFH(fh uint64) *sharedFile {
+	fs.Lock()
+	defer fs.Unlock()
+	return fs.open[fh]
+}
+
+func (fs *keepFS) Init() {
+	defer fs.debugPanics()
+	fs.root = fs.Client.SiteFileSystem(fs.KeepClient)
+	fs.root.MountProject("home", "")
+	if fs.ready != nil {
+		close(fs.ready)
+	}
+}
+
+func (fs *keepFS) Create(path string, flags int, mode uint32) (errc int, fh uint64) {
+	defer fs.debugPanics()
+	if fs.ReadOnly {
+		return -fuse.EROFS, invalidFH
+	}
+	f, err := fs.root.OpenFile(path, flags|os.O_CREATE, os.FileMode(mode))
+	if err == os.ErrExist {
+		return -fuse.EEXIST, invalidFH
+	} else if err != nil {
+		return -fuse.EINVAL, invalidFH
+	}
+	return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Open(path string, flags int) (errc int, fh uint64) {
+	defer fs.debugPanics()
+	if fs.ReadOnly && flags&(os.O_RDWR|os.O_WRONLY|os.O_CREATE) != 0 {
+		return -fuse.EROFS, invalidFH
+	}
+	f, err := fs.root.OpenFile(path, flags, 0)
+	if err != nil {
+		return -fuse.ENOENT, invalidFH
+	} else if fi, err := f.Stat(); err != nil {
+		return -fuse.EIO, invalidFH
+	} else if fi.IsDir() {
+		f.Close()
+		return -fuse.EISDIR, invalidFH
+	}
+	return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Utimens(path string, tmsp []fuse.Timespec) int {
+	defer fs.debugPanics()
+	if fs.ReadOnly {
+		return -fuse.EROFS
+	}
+	f, err := fs.root.OpenFile(path, 0, 0)
+	if err != nil {
+		return fs.errCode(err)
+	}
+	f.Close()
+	return 0
+}
+
+func (fs *keepFS) errCode(err error) int {
+	if os.IsNotExist(err) {
+		return -fuse.ENOENT
+	}
+	switch err {
+	case os.ErrExist:
+		return -fuse.EEXIST
+	case arvados.ErrInvalidArgument:
+		return -fuse.EINVAL
+	case arvados.ErrInvalidOperation:
+		return -fuse.ENOSYS
+	case arvados.ErrDirectoryNotEmpty:
+		return -fuse.ENOTEMPTY
+	case nil:
+		return 0
+	default:
+		return -fuse.EIO
+	}
+}
+
+func (fs *keepFS) Mkdir(path string, mode uint32) int {
+	defer fs.debugPanics()
+	if fs.ReadOnly {
+		return -fuse.EROFS
+	}
+	f, err := fs.root.OpenFile(path, os.O_CREATE|os.O_EXCL, os.FileMode(mode)|os.ModeDir)
+	if err != nil {
+		return fs.errCode(err)
+	}
+	f.Close()
+	return 0
+}
+
+func (fs *keepFS) Opendir(path string) (errc int, fh uint64) {
+	defer fs.debugPanics()
+	f, err := fs.root.OpenFile(path, 0, 0)
+	if err != nil {
+		return fs.errCode(err), invalidFH
+	} else if fi, err := f.Stat(); err != nil {
+		return fs.errCode(err), invalidFH
+	} else if !fi.IsDir() {
+		f.Close()
+		return -fuse.ENOTDIR, invalidFH
+	}
+	return 0, fs.newFH(f)
+}
+
+func (fs *keepFS) Releasedir(path string, fh uint64) (errc int) {
+	defer fs.debugPanics()
+	return fs.Release(path, fh)
+}
+
+func (fs *keepFS) Rmdir(path string) int {
+	defer fs.debugPanics()
+	return fs.errCode(fs.root.Remove(path))
+}
+
+func (fs *keepFS) Release(path string, fh uint64) (errc int) {
+	defer fs.debugPanics()
+	fs.Lock()
+	defer fs.Unlock()
+	defer delete(fs.open, fh)
+	if f := fs.open[fh]; f != nil {
+		err := f.Close()
+		if err != nil {
+			return -fuse.EIO
+		}
+	}
+	return 0
+}
+
+func (fs *keepFS) Rename(oldname, newname string) (errc int) {
+	defer fs.debugPanics()
+	if fs.ReadOnly {
+		return -fuse.EROFS
+	}
+	return fs.errCode(fs.root.Rename(oldname, newname))
+}
+
+func (fs *keepFS) Unlink(path string) (errc int) {
+	defer fs.debugPanics()
+	if fs.ReadOnly {
+		return -fuse.EROFS
+	}
+	return fs.errCode(fs.root.Remove(path))
+}
+
+func (fs *keepFS) Truncate(path string, size int64, fh uint64) (errc int) {
+	defer fs.debugPanics()
+	if fs.ReadOnly {
+		return -fuse.EROFS
+	}
+
+	// Sometimes fh is a valid filehandle and we don't need to
+	// waste a name lookup.
+	if f := fs.lookupFH(fh); f != nil {
+		return fs.errCode(f.Truncate(size))
+	}
+
+	// Other times, fh is invalid and we need to lookup path.
+	f, err := fs.root.OpenFile(path, os.O_RDWR, 0)
+	if err != nil {
+		return fs.errCode(err)
+	}
+	defer f.Close()
+	return fs.errCode(f.Truncate(size))
+}
+
+func (fs *keepFS) Getattr(path string, stat *fuse.Stat_t, fh uint64) (errc int) {
+	defer fs.debugPanics()
+	var fi os.FileInfo
+	var err error
+	if f := fs.lookupFH(fh); f != nil {
+		// Valid filehandle -- ignore path.
+		fi, err = f.Stat()
+	} else {
+		// Invalid filehandle -- lookup path.
+		fi, err = fs.root.Stat(path)
+	}
+	if err != nil {
+		return fs.errCode(err)
+	}
+	fs.fillStat(stat, fi)
+	return 0
+}
+
+func (fs *keepFS) Chmod(path string, mode uint32) (errc int) {
+	if fs.ReadOnly {
+		return -fuse.EROFS
+	}
+	if fi, err := fs.root.Stat(path); err != nil {
+		return fs.errCode(err)
+	} else if mode & ^uint32(fuse.S_IFREG|fuse.S_IFDIR|0777) != 0 || (fi.Mode()&os.ModeDir != 0) != (mode&fuse.S_IFDIR != 0) {
+		return -fuse.ENOSYS
+	} else {
+		return 0
+	}
+}
+
+func (fs *keepFS) fillStat(stat *fuse.Stat_t, fi os.FileInfo) {
+	defer fs.debugPanics()
+	var m uint32
+	if fi.IsDir() {
+		m = m | fuse.S_IFDIR
+	} else {
+		m = m | fuse.S_IFREG
+	}
+	m = m | uint32(fi.Mode()&os.ModePerm)
+	stat.Mode = m
+	stat.Nlink = 1
+	stat.Size = fi.Size()
+	t := fuse.NewTimespec(fi.ModTime())
+	stat.Mtim = t
+	stat.Ctim = t
+	stat.Atim = t
+	stat.Birthtim = t
+	stat.Blksize = 1024
+	stat.Blocks = (stat.Size + stat.Blksize - 1) / stat.Blksize
+	if fs.Uid > 0 && int64(fs.Uid) < 1<<31 {
+		stat.Uid = uint32(fs.Uid)
+	}
+	if fs.Gid > 0 && int64(fs.Gid) < 1<<31 {
+		stat.Gid = uint32(fs.Gid)
+	}
+}
+
+func (fs *keepFS) Write(path string, buf []byte, ofst int64, fh uint64) (n int) {
+	defer fs.debugPanics()
+	if fs.ReadOnly {
+		return -fuse.EROFS
+	}
+	f := fs.lookupFH(fh)
+	if f == nil {
+		return -fuse.EBADF
+	}
+	f.Lock()
+	defer f.Unlock()
+	if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+		return fs.errCode(err)
+	}
+	n, err := f.Write(buf)
+	if err != nil {
+		log.Printf("error writing %q: %s", path, err)
+		return fs.errCode(err)
+	}
+	return n
+}
+
+func (fs *keepFS) Read(path string, buf []byte, ofst int64, fh uint64) (n int) {
+	defer fs.debugPanics()
+	f := fs.lookupFH(fh)
+	if f == nil {
+		return -fuse.EBADF
+	}
+	f.Lock()
+	defer f.Unlock()
+	if _, err := f.Seek(ofst, io.SeekStart); err != nil {
+		return fs.errCode(err)
+	}
+	n, err := f.Read(buf)
+	if err != nil && err != io.EOF {
+		log.Printf("error reading %q: %s", path, err)
+		return fs.errCode(err)
+	}
+	return n
+}
+
+func (fs *keepFS) Readdir(path string,
+	fill func(name string, stat *fuse.Stat_t, ofst int64) bool,
+	ofst int64,
+	fh uint64) (errc int) {
+	defer fs.debugPanics()
+	f := fs.lookupFH(fh)
+	if f == nil {
+		return -fuse.EBADF
+	}
+	fill(".", nil, 0)
+	fill("..", nil, 0)
+	var stat fuse.Stat_t
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		return fs.errCode(err)
+	}
+	for _, fi := range fis {
+		fs.fillStat(&stat, fi)
+		fill(fi.Name(), &stat, 0)
+	}
+	return 0
+}
+
+func (fs *keepFS) Fsync(path string, datasync bool, fh uint64) int {
+	defer fs.debugPanics()
+	f := fs.lookupFH(fh)
+	if f == nil {
+		return -fuse.EBADF
+	}
+	return fs.errCode(f.Sync())
+}
+
+func (fs *keepFS) Fsyncdir(path string, datasync bool, fh uint64) int {
+	return fs.Fsync(path, datasync, fh)
+}
+
+// debugPanics (when deferred by keepFS handlers) prints an error and
+// stack trace on stderr when a handler crashes. (Without this,
+// cgofuse recovers from panics silently and returns EIO.)
+func (fs *keepFS) debugPanics() {
+	if err := recover(); err != nil {
+		log.Printf("(%T) %v", err, err)
+		debug.PrintStack()
+		panic(err)
+	}
+}
diff --git a/lib/mount/fs_test.go b/lib/mount/fs_test.go
new file mode 100644
index 000000000..fef2c0f06
--- /dev/null
+++ b/lib/mount/fs_test.go
@@ -0,0 +1,49 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mount
+
+import (
+	"testing"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+	"git.arvados.org/arvados.git/sdk/go/keepclient"
+	"github.com/arvados/cgofuse/fuse"
+	check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+	check.TestingT(t)
+}
+
+var _ = check.Suite(&FSSuite{})
+
+type FSSuite struct{}
+
+func (*FSSuite) TestFuseInterface(c *check.C) {
+	var _ fuse.FileSystemInterface = &keepFS{}
+}
+
+func (*FSSuite) TestOpendir(c *check.C) {
+	client := arvados.NewClientFromEnv()
+	ac, err := arvadosclient.New(client)
+	c.Assert(err, check.IsNil)
+	kc, err := keepclient.MakeKeepClient(ac)
+	c.Assert(err, check.IsNil)
+
+	var fs fuse.FileSystemInterface = &keepFS{
+		Client:     client,
+		KeepClient: kc,
+	}
+	fs.Init()
+	errc, fh := fs.Opendir("/by_id")
+	c.Check(errc, check.Equals, 0)
+	c.Check(fh, check.Not(check.Equals), uint64(0))
+	c.Check(fh, check.Not(check.Equals), invalidFH)
+	errc, fh = fs.Opendir("/bogus")
+	c.Check(errc, check.Equals, -fuse.ENOENT)
+	c.Check(fh, check.Equals, invalidFH)
+}

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list