[arvados] created: 2.7.0-39-g2af7f0d3fe
git repository hosting
git at public.arvados.org
Wed Dec 6 22:12:08 UTC 2023
at 2af7f0d3fe7cb0634e0496e12b8332136cb481ac (commit)
commit 2af7f0d3fe7cb0634e0496e12b8332136cb481ac
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Sep 29 18:48:41 2023 -0400
21028: Build Ruby from source on rocky8
Same rationale as fe8da34676be7c6fbb1042fcfcabee19bfa424d0.
Refs #21028.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/package-test-dockerfiles/rocky8/Dockerfile b/build/package-test-dockerfiles/rocky8/Dockerfile
index aba9ae5e94..809f3626ca 100644
--- a/build/package-test-dockerfiles/rocky8/Dockerfile
+++ b/build/package-test-dockerfiles/rocky8/Dockerfile
@@ -41,7 +41,7 @@ RUN touch /var/lib/rpm/* && \
gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm install --disable-binary 2.7 -j $(grep -c processor /proc/cpuinfo) && \
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19
commit ac3e721dd5851da5aa6153c350fa33a10f67063a
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Sep 29 15:23:56 2023 -0400
21028: Build Ruby from source on rocky8
Without this, RVM downloads a binary Ruby with a bunch of configuration
options turned on. This can lead to problems building extensions when
Ruby thinks a certain feature is supported but our Docker image doesn't
have the headers to support it. Right now we're seeing that in the zlib
gem: it has extra code to turn on when Ruby supports Valgrind, but we
don't have the Valgrind headers necessary to build that code.
Closes #21028.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/package-build-dockerfiles/rocky8/Dockerfile b/build/package-build-dockerfiles/rocky8/Dockerfile
index c410a8b591..138a10b15c 100644
--- a/build/package-build-dockerfiles/rocky8/Dockerfile
+++ b/build/package-build-dockerfiles/rocky8/Dockerfile
@@ -70,7 +70,7 @@ ADD generated/pkuczynski.asc /tmp/
RUN gpg --import --no-tty /tmp/mpapis.asc && \
gpg --import --no-tty /tmp/pkuczynski.asc && \
curl -L https://get.rvm.io | bash -s stable && \
- /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \
+ /usr/local/rvm/bin/rvm install --disable-binary 2.7 -j $(grep -c processor /proc/cpuinfo) && \
/usr/local/rvm/bin/rvm alias create default ruby-2.7 && \
echo "gem: --no-document" >> ~/.gemrc && \
/usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \
commit 014f114d110580f89ffb28a794ba09d57ea63fd7
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Dec 6 16:14:02 2023 -0500
Update upgrading notes, refs #21089
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 3e7428eb17..faa8c0631d 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -28,10 +28,16 @@ TODO: extract this information based on git commit messages and generate changel
<div class="releasenotes">
</notextile>
-h2(#2_7_1). v2.7.1 (2023-11-29)
+h2(#2_7_1). v2.7.1 (2023-12-07)
"previous: Upgrading to 2.7.0":#v2_7_0
+h3. Separate configs for MaxConcurrentRequests and MaxConcurrentRailsRequests
+
+The default configuration value @API.MaxConcurrentRequests@ (the number of concurrent requests that will be processed by a single instance of an arvados service process) is raised from 8 to 64.
+
+A new configuration key @API.MaxConcurrentRailsRequests@ (default 8) limits the number of concurrent requests processed by a RailsAPI service process.
+
h3. Check implications of Containers.MaximumPriceFactor 1.5
When scheduling a container, Arvados now considers using instance types other than the lowest-cost type consistent with the container's resource constraints. If a larger instance is already running and idle, or the cloud provider reports that the optimal instance type is not currently available, Arvados will select a larger instance type, provided the cost does not exceed 1.5x the optimal instance type cost.
@@ -42,12 +48,6 @@ h3. Synchronize keepstore and keep-balance upgrades
The internal communication between keepstore and keep-balance about read-only volumes has changed. After keep-balance is upgraded, old versions of keepstore will be treated as read-only. We recommend upgrading and restarting all keepstore services first, then upgrading and restarting keep-balance.
-h3. Separate configs for MaxConcurrentRequests and MaxConcurrentRailsRequests
-
-The default configuration value @API.MaxConcurrentRequests@ (the number of concurrent requests that will be processed by a single instance of an arvados service process) is raised from 8 to 64.
-
-A new configuration key @API.MaxConcurrentRailsRequests@ (default 8) limits the number of concurrent requests processed by a RailsAPI service process.
-
h2(#v2_7_0). v2.7.0 (2023-09-21)
"previous: Upgrading to 2.6.3":#v2_6_3
commit 1dd06b9c849b4cf20b6f21e6004d5b11234cf348
Author: Tom Clegg <tom at curii.com>
Date: Wed Dec 6 11:55:52 2023 -0500
Merge branch '21227-keep-web-panic'
fixes #21227
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 586b724b69..28051318b2 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1000,6 +1000,7 @@ test_gofmt() {
cd "$WORKSPACE" || return 1
dirs=$(ls -d */ | egrep -v 'vendor|tmp')
[[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
+ go vet -composites=false ./...
}
test_services/api() {
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 55fe9d2e37..19649b755b 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -112,7 +112,7 @@ func (super *Supervisor) Start(ctx context.Context) {
super.ctx, super.cancel = context.WithCancel(ctx)
super.done = make(chan struct{})
- sigch := make(chan os.Signal)
+ sigch := make(chan os.Signal, 1)
signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
go func() {
defer signal.Stop(sigch)
@@ -205,15 +205,24 @@ func (super *Supervisor) Wait() error {
func (super *Supervisor) startFederation(cfg *arvados.Config) {
super.children = map[string]*Supervisor{}
for id, cc := range cfg.Clusters {
- super2 := *super
yaml, err := json.Marshal(arvados.Config{Clusters: map[string]arvados.Cluster{id: cc}})
if err != nil {
panic(fmt.Sprintf("json.Marshal partial config: %s", err))
}
- super2.ConfigYAML = string(yaml)
- super2.ConfigPath = "-"
- super2.children = nil
-
+ super2 := &Supervisor{
+ ConfigPath: "-",
+ ConfigYAML: string(yaml),
+ SourcePath: super.SourcePath,
+ SourceVersion: super.SourceVersion,
+ ClusterType: super.ClusterType,
+ ListenHost: super.ListenHost,
+ ControllerAddr: super.ControllerAddr,
+ NoWorkbench1: super.NoWorkbench1,
+ NoWorkbench2: super.NoWorkbench2,
+ OwnTemporaryDatabase: super.OwnTemporaryDatabase,
+ Stdin: super.Stdin,
+ Stderr: super.Stderr,
+ }
if super2.ClusterType == "test" {
super2.Stderr = &service.LogPrefixer{
Writer: super.Stderr,
@@ -221,7 +230,7 @@ func (super *Supervisor) startFederation(cfg *arvados.Config) {
}
}
super2.Start(super.ctx)
- super.children[id] = &super2
+ super.children[id] = super2
}
}
diff --git a/lib/controller/localdb/container_test.go b/lib/controller/localdb/container_test.go
index 65d9fac5bb..0a74d52807 100644
--- a/lib/controller/localdb/container_test.go
+++ b/lib/controller/localdb/container_test.go
@@ -204,6 +204,7 @@ func (s *containerSuite) TestUpdatePriorityMultiLevelWorkflow(c *C) {
defer deadlockCancel()
for _, cr := range allcrs {
if strings.Contains(cr.Command[2], " j ") && !strings.Contains(cr.Command[2], " k ") {
+ cr := cr
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index c2f6361334..e3c1432660 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -26,6 +26,7 @@ import (
"regexp"
"strconv"
"strings"
+ "sync"
"sync/atomic"
"time"
@@ -88,7 +89,10 @@ type Client struct {
// differs from an outgoing connection limit (a feature
// provided by http.Transport) when concurrent calls are
// multiplexed on a single http2 connection.
- requestLimiter requestLimiter
+ //
+ // getRequestLimiter() should always be used, because this can
+ // be nil.
+ requestLimiter *requestLimiter
last503 atomic.Value
}
@@ -150,7 +154,7 @@ func NewClientFromConfig(cluster *Cluster) (*Client, error) {
APIHost: ctrlURL.Host,
Insecure: cluster.TLS.Insecure,
Timeout: 5 * time.Minute,
- requestLimiter: requestLimiter{maxlimit: int64(cluster.API.MaxConcurrentRequests / 4)},
+ requestLimiter: &requestLimiter{maxlimit: int64(cluster.API.MaxConcurrentRequests / 4)},
}, nil
}
@@ -291,7 +295,7 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
}
rclient.CheckRetry = func(ctx context.Context, resp *http.Response, respErr error) (bool, error) {
checkRetryCalled++
- if c.requestLimiter.Report(resp, respErr) {
+ if c.getRequestLimiter().Report(resp, respErr) {
c.last503.Store(time.Now())
}
if c.Timeout == 0 {
@@ -321,9 +325,10 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
}
rclient.Logger = nil
- c.requestLimiter.Acquire(ctx)
+ limiter := c.getRequestLimiter()
+ limiter.Acquire(ctx)
if ctx.Err() != nil {
- c.requestLimiter.Release()
+ limiter.Release()
cancel()
return nil, ctx.Err()
}
@@ -342,7 +347,7 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
}
}
if err != nil {
- c.requestLimiter.Release()
+ limiter.Release()
cancel()
return nil, err
}
@@ -352,7 +357,7 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
resp.Body = cancelOnClose{
ReadCloser: resp.Body,
cancel: func() {
- c.requestLimiter.Release()
+ limiter.Release()
cancel()
},
}
@@ -366,6 +371,30 @@ func (c *Client) Last503() time.Time {
return t
}
+// globalRequestLimiter entries (one for each APIHost) don't have a
+// hard limit on outgoing connections, but do add a delay and reduce
+// concurrency after 503 errors.
+var (
+ globalRequestLimiter = map[string]*requestLimiter{}
+ globalRequestLimiterLock sync.Mutex
+)
+
+// Get this client's requestLimiter, or a global requestLimiter
+// singleton for c's APIHost if this client doesn't have its own.
+func (c *Client) getRequestLimiter() *requestLimiter {
+ if c.requestLimiter != nil {
+ return c.requestLimiter
+ }
+ globalRequestLimiterLock.Lock()
+ defer globalRequestLimiterLock.Unlock()
+ limiter := globalRequestLimiter[c.APIHost]
+ if limiter == nil {
+ limiter = &requestLimiter{}
+ globalRequestLimiter[c.APIHost] = limiter
+ }
+ return limiter
+}
+
// cancelOnClose calls a provided CancelFunc when its wrapped
// ReadCloser's Close() method is called.
type cancelOnClose struct {
diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 68ac886ddd..86001c01e0 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -120,6 +120,27 @@ type KeepClient struct {
disableDiscovery bool
}
+func (kc *KeepClient) Clone() *KeepClient {
+ kc.lock.Lock()
+ defer kc.lock.Unlock()
+ return &KeepClient{
+ Arvados: kc.Arvados,
+ Want_replicas: kc.Want_replicas,
+ localRoots: kc.localRoots,
+ writableLocalRoots: kc.writableLocalRoots,
+ gatewayRoots: kc.gatewayRoots,
+ HTTPClient: kc.HTTPClient,
+ Retries: kc.Retries,
+ BlockCache: kc.BlockCache,
+ RequestID: kc.RequestID,
+ StorageClasses: kc.StorageClasses,
+ DefaultStorageClasses: kc.DefaultStorageClasses,
+ replicasPerService: kc.replicasPerService,
+ foundNonDiskSvc: kc.foundNonDiskSvc,
+ disableDiscovery: kc.disableDiscovery,
+ }
+}
+
func (kc *KeepClient) loadDefaultClasses() error {
scData, err := kc.Arvados.ClusterConfig("StorageClasses")
if err != nil {
diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index b20144e3af..480791ffa2 100644
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -98,9 +98,7 @@ func (srv *Server) runForever(ctx context.Context) error {
ticker := time.NewTicker(time.Duration(srv.Cluster.Collections.BalancePeriod))
- // The unbuffered channel here means we only hear SIGUSR1 if
- // it arrives while we're waiting in select{}.
- sigUSR1 := make(chan os.Signal)
+ sigUSR1 := make(chan os.Signal, 1)
signal.Notify(sigUSR1, syscall.SIGUSR1)
logger.Info("acquiring service lock")
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 2090c50686..a79883147b 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -175,13 +175,18 @@ func (h *proxyHandler) checkAuthorizationHeader(req *http.Request) (pass bool, t
return true, tok, user
}
-// We need to make a private copy of the default http transport early
-// in initialization, then make copies of our private copy later. It
-// won't be safe to copy http.DefaultTransport itself later, because
-// its private mutexes might have already been used. (Without this,
-// the test suite sometimes panics "concurrent map writes" in
-// net/http.(*Transport).removeIdleConnLocked().)
-var defaultTransport = *(http.DefaultTransport.(*http.Transport))
+// We can't copy the default http transport because http.Transport has
+// a mutex field, so we make our own using the values of the exported
+// fields.
+var defaultTransport = http.Transport{
+ Proxy: http.DefaultTransport.(*http.Transport).Proxy,
+ DialContext: http.DefaultTransport.(*http.Transport).DialContext,
+ ForceAttemptHTTP2: http.DefaultTransport.(*http.Transport).ForceAttemptHTTP2,
+ MaxIdleConns: http.DefaultTransport.(*http.Transport).MaxIdleConns,
+ IdleConnTimeout: http.DefaultTransport.(*http.Transport).IdleConnTimeout,
+ TLSHandshakeTimeout: http.DefaultTransport.(*http.Transport).TLSHandshakeTimeout,
+ ExpectContinueTimeout: http.DefaultTransport.(*http.Transport).ExpectContinueTimeout,
+}
type proxyHandler struct {
http.Handler
@@ -195,14 +200,23 @@ type proxyHandler struct {
func newHandler(ctx context.Context, kc *keepclient.KeepClient, timeout time.Duration, cluster *arvados.Cluster) (service.Handler, error) {
rest := mux.NewRouter()
- transport := defaultTransport
- transport.DialContext = (&net.Dialer{
- Timeout: keepclient.DefaultConnectTimeout,
- KeepAlive: keepclient.DefaultKeepAlive,
- DualStack: true,
- }).DialContext
- transport.TLSClientConfig = arvadosclient.MakeTLSConfig(kc.Arvados.ApiInsecure)
- transport.TLSHandshakeTimeout = keepclient.DefaultTLSHandshakeTimeout
+ // We can't copy the default http transport because
+ // http.Transport has a mutex field, so we copy the fields
+ // that we know have non-zero values in http.DefaultTransport.
+ transport := &http.Transport{
+ Proxy: http.DefaultTransport.(*http.Transport).Proxy,
+ ForceAttemptHTTP2: http.DefaultTransport.(*http.Transport).ForceAttemptHTTP2,
+ MaxIdleConns: http.DefaultTransport.(*http.Transport).MaxIdleConns,
+ IdleConnTimeout: http.DefaultTransport.(*http.Transport).IdleConnTimeout,
+ ExpectContinueTimeout: http.DefaultTransport.(*http.Transport).ExpectContinueTimeout,
+ DialContext: (&net.Dialer{
+ Timeout: keepclient.DefaultConnectTimeout,
+ KeepAlive: keepclient.DefaultKeepAlive,
+ DualStack: true,
+ }).DialContext,
+ TLSClientConfig: arvadosclient.MakeTLSConfig(kc.Arvados.ApiInsecure),
+ TLSHandshakeTimeout: keepclient.DefaultTLSHandshakeTimeout,
+ }
cacheQ, err := lru.New2Q(500)
if err != nil {
@@ -213,7 +227,7 @@ func newHandler(ctx context.Context, kc *keepclient.KeepClient, timeout time.Dur
Handler: rest,
KeepClient: kc,
timeout: timeout,
- transport: &transport,
+ transport: transport,
apiTokenCache: &apiTokenCache{
tokens: cacheQ,
expireTime: 300,
@@ -566,7 +580,7 @@ func (h *proxyHandler) Index(resp http.ResponseWriter, req *http.Request) {
}
func (h *proxyHandler) makeKeepClient(req *http.Request) *keepclient.KeepClient {
- kc := *h.KeepClient
+ kc := h.KeepClient.Clone()
kc.RequestID = req.Header.Get("X-Request-Id")
kc.HTTPClient = &proxyClient{
client: &http.Client{
@@ -575,5 +589,5 @@ func (h *proxyHandler) makeKeepClient(req *http.Request) *keepclient.KeepClient
},
proto: req.Proto,
}
- return &kc
+ return kc
}
diff --git a/services/keepstore/proxy_remote.go b/services/keepstore/proxy_remote.go
index 526bc25299..66a7b43751 100644
--- a/services/keepstore/proxy_remote.go
+++ b/services/keepstore/proxy_remote.go
@@ -130,14 +130,14 @@ func (rp *remoteProxy) remoteClient(remoteID string, remoteCluster arvados.Remot
}
accopy := *kc.Arvados
accopy.ApiToken = token
- kccopy := *kc
+ kccopy := kc.Clone()
kccopy.Arvados = &accopy
token, err := auth.SaltToken(token, remoteID)
if err != nil {
return nil, err
}
kccopy.Arvados.ApiToken = token
- return &kccopy, nil
+ return kccopy, nil
}
var localOrRemoteSignature = regexp.MustCompile(`\+[AR][^\+]*`)
diff --git a/services/keepstore/pull_worker.go b/services/keepstore/pull_worker.go
index abe3dc3857..b9194fe6f6 100644
--- a/services/keepstore/pull_worker.go
+++ b/services/keepstore/pull_worker.go
@@ -50,7 +50,7 @@ func (h *handler) pullItemAndProcess(pullRequest PullRequest) error {
// Make a private copy of keepClient so we can set
// ServiceRoots to the source servers specified in the pull
// request.
- keepClient := *h.keepClient
+ keepClient := h.keepClient.Clone()
serviceRoots := make(map[string]string)
for _, addr := range pullRequest.Servers {
serviceRoots[addr] = addr
@@ -59,7 +59,7 @@ func (h *handler) pullItemAndProcess(pullRequest PullRequest) error {
signedLocator := SignLocator(h.Cluster, pullRequest.Locator, keepClient.Arvados.ApiToken, time.Now().Add(time.Minute))
- reader, contentLen, _, err := GetContent(signedLocator, &keepClient)
+ reader, contentLen, _, err := GetContent(signedLocator, keepClient)
if err != nil {
return err
}
commit 318684addb994eae446bdc9907bd5d44c8f58e2a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Dec 6 09:31:40 2023 -0500
Merge branch '21223-arv-mount-nofile' refs #21223
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index c2242eb2bf..9c607c7f0c 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -134,21 +134,40 @@ class Mount(object):
if self.args.logfile:
self.args.logfile = os.path.realpath(self.args.logfile)
+ try:
+ self._setup_logging()
+ except Exception as e:
+ self.logger.exception("exception during setup: %s", e)
+ exit(1)
+
try:
nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
- if nofile_limit[0] < 10240:
- resource.setrlimit(resource.RLIMIT_NOFILE, (min(10240, nofile_limit[1]), nofile_limit[1]))
+
+ minlimit = 10240
+ if self.args.file_cache:
+ # Adjust the file handle limit so it can meet
+ # the desired cache size. Multiply by 8 because the
+ # number of 64 MiB cache slots that keepclient
+ # allocates is RLIMIT_NOFILE / 8
+ minlimit = int((self.args.file_cache/(64*1024*1024)) * 8)
+
+ if nofile_limit[0] < minlimit:
+ resource.setrlimit(resource.RLIMIT_NOFILE, (min(minlimit, nofile_limit[1]), nofile_limit[1]))
+
+ if minlimit > nofile_limit[1]:
+ self.logger.warning("file handles required to meet --file-cache (%s) exceeds hard file handle limit (%s), cache size will be smaller than requested", minlimit, nofile_limit[1])
+
except Exception as e:
- self.logger.warning("arv-mount: unable to adjust file handle limit: %s", e)
+ self.logger.warning("unable to adjust file handle limit: %s", e)
- self.logger.debug("arv-mount: file handle limit is %s", resource.getrlimit(resource.RLIMIT_NOFILE))
+ nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+ self.logger.info("file cache capped at %s bytes or less based on available disk (RLIMIT_NOFILE is %s)", ((nofile_limit[0]//8)*64*1024*1024), nofile_limit)
try:
- self._setup_logging()
self._setup_api()
self._setup_mount()
except Exception as e:
- self.logger.exception("arv-mount: exception during setup: %s", e)
+ self.logger.exception("exception during setup: %s", e)
exit(1)
def __enter__(self):
diff --git a/services/fuse/tests/test_command_args.py b/services/fuse/tests/test_command_args.py
index 600bb0fe22..b08ab19335 100644
--- a/services/fuse/tests/test_command_args.py
+++ b/services/fuse/tests/test_command_args.py
@@ -20,6 +20,7 @@ from . import run_test_server
import sys
import tempfile
import unittest
+import resource
def noexit(func):
"""If argparse or arvados_fuse tries to exit, fail the test instead"""
@@ -261,6 +262,50 @@ class MountArgsTest(unittest.TestCase):
'--foreground', self.mntdir])
arvados_fuse.command.Mount(args)
+ @noexit
+ @mock.patch('resource.setrlimit')
+ @mock.patch('resource.getrlimit')
+ def test_default_file_cache(self, getrlimit, setrlimit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, None)
+ getrlimit.return_value = (1024, 1048576)
+ self.mnt = arvados_fuse.command.Mount(args)
+ setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (10240, 1048576))
+
+ @noexit
+ @mock.patch('resource.setrlimit')
+ @mock.patch('resource.getrlimit')
+ def test_small_file_cache(self, getrlimit, setrlimit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', '--file-cache=256000000', self.mntdir])
+ self.assertEqual(args.mode, None)
+ getrlimit.return_value = (1024, 1048576)
+ self.mnt = arvados_fuse.command.Mount(args)
+ setrlimit.assert_not_called()
+
+ @noexit
+ @mock.patch('resource.setrlimit')
+ @mock.patch('resource.getrlimit')
+ def test_large_file_cache(self, getrlimit, setrlimit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', '--file-cache=256000000000', self.mntdir])
+ self.assertEqual(args.mode, None)
+ getrlimit.return_value = (1024, 1048576)
+ self.mnt = arvados_fuse.command.Mount(args)
+ setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (30517, 1048576))
+
+ @noexit
+ @mock.patch('resource.setrlimit')
+ @mock.patch('resource.getrlimit')
+ def test_file_cache_hard_limit(self, getrlimit, setrlimit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', '--file-cache=256000000000', self.mntdir])
+ self.assertEqual(args.mode, None)
+ getrlimit.return_value = (1024, 2048)
+ self.mnt = arvados_fuse.command.Mount(args)
+ setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (2048, 2048))
+
class MountErrorTest(unittest.TestCase):
def setUp(self):
self.mntdir = tempfile.mkdtemp()
commit 29e6998044b508d5a221c06ab6b784779cd299c6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Dec 5 16:37:14 2023 -0500
Merge branch '21160-user-activation' refs #21160
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/api/app/models/link.rb b/services/api/app/models/link.rb
index 4d4c2832bb..2eb6b88a0c 100644
--- a/services/api/app/models/link.rb
+++ b/services/api/app/models/link.rb
@@ -17,11 +17,12 @@ class Link < ArvadosModel
before_update :apply_max_overlapping_permissions
before_create :apply_max_overlapping_permissions
after_update :delete_overlapping_permissions
- after_update :call_update_permissions
- after_create :call_update_permissions
+ after_update :call_update_permissions, :if => Proc.new { @need_update_permissions }
+ after_create :call_update_permissions, :if => Proc.new { @need_update_permissions }
before_destroy :clear_permissions
after_destroy :delete_overlapping_permissions
after_destroy :check_permissions
+ before_save :check_need_update_permissions
api_accessible :user, extend: :common do |t|
t.add :tail_uuid
@@ -189,11 +190,13 @@ class Link < ArvadosModel
'can_manage' => 3,
}
+ def check_need_update_permissions
+ @need_update_permissions = self.link_class == 'permission' && (name != name_was || new_record?)
+ end
+
def call_update_permissions
- if self.link_class == 'permission'
update_permissions tail_uuid, head_uuid, PERM_LEVEL[name], self.uuid
current_user.forget_cached_group_perms
- end
end
def clear_permissions
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index d4d4a82458..c73f31a99d 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -512,11 +512,11 @@ SELECT target_uuid, perm_level
update_attributes!(redirect_to_user_uuid: new_user.uuid, username: nil)
end
skip_check_permissions_against_full_refresh do
- update_permissions self.uuid, self.uuid, CAN_MANAGE_PERM
- update_permissions new_user.uuid, new_user.uuid, CAN_MANAGE_PERM
- update_permissions new_user.owner_uuid, new_user.uuid, CAN_MANAGE_PERM
+ update_permissions self.uuid, self.uuid, CAN_MANAGE_PERM, nil, true
+ update_permissions new_user.uuid, new_user.uuid, CAN_MANAGE_PERM, nil, true
+ update_permissions new_user.owner_uuid, new_user.uuid, CAN_MANAGE_PERM, nil, true
end
- update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
+ update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM, nil, true
end
end
diff --git a/services/api/lib/update_permissions.rb b/services/api/lib/update_permissions.rb
index 5c8072b48a..7e946b4311 100644
--- a/services/api/lib/update_permissions.rb
+++ b/services/api/lib/update_permissions.rb
@@ -7,7 +7,7 @@ require '20200501150153_permission_table_constants'
REVOKE_PERM = 0
CAN_MANAGE_PERM = 3
-def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
+def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil, update_all_users=false
return if Thread.current[:suppress_update_permissions]
#
@@ -109,6 +109,70 @@ def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
# Disable merge join for just this query (also local for this transaction), then reenable it.
ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to false;"
+ if perm_origin_uuid[5..11] == '-tpzed-' && !update_all_users
+ # Modifying permission granted to a user, recompute the all permissions for that user
+
+ ActiveRecord::Base.connection.exec_query %{
+with origin_user_perms as (
+ select pq.origin_uuid as user_uuid, target_uuid, pq.val, pq.traverse_owned from (
+ #{PERM_QUERY_TEMPLATE % {:base_case => %{
+ select '#{perm_origin_uuid}'::varchar(255), '#{perm_origin_uuid}'::varchar(255), 3, true, true
+ where exists (select uuid from users where uuid='#{perm_origin_uuid}')
+},
+:edge_perm => %{
+case (edges.edge_id = '#{edge_id}')
+ when true then #{perm_level}
+ else edges.val
+ end
+}
+} }) as pq),
+
+/*
+ Because users always have permission on themselves, this
+ query also makes sure those permission rows are always
+ returned.
+*/
+temptable_perms as (
+ select * from origin_user_perms
+ union all
+ select target_uuid as user_uuid, target_uuid, 3, true
+ from origin_user_perms
+ where origin_user_perms.target_uuid like '_____-tpzed-_______________' and
+ origin_user_perms.target_uuid != '#{perm_origin_uuid}'
+),
+
+/*
+ Now that we have recomputed a set of permissions, delete any
+ rows from the materialized_permissions table where (target_uuid,
+ user_uuid) is not present or has perm_level=0 in the recomputed
+ set.
+*/
+delete_rows as (
+ delete from #{PERMISSION_VIEW} where
+ user_uuid='#{perm_origin_uuid}' and
+ not exists (select 1 from temptable_perms
+ where target_uuid=#{PERMISSION_VIEW}.target_uuid and
+ user_uuid='#{perm_origin_uuid}' and
+ val>0)
+)
+
+/*
+ Now insert-or-update permissions in the recomputed set. The
+ WHERE clause is important to avoid redundantly updating rows
+ that haven't actually changed.
+*/
+insert into #{PERMISSION_VIEW} (user_uuid, target_uuid, perm_level, traverse_owned)
+ select user_uuid, target_uuid, val as perm_level, traverse_owned from temptable_perms where val>0
+on conflict (user_uuid, target_uuid) do update
+set perm_level=EXCLUDED.perm_level, traverse_owned=EXCLUDED.traverse_owned
+where #{PERMISSION_VIEW}.user_uuid=EXCLUDED.user_uuid and
+ #{PERMISSION_VIEW}.target_uuid=EXCLUDED.target_uuid and
+ (#{PERMISSION_VIEW}.perm_level != EXCLUDED.perm_level or
+ #{PERMISSION_VIEW}.traverse_owned != EXCLUDED.traverse_owned);
+
+}
+ else
+ # Modifying permission granted to a group, recompute permissions for everything accessible through that group
ActiveRecord::Base.connection.exec_query %{
with temptable_perms as (
select * from compute_permission_subgraph($1, $2, $3, $4)),
@@ -147,6 +211,7 @@ where #{PERMISSION_VIEW}.user_uuid=EXCLUDED.user_uuid and
[nil, starting_uuid],
[nil, perm_level],
[nil, edge_id]]
+ end
if perm_level>0
check_permissions_against_full_refresh
commit d53b38d1fa91725b10c7359278e52a7f2f6a15db
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Tue Dec 5 12:12:18 2023 -0300
Merge branch '21262-dependency-upgrades'. Closes #21262
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/go.mod b/go.mod
index d7bb5768eb..218e2ddde8 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
github.com/coreos/go-oidc/v3 v3.5.0
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e
github.com/creack/pty v1.1.18
- github.com/docker/docker v24.0.5+incompatible
+ github.com/docker/docker v24.0.7+incompatible
github.com/dustin/go-humanize v1.0.0
github.com/fsnotify/fsnotify v1.4.9
github.com/ghodss/yaml v1.0.0
@@ -37,11 +37,11 @@ require (
github.com/prometheus/client_model v0.3.0
github.com/prometheus/common v0.39.0
github.com/sirupsen/logrus v1.8.1
- golang.org/x/crypto v0.5.0
- golang.org/x/net v0.9.0
- golang.org/x/oauth2 v0.7.0
- golang.org/x/sys v0.7.0
- google.golang.org/api v0.114.0
+ golang.org/x/crypto v0.16.0
+ golang.org/x/net v0.19.0
+ golang.org/x/oauth2 v0.11.0
+ golang.org/x/sys v0.15.0
+ google.golang.org/api v0.126.0
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
gopkg.in/square/go-jose.v2 v2.5.1
gopkg.in/src-d/go-billy.v4 v4.0.1
@@ -50,7 +50,7 @@ require (
)
require (
- cloud.google.com/go/compute v1.19.1 // indirect
+ cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
@@ -74,13 +74,14 @@ require (
github.com/docker/go-units v0.4.0 // indirect
github.com/gliderlabs/ssh v0.2.2 // indirect
github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
- github.com/go-jose/go-jose/v3 v3.0.0 // indirect
+ github.com/go-jose/go-jose/v3 v3.0.1 // indirect
github.com/golang-jwt/jwt/v4 v4.1.0 // indirect
- github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
- github.com/google/uuid v1.3.0 // indirect
+ github.com/google/s2a-go v0.1.4 // indirect
+ github.com/google/uuid v1.3.1 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
- github.com/googleapis/gax-go/v2 v2.7.1 // indirect
+ github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
@@ -103,13 +104,13 @@ require (
github.com/src-d/gcfg v1.3.0 // indirect
github.com/xanzy/ssh-agent v0.1.0 // indirect
go.opencensus.io v0.24.0 // indirect
- golang.org/x/text v0.9.0 // indirect
+ golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
golang.org/x/tools v0.6.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
- google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
- google.golang.org/grpc v1.56.1 // indirect
- google.golang.org/protobuf v1.30.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
+ google.golang.org/grpc v1.59.0 // indirect
+ google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
diff --git a/go.sum b/go.sum
index 7f71b206d2..31ddd88621 100644
--- a/go.sum
+++ b/go.sum
@@ -1,11 +1,10 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys=
-cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY=
-cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY=
+cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
cloud.google.com/go/compute/metadata v0.2.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
github.com/Azure/azure-sdk-for-go v45.1.0+incompatible h1:kxtaPD8n2z5Za+9e3sKsYG2IX6PG2R6VXtgS7gAbh3A=
github.com/Azure/azure-sdk-for-go v45.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
@@ -44,6 +43,7 @@ github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBb
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/arvados/cgofuse v1.2.0-arvados1 h1:4Q4vRJ4hbTCcI4gGEaa6hqwj3rqlUuzeFQkfoEA2HqE=
github.com/arvados/cgofuse v1.2.0-arvados1/go.mod h1:79WFV98hrkRHK9XPhh2IGGOwpFSjocsWubgxAs2KhRc=
github.com/arvados/goamz v0.0.0-20190905141525-1bba09f407ef h1:cl7DIRbiAYNqaVxg3CZY8qfZoBOKrj06H/x9SPGaxas=
@@ -63,10 +63,16 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2
github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092 h1:0Di2onNnlN5PAyWPbqlPyN45eOQ+QW/J9eqLynt4IV4=
github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092/go.mod h1:8IzBjZCRSnsvM6MJMG8HNNtnzMl48H22rbJL2kRUJ0Y=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI=
+github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/coreos/go-oidc/v3 v3.5.0 h1:VxKtbccHZxs8juq7RdJntSqtXFtde9YpNpGn0yqgEHw=
github.com/coreos/go-oidc/v3 v3.5.0/go.mod h1:ecXRtV4romGPeO6ieExAsUK9cb/3fp9hXNz1tlv8PIM=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8=
@@ -83,8 +89,8 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.5+incompatible h1:WmgcE4fxyI6EEXxBRxsHnZXrO1pQ3smi0k/jho4HLeY=
-github.com/docker/docker v24.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o=
github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
@@ -94,6 +100,8 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -104,8 +112,9 @@ github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw=
github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
-github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
+github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
github.com/go-ldap/ldap v3.0.3+incompatible h1:HTeSZO8hWMS1Rgb2Ziku6b8a7qRIZZMHjsvuZyatzwk=
github.com/go-ldap/ldap v3.0.3+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
@@ -117,12 +126,14 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw
github.com/golang-jwt/jwt/v4 v4.1.0 h1:XUgk2Ex5veyVFVeLm0xhusUTQybEbexJXrvPNOKkSY0=
github.com/golang-jwt/jwt/v4 v4.1.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
@@ -130,6 +141,7 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@@ -144,17 +156,20 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc=
+github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k=
github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k=
-github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A=
-github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI=
+github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4=
+github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=
@@ -228,6 +243,7 @@ github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8u
github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y=
github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
github.com/satori/go.uuid v1.2.1-0.20180103174451-36e9d2ebbde5 h1:Jw7W4WMfQDxsXvfeFSaS2cHlY7bAF4MGrgnbd0+Uo78=
@@ -249,6 +265,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -262,14 +279,16 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
-golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
+golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -279,6 +298,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190310074541-c10a0554eabf/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -287,19 +307,22 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
-golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM=
-golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk=
-golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
-golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
+golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU=
+golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -307,36 +330,41 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
-golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
-golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ=
+golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -356,24 +384,30 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE=
-google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg=
+google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o=
+google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A=
-google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU=
+google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY=
+google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ=
-google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
+google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk=
+google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -386,8 +420,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
commit 8674a32277bc11484d7aa9b16649869256bbdc17
Author: Tom Clegg <tom at curii.com>
Date: Mon Dec 4 14:35:36 2023 -0500
Merge branch '21217-invalid-auth-header'
refs #21217
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 735a44d24c..c2f6361334 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -238,6 +238,8 @@ var reqIDGen = httpserver.IDGenerator{Prefix: "req-"}
var nopCancelFunc context.CancelFunc = func() {}
+var reqErrorRe = regexp.MustCompile(`net/http: invalid header `)
+
// Do augments (*http.Client)Do(): adds Authorization and X-Request-Id
// headers, delays in order to comply with rate-limiting restrictions,
// and retries failed requests when appropriate.
@@ -274,6 +276,7 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
var lastResp *http.Response
var lastRespBody io.ReadCloser
var lastErr error
+ var checkRetryCalled int
rclient := retryablehttp.NewClient()
rclient.HTTPClient = c.httpClient()
@@ -287,11 +290,15 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
rclient.RetryMax = 0
}
rclient.CheckRetry = func(ctx context.Context, resp *http.Response, respErr error) (bool, error) {
+ checkRetryCalled++
if c.requestLimiter.Report(resp, respErr) {
c.last503.Store(time.Now())
}
if c.Timeout == 0 {
- return false, err
+ return false, nil
+ }
+ if respErr != nil && reqErrorRe.MatchString(respErr.Error()) {
+ return false, nil
}
retrying, err := retryablehttp.DefaultRetryPolicy(ctx, resp, respErr)
if retrying {
@@ -322,7 +329,14 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
}
resp, err := rclient.Do(rreq)
if (errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) && (lastResp != nil || lastErr != nil) {
- resp, err = lastResp, lastErr
+ resp = lastResp
+ err = lastErr
+ if checkRetryCalled > 0 && err != nil {
+ // Mimic retryablehttp's "giving up after X
+ // attempts" message, even if we gave up
+ // because of time rather than maxretries.
+ err = fmt.Errorf("%s %s giving up after %d attempt(s): %w", req.Method, req.URL.String(), checkRetryCalled, err)
+ }
if resp != nil {
resp.Body = lastRespBody
}
@@ -619,7 +633,11 @@ func (c *Client) apiURL(path string) string {
if scheme == "" {
scheme = "https"
}
- return scheme + "://" + c.APIHost + "/" + path
+ // Double-slash in URLs tend to cause subtle hidden problems
+ // (e.g., they can behave differently when a load balancer is
+ // in the picture). Here we ensure exactly one "/" regardless
+ // of whether the given APIHost or path has a superfluous one.
+ return scheme + "://" + strings.TrimSuffix(c.APIHost, "/") + "/" + strings.TrimPrefix(path, "/")
}
// DiscoveryDocument is the Arvados server's description of itself.
diff --git a/sdk/go/arvados/client_test.go b/sdk/go/arvados/client_test.go
index a196003b8f..55e2f998c4 100644
--- a/sdk/go/arvados/client_test.go
+++ b/sdk/go/arvados/client_test.go
@@ -341,6 +341,20 @@ func (s *clientRetrySuite) TestNonRetryableError(c *check.C) {
c.Check(s.reqs, check.HasLen, 1)
}
+// as of 0.7.2., retryablehttp does not recognize this as a
+// non-retryable error.
+func (s *clientRetrySuite) TestNonRetryableStdlibError(c *check.C) {
+ s.respStatus <- http.StatusOK
+ req, err := http.NewRequest(http.MethodGet, "https://"+s.client.APIHost+"/test", nil)
+ c.Assert(err, check.IsNil)
+ req.Header.Set("Good-Header", "T\033rrible header value")
+ err = s.client.DoAndDecode(&struct{}{}, req)
+ c.Check(err, check.ErrorMatches, `.*after 1 attempt.*net/http: invalid header .*`)
+ if !c.Check(s.reqs, check.HasLen, 0) {
+ c.Logf("%v", s.reqs[0])
+ }
+}
+
func (s *clientRetrySuite) TestNonRetryableAfter503s(c *check.C) {
time.AfterFunc(time.Second, func() { s.respStatus <- http.StatusNotFound })
err := s.client.RequestAndDecode(&struct{}{}, http.MethodGet, "test", nil, nil)
diff --git a/sdk/go/auth/auth.go b/sdk/go/auth/auth.go
index f1c2e243b5..da9b4ea5b8 100644
--- a/sdk/go/auth/auth.go
+++ b/sdk/go/auth/auth.go
@@ -54,13 +54,13 @@ func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
// Load plain token from "Authorization: OAuth2 ..." header
// (typically used by smart API clients)
if toks := strings.SplitN(r.Header.Get("Authorization"), " ", 2); len(toks) == 2 && (toks[0] == "OAuth2" || toks[0] == "Bearer") {
- a.Tokens = append(a.Tokens, toks[1])
+ a.Tokens = append(a.Tokens, strings.TrimSpace(toks[1]))
}
// Load base64-encoded token from "Authorization: Basic ..."
// header (typically used by git via credential helper)
if _, password, ok := r.BasicAuth(); ok {
- a.Tokens = append(a.Tokens, password)
+ a.Tokens = append(a.Tokens, strings.TrimSpace(password))
}
// Load tokens from query string. It's generally not a good
@@ -76,7 +76,9 @@ func (a *Credentials) LoadTokensFromHTTPRequest(r *http.Request) {
// find/report decoding errors in a suitable way.
qvalues, _ := url.ParseQuery(r.URL.RawQuery)
if val, ok := qvalues["api_token"]; ok {
- a.Tokens = append(a.Tokens, val...)
+ for _, token := range val {
+ a.Tokens = append(a.Tokens, strings.TrimSpace(token))
+ }
}
a.loadTokenFromCookie(r)
@@ -94,7 +96,7 @@ func (a *Credentials) loadTokenFromCookie(r *http.Request) {
if err != nil {
return
}
- a.Tokens = append(a.Tokens, string(token))
+ a.Tokens = append(a.Tokens, strings.TrimSpace(string(token)))
}
// LoadTokensFromHTTPRequestBody loads credentials from the request
@@ -111,7 +113,7 @@ func (a *Credentials) LoadTokensFromHTTPRequestBody(r *http.Request) error {
return err
}
if t := r.PostFormValue("api_token"); t != "" {
- a.Tokens = append(a.Tokens, t)
+ a.Tokens = append(a.Tokens, strings.TrimSpace(t))
}
return nil
}
diff --git a/sdk/go/auth/handlers_test.go b/sdk/go/auth/handlers_test.go
index 362aeb7f04..85ea8893a5 100644
--- a/sdk/go/auth/handlers_test.go
+++ b/sdk/go/auth/handlers_test.go
@@ -7,6 +7,8 @@ package auth
import (
"net/http"
"net/http/httptest"
+ "net/url"
+ "strings"
"testing"
check "gopkg.in/check.v1"
@@ -32,9 +34,36 @@ func (s *HandlersSuite) SetUpTest(c *check.C) {
func (s *HandlersSuite) TestLoadToken(c *check.C) {
handler := LoadToken(s)
handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/foo/bar?api_token=xyzzy", nil))
- c.Assert(s.gotCredentials, check.NotNil)
- c.Assert(s.gotCredentials.Tokens, check.HasLen, 1)
- c.Check(s.gotCredentials.Tokens[0], check.Equals, "xyzzy")
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
+}
+
+// Ignore leading and trailing spaces, newlines, etc. in case a user
+// has added them inadvertently during copy/paste.
+func (s *HandlersSuite) TestTrimSpaceInQuery(c *check.C) {
+ handler := LoadToken(s)
+ handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "/foo/bar?api_token=%20xyzzy%0a", nil))
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
+}
+func (s *HandlersSuite) TestTrimSpaceInPostForm(c *check.C) {
+ handler := LoadToken(s)
+ req := httptest.NewRequest("POST", "/foo/bar", strings.NewReader(url.Values{"api_token": []string{"\nxyzzy\n"}}.Encode()))
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ handler.ServeHTTP(httptest.NewRecorder(), req)
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
+}
+func (s *HandlersSuite) TestTrimSpaceInCookie(c *check.C) {
+ handler := LoadToken(s)
+ req := httptest.NewRequest("GET", "/foo/bar", nil)
+ req.AddCookie(&http.Cookie{Name: "arvados_api_token", Value: EncodeTokenCookie([]byte("\vxyzzy\n"))})
+ handler.ServeHTTP(httptest.NewRecorder(), req)
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
+}
+func (s *HandlersSuite) TestTrimSpaceInBasicAuth(c *check.C) {
+ handler := LoadToken(s)
+ req := httptest.NewRequest("GET", "/foo/bar", nil)
+ req.SetBasicAuth("username", "\txyzzy\n")
+ handler.ServeHTTP(httptest.NewRecorder(), req)
+ c.Check(s.gotCredentials.Tokens, check.DeepEquals, []string{"xyzzy"})
}
func (s *HandlersSuite) TestRequireLiteralTokenEmpty(c *check.C) {
@@ -76,4 +105,5 @@ func (s *HandlersSuite) TestRequireLiteralToken(c *check.C) {
func (s *HandlersSuite) ServeHTTP(w http.ResponseWriter, r *http.Request) {
s.served++
s.gotCredentials = CredentialsFromRequest(r)
+ s.gotCredentials.LoadTokensFromHTTPRequestBody(r)
}
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 604efd29d9..df5705ed32 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -221,7 +221,7 @@ func (c *cache) GetSession(token string) (arvados.CustomFileSystem, *cachedSessi
// using the new fs).
sess.inuse.Lock()
if !sess.userLoaded || refresh {
- err := sess.client.RequestAndDecode(&sess.user, "GET", "/arvados/v1/users/current", nil, nil)
+ err := sess.client.RequestAndDecode(&sess.user, "GET", "arvados/v1/users/current", nil, nil)
if he := errorWithHTTPStatus(nil); errors.As(err, &he) && he.HTTPStatus() == http.StatusForbidden {
// token is OK, but "get user id" api is out
// of scope -- use existing/expired info if
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 5e81f3a01e..c14789889d 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -328,9 +328,10 @@ func (s *IntegrationSuite) TestVhostViaAuthzHeaderOAuth2(c *check.C) {
s.doVhostRequests(c, authzViaAuthzHeaderOAuth2)
}
func authzViaAuthzHeaderOAuth2(r *http.Request, tok string) int {
- r.Header.Add("Authorization", "Bearer "+tok)
+ r.Header.Add("Authorization", "OAuth2 "+tok)
return http.StatusUnauthorized
}
+
func (s *IntegrationSuite) TestVhostViaAuthzHeaderBearer(c *check.C) {
s.doVhostRequests(c, authzViaAuthzHeaderBearer)
}
@@ -350,6 +351,27 @@ func authzViaCookieValue(r *http.Request, tok string) int {
return http.StatusUnauthorized
}
+func (s *IntegrationSuite) TestVhostViaHTTPBasicAuth(c *check.C) {
+ s.doVhostRequests(c, authzViaHTTPBasicAuth)
+}
+func authzViaHTTPBasicAuth(r *http.Request, tok string) int {
+ r.AddCookie(&http.Cookie{
+ Name: "arvados_api_token",
+ Value: auth.EncodeTokenCookie([]byte(tok)),
+ })
+ return http.StatusUnauthorized
+}
+
+func (s *IntegrationSuite) TestVhostViaHTTPBasicAuthWithExtraSpaceChars(c *check.C) {
+ s.doVhostRequests(c, func(r *http.Request, tok string) int {
+ r.AddCookie(&http.Cookie{
+ Name: "arvados_api_token",
+ Value: auth.EncodeTokenCookie([]byte(" " + tok + "\n")),
+ })
+ return http.StatusUnauthorized
+ })
+}
+
func (s *IntegrationSuite) TestVhostViaPath(c *check.C) {
s.doVhostRequests(c, authzViaPath)
}
commit 39bed89acebb3cff63831f36b0915378480f4ae5
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Nov 29 15:49:27 2023 -0500
Merge branch '21205-ensure-unique' refs #21205
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index d910320ec0..79ab666ee1 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -24,6 +24,7 @@ class ArvadosModel < ApplicationRecord
before_destroy :ensure_owner_uuid_is_permitted
before_destroy :ensure_permission_to_destroy
before_create :update_modified_by_fields
+ before_create :add_uuid_to_name, :if => Proc.new { @_add_uuid_to_name }
before_update :maybe_update_modified_by_fields
after_create :log_create
after_update :log_update
@@ -470,8 +471,6 @@ class ArvadosModel < ApplicationRecord
end
def save_with_unique_name!
- uuid_was = uuid
- name_was = name
max_retries = 2
transaction do
conn = ActiveRecord::Base.connection
@@ -502,24 +501,20 @@ class ArvadosModel < ApplicationRecord
conn.exec_query 'ROLLBACK TO SAVEPOINT save_with_unique_name'
- new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
- if new_name == name
- # If the database is fast enough to do two attempts in the
- # same millisecond, we need to wait to ensure we try a
- # different timestamp on each attempt.
- sleep 0.002
- new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
- end
-
- self[:name] = new_name
- if uuid_was.nil? && !uuid.nil?
+ if uuid_was.nil?
+ # new record, the uuid caused a name collision (very
+ # unlikely but possible), so generate new uuid
self[:uuid] = nil
if self.is_a? Collection
- # Reset so that is assigned to the new UUID
+ # Also needs to be reset
self[:current_version_uuid] = nil
end
+ # need to adjust the name after the uuid has been generated
+ add_uuid_to_make_unique_name
+ else
+ # existing record, just update the name directly.
+ add_uuid_to_name
end
-
retry
end
end
@@ -580,6 +575,26 @@ class ArvadosModel < ApplicationRecord
*ft[:param_out])
end
+ @_add_uuid_to_name = false
+ def add_uuid_to_make_unique_name
+ @_add_uuid_to_name = true
+ end
+
+ def add_uuid_to_name
+ # Incorporate the random part of the UUID into the name. This
+ # lets us prevent name collision but the part we add to the name
+ # is still somewhat meaningful (instead of generating a second
+ # random meaningless string).
+ #
+ # Because ArvadosModel is an abstract class and assign_uuid is
+ # part of HasUuid (which is included by the other concrete
+ # classes) the assign_uuid hook gets added (and run) after this
+ # one. So we need to call assign_uuid here to make sure we have a
+ # uuid.
+ assign_uuid
+ self.name = "#{self.name[0..236]} (#{self.uuid[-15..-1]})"
+ end
+
protected
def self.deep_sort_hash(x)
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 8a1d044d6a..9e90ef6149 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -409,7 +409,7 @@ EOS
ensure_unique_name: true
}
assert_response :success
- assert_match /^owned_by_active \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^owned_by_active \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
end
@@ -1271,7 +1271,7 @@ EOS
assert_equal false, json_response['is_trashed']
assert_nil json_response['trash_at']
assert_nil json_response['delete_at']
- assert_match /^same name for trashed and persisted collections \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^same name for trashed and persisted collections \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
test 'cannot show collection in trashed subproject' do
diff --git a/services/api/test/functional/arvados/v1/groups_controller_test.rb b/services/api/test/functional/arvados/v1/groups_controller_test.rb
index a64ea76692..88d96c51a3 100644
--- a/services/api/test/functional/arvados/v1/groups_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/groups_controller_test.rb
@@ -474,7 +474,7 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
assert_not_equal(new_project['uuid'],
groups(:aproject).uuid,
"create returned same uuid as existing project")
- assert_match(/^A Project \(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z\)$/,
+ assert_match(/^A Project \(#{new_project['uuid'][-15..-1]}\)$/,
new_project['name'])
end
@@ -800,7 +800,7 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
ensure_unique_name: true
}
assert_response :success
- assert_match /^trashed subproject 3 \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^trashed subproject 3 \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
test "move trashed subproject to new owner #{auth}" do
diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb
index a64adba6ff..0dd27d5ff5 100644
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@ -1131,13 +1131,13 @@ class ContainerRequestTest < ActiveSupport::TestCase
assert_equal ContainerRequest::Final, cr.state
output_coll = Collection.find_by_uuid(cr.output_uuid)
# Make sure the resulting output collection name include the original name
- # plus the date
+ # plus the last 15 characters of uuid
assert_not_equal output_name, output_coll.name,
"more than one collection with the same owner and name"
assert output_coll.name.include?(output_name),
"New name should include original name"
- assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name,
- "New name should include ISO8601 date"
+ assert_match /#{output_coll.uuid[-15..-1]}/, output_coll.name,
+ "New name should include last 15 characters of uuid"
end
[[0, :check_output_ttl_0],
commit 7a8c6b58534518990d05d895cb33d3f74553755d
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Oct 11 10:25:41 2023 -0400
Close span
refs #20937
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index e7c250ac48..a05620d62d 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -98,7 +98,7 @@ We will use the uuid @jutro-j7d0g-xj19djofle3aryq@ as an example project.
Arv-copy will infer the source cluster is @jutro@ from the source project uuid, and destination cluster is @pirca@ from @--project-uuid at .
<notextile>
-<pre><code>~$ <span class="userinput">arv-copy --project-uuid pirca-j7d0g-lr8sq3tx3ovn68k jutro-j7d0g-xj19djofle3aryq
+<pre><code>~$ <span class="userinput">arv-copy --project-uuid pirca-j7d0g-lr8sq3tx3ovn68k jutro-j7d0g-xj19djofle3aryq</span>
2021-09-08 21:29:32 arvados.arv-copy[6377] INFO:
2021-09-08 21:29:32 arvados.arv-copy[6377] INFO: Success: created copy with uuid pirca-j7d0g-ig9gvu5piznducp
</code></pre>
@@ -113,7 +113,7 @@ h3. Importing HTTP resources to Keep
You can also use @arv-copy@ to copy the contents of a HTTP URL into Keep. When you do this, Arvados keeps track of the original URL the resource came from. This allows you to refer to the resource by its original URL in Workflow inputs, but actually read from the local copy in Keep.
<notextile>
-<pre><code>~$ <span class="userinput">arv-copy --project-uuid tordo-j7d0g-lr8sq3tx3ovn68k https://example.com/index.html
+<pre><code>~$ <span class="userinput">arv-copy --project-uuid tordo-j7d0g-lr8sq3tx3ovn68k https://example.com/index.html</span>
tordo-4zz18-dhpb6y9km2byb94
2023-10-06 10:15:36 arvados.arv-copy[374147] INFO: Success: created copy with uuid tordo-4zz18-dhpb6y9km2byb94
</code></pre>
commit a24ea36c4b5175229179040e842760fd11ba7b7a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Oct 11 10:16:44 2023 -0400
Remove duplicated ~$
refs #20937
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index 46ecfb2394..e7c250ac48 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -98,7 +98,7 @@ We will use the uuid @jutro-j7d0g-xj19djofle3aryq@ as an example project.
Arv-copy will infer the source cluster is @jutro@ from the source project uuid, and destination cluster is @pirca@ from @--project-uuid at .
<notextile>
-<pre><code>~$ <span class="userinput">~$ arv-copy --project-uuid pirca-j7d0g-lr8sq3tx3ovn68k jutro-j7d0g-xj19djofle3aryq
+<pre><code>~$ <span class="userinput">arv-copy --project-uuid pirca-j7d0g-lr8sq3tx3ovn68k jutro-j7d0g-xj19djofle3aryq
2021-09-08 21:29:32 arvados.arv-copy[6377] INFO:
2021-09-08 21:29:32 arvados.arv-copy[6377] INFO: Success: created copy with uuid pirca-j7d0g-ig9gvu5piznducp
</code></pre>
@@ -113,7 +113,7 @@ h3. Importing HTTP resources to Keep
You can also use @arv-copy@ to copy the contents of a HTTP URL into Keep. When you do this, Arvados keeps track of the original URL the resource came from. This allows you to refer to the resource by its original URL in Workflow inputs, but actually read from the local copy in Keep.
<notextile>
-<pre><code>~$ <span class="userinput">~$ arv-copy --project-uuid tordo-j7d0g-lr8sq3tx3ovn68k https://example.com/index.html
+<pre><code>~$ <span class="userinput">arv-copy --project-uuid tordo-j7d0g-lr8sq3tx3ovn68k https://example.com/index.html
tordo-4zz18-dhpb6y9km2byb94
2023-10-06 10:15:36 arvados.arv-copy[374147] INFO: Success: created copy with uuid tordo-4zz18-dhpb6y9km2byb94
</code></pre>
commit b9fbbeca2c603820815cc82293edb795f1e95127
Author: Brett Smith <brett.smith at curii.com>
Date: Thu Oct 12 14:52:55 2023 -0400
Merge branch '19821-collection-docstrings'
Closes #19821.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index bfb43be5eb..9e6bd06071 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1,6 +1,16 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Tools to work with Arvados collections
+
+This module provides high-level interfaces to create, read, and update
+Arvados collections. Most users will want to instantiate `Collection`
+objects, and use methods like `Collection.open` and `Collection.mkdirs` to
+read and write data in the collection. Refer to the Arvados Python SDK
+cookbook for [an introduction to using the Collection class][cookbook].
+
+[cookbook]: https://doc.arvados.org/sdk/python/cookbook.html#working-with-collections
+"""
from __future__ import absolute_import
from future.utils import listitems, listvalues, viewkeys
@@ -35,15 +45,65 @@ import arvados.util
import arvados.events as events
from arvados.retry import retry_method
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ IO,
+ Iterator,
+ List,
+ Mapping,
+ Optional,
+ Tuple,
+ Union,
+)
+
+if sys.version_info < (3, 8):
+ from typing_extensions import Literal
+else:
+ from typing import Literal
+
_logger = logging.getLogger('arvados.collection')
+ADD = "add"
+"""Argument value for `Collection` methods to represent an added item"""
+DEL = "del"
+"""Argument value for `Collection` methods to represent a removed item"""
+MOD = "mod"
+"""Argument value for `Collection` methods to represent a modified item"""
+TOK = "tok"
+"""Argument value for `Collection` methods to represent an item with token differences"""
+FILE = "file"
+"""`create_type` value for `Collection.find_or_create`"""
+COLLECTION = "collection"
+"""`create_type` value for `Collection.find_or_create`"""
+
+ChangeList = List[Union[
+ Tuple[Literal[ADD, DEL], str, 'Collection'],
+ Tuple[Literal[MOD, TOK], str, 'Collection', 'Collection'],
+]]
+ChangeType = Literal[ADD, DEL, MOD, TOK]
+CollectionItem = Union[ArvadosFile, 'Collection']
+ChangeCallback = Callable[[ChangeType, 'Collection', str, CollectionItem], object]
+CreateType = Literal[COLLECTION, FILE]
+Properties = Dict[str, Any]
+StorageClasses = List[str]
+
class CollectionBase(object):
- """Abstract base class for Collection classes."""
+ """Abstract base class for Collection classes
+
+ .. ATTENTION:: Internal
+ This class is meant to be used by other parts of the SDK. User code
+ should instantiate or subclass `Collection` or one of its subclasses
+ directly.
+ """
def __enter__(self):
+ """Enter a context block with this collection instance"""
return self
def __exit__(self, exc_type, exc_value, traceback):
+ """Exit a context block with this collection instance"""
pass
def _my_keep(self):
@@ -52,12 +112,13 @@ class CollectionBase(object):
num_retries=self.num_retries)
return self._keep_client
- def stripped_manifest(self):
- """Get the manifest with locator hints stripped.
+ def stripped_manifest(self) -> str:
+ """Create a copy of the collection manifest with only size hints
- Return the manifest for the current collection with all
- non-portable hints (i.e., permission signatures and other
- hints other than size hints) removed from the locators.
+ This method returns a string with the current collection's manifest
+ text with all non-portable locator hints like permission hints and
+ remote cluster hints removed. The only hints in the returned manifest
+ will be size hints.
"""
raw = self.manifest_text()
clean = []
@@ -96,709 +157,379 @@ class _WriterFile(_FileLikeObjectBase):
self.dest.flush_data()
-class CollectionWriter(CollectionBase):
- """Deprecated, use Collection instead."""
+class RichCollectionBase(CollectionBase):
+ """Base class for Collection classes
- @arvados.util._deprecated('3.0', 'arvados.collection.Collection')
- def __init__(self, api_client=None, num_retries=0, replication=None):
- """Instantiate a CollectionWriter.
+ .. ATTENTION:: Internal
+ This class is meant to be used by other parts of the SDK. User code
+ should instantiate or subclass `Collection` or one of its subclasses
+ directly.
+ """
- CollectionWriter lets you build a new Arvados Collection from scratch.
- Write files to it. The CollectionWriter will upload data to Keep as
- appropriate, and provide you with the Collection manifest text when
- you're finished.
+ def __init__(self, parent=None):
+ self.parent = parent
+ self._committed = False
+ self._has_remote_blocks = False
+ self._callback = None
+ self._items = {}
- Arguments:
- * api_client: The API client to use to look up Collections. If not
- provided, CollectionReader will build one from available Arvados
- configuration.
- * num_retries: The default number of times to retry failed
- service requests. Default 0. You may change this value
- after instantiation, but note those changes may not
- propagate to related objects like the Keep client.
- * replication: The number of copies of each block to store.
- If this argument is None or not supplied, replication is
- the server-provided default if available, otherwise 2.
- """
- self._api_client = api_client
- self.num_retries = num_retries
- self.replication = (2 if replication is None else replication)
- self._keep_client = None
- self._data_buffer = []
- self._data_buffer_len = 0
- self._current_stream_files = []
- self._current_stream_length = 0
- self._current_stream_locators = []
- self._current_stream_name = '.'
- self._current_file_name = None
- self._current_file_pos = 0
- self._finished_streams = []
- self._close_file = None
- self._queued_file = None
- self._queued_dirents = deque()
- self._queued_trees = deque()
- self._last_open = None
+ def _my_api(self):
+ raise NotImplementedError()
- def __exit__(self, exc_type, exc_value, traceback):
- if exc_type is None:
- self.finish()
+ def _my_keep(self):
+ raise NotImplementedError()
- def do_queued_work(self):
- # The work queue consists of three pieces:
- # * _queued_file: The file object we're currently writing to the
- # Collection.
- # * _queued_dirents: Entries under the current directory
- # (_queued_trees[0]) that we want to write or recurse through.
- # This may contain files from subdirectories if
- # max_manifest_depth == 0 for this directory.
- # * _queued_trees: Directories that should be written as separate
- # streams to the Collection.
- # This function handles the smallest piece of work currently queued
- # (current file, then current directory, then next directory) until
- # no work remains. The _work_THING methods each do a unit of work on
- # THING. _queue_THING methods add a THING to the work queue.
- while True:
- if self._queued_file:
- self._work_file()
- elif self._queued_dirents:
- self._work_dirents()
- elif self._queued_trees:
- self._work_trees()
- else:
- break
+ def _my_block_manager(self):
+ raise NotImplementedError()
- def _work_file(self):
- while True:
- buf = self._queued_file.read(config.KEEP_BLOCK_SIZE)
- if not buf:
- break
- self.write(buf)
- self.finish_current_file()
- if self._close_file:
- self._queued_file.close()
- self._close_file = None
- self._queued_file = None
+ def writable(self) -> bool:
+ """Indicate whether this collection object can be modified
- def _work_dirents(self):
- path, stream_name, max_manifest_depth = self._queued_trees[0]
- if stream_name != self.current_stream_name():
- self.start_new_stream(stream_name)
- while self._queued_dirents:
- dirent = self._queued_dirents.popleft()
- target = os.path.join(path, dirent)
- if os.path.isdir(target):
- self._queue_tree(target,
- os.path.join(stream_name, dirent),
- max_manifest_depth - 1)
- else:
- self._queue_file(target, dirent)
- break
- if not self._queued_dirents:
- self._queued_trees.popleft()
+ This method returns `False` if this object is a `CollectionReader`,
+ else `True`.
+ """
+ raise NotImplementedError()
- def _work_trees(self):
- path, stream_name, max_manifest_depth = self._queued_trees[0]
- d = arvados.util.listdir_recursive(
- path, max_depth = (None if max_manifest_depth == 0 else 0))
- if d:
- self._queue_dirents(stream_name, d)
- else:
- self._queued_trees.popleft()
+ def root_collection(self) -> 'Collection':
+ """Get this collection's root collection object
- def _queue_file(self, source, filename=None):
- assert (self._queued_file is None), "tried to queue more than one file"
- if not hasattr(source, 'read'):
- source = open(source, 'rb')
- self._close_file = True
- else:
- self._close_file = False
- if filename is None:
- filename = os.path.basename(source.name)
- self.start_new_file(filename)
- self._queued_file = source
+ If you open a subcollection with `Collection.find`, calling this method
+ on that subcollection returns the source Collection object.
+ """
+ raise NotImplementedError()
- def _queue_dirents(self, stream_name, dirents):
- assert (not self._queued_dirents), "tried to queue more than one tree"
- self._queued_dirents = deque(sorted(dirents))
+ def stream_name(self) -> str:
+ """Get the name of the manifest stream represented by this collection
- def _queue_tree(self, path, stream_name, max_manifest_depth):
- self._queued_trees.append((path, stream_name, max_manifest_depth))
+ If you open a subcollection with `Collection.find`, calling this method
+ on that subcollection returns the name of the stream you opened.
+ """
+ raise NotImplementedError()
- def write_file(self, source, filename=None):
- self._queue_file(source, filename)
- self.do_queued_work()
+ @synchronized
+ def has_remote_blocks(self) -> bool:
+ """Indiciate whether the collection refers to remote data
- def write_directory_tree(self,
- path, stream_name='.', max_manifest_depth=-1):
- self._queue_tree(path, stream_name, max_manifest_depth)
- self.do_queued_work()
+ Returns `True` if the collection manifest includes any Keep locators
+ with a remote hint (`+R`), else `False`.
+ """
+ if self._has_remote_blocks:
+ return True
+ for item in self:
+ if self[item].has_remote_blocks():
+ return True
+ return False
- def write(self, newdata):
- if isinstance(newdata, bytes):
- pass
- elif isinstance(newdata, str):
- newdata = newdata.encode()
- elif hasattr(newdata, '__iter__'):
- for s in newdata:
- self.write(s)
- return
- self._data_buffer.append(newdata)
- self._data_buffer_len += len(newdata)
- self._current_stream_length += len(newdata)
- while self._data_buffer_len >= config.KEEP_BLOCK_SIZE:
- self.flush_data()
+ @synchronized
+ def set_has_remote_blocks(self, val: bool) -> None:
+ """Cache whether this collection refers to remote blocks
- def open(self, streampath, filename=None):
- """open(streampath[, filename]) -> file-like object
+ .. ATTENTION:: Internal
+ This method is only meant to be used by other Collection methods.
- Pass in the path of a file to write to the Collection, either as a
- single string or as two separate stream name and file name arguments.
- This method returns a file-like object you can write to add it to the
- Collection.
+ Set this collection's cached "has remote blocks" flag to the given
+ value.
+ """
+ self._has_remote_blocks = val
+ if self.parent:
+ self.parent.set_has_remote_blocks(val)
- You may only have one file object from the Collection open at a time,
- so be sure to close the object when you're done. Using the object in
- a with statement makes that easy::
+ @must_be_writable
+ @synchronized
+ def find_or_create(
+ self,
+ path: str,
+ create_type: CreateType,
+ ) -> CollectionItem:
+ """Get the item at the given path, creating it if necessary
+
+ If `path` refers to a stream in this collection, returns a
+ corresponding `Subcollection` object. If `path` refers to a file in
+ this collection, returns a corresponding
+ `arvados.arvfile.ArvadosFile` object. If `path` does not exist in
+ this collection, then this method creates a new object and returns
+ it, creating parent streams as needed. The type of object created is
+ determined by the value of `create_type`.
+
+ Arguments:
+
+ * path: str --- The path to find or create within this collection.
- with cwriter.open('./doc/page1.txt') as outfile:
- outfile.write(page1_data)
- with cwriter.open('./doc/page2.txt') as outfile:
- outfile.write(page2_data)
+ * create_type: Literal[COLLECTION, FILE] --- The type of object to
+ create at `path` if one does not exist. Passing `COLLECTION`
+ creates a stream and returns the corresponding
+ `Subcollection`. Passing `FILE` creates a new file and returns the
+ corresponding `arvados.arvfile.ArvadosFile`.
"""
- if filename is None:
- streampath, filename = split(streampath)
- if self._last_open and not self._last_open.closed:
- raise errors.AssertionError(
- u"can't open '{}' when '{}' is still open".format(
- filename, self._last_open.name))
- if streampath != self.current_stream_name():
- self.start_new_stream(streampath)
- self.set_current_file_name(filename)
- self._last_open = _WriterFile(self, filename)
- return self._last_open
+ pathcomponents = path.split("/", 1)
+ if pathcomponents[0]:
+ item = self._items.get(pathcomponents[0])
+ if len(pathcomponents) == 1:
+ if item is None:
+ # create new file
+ if create_type == COLLECTION:
+ item = Subcollection(self, pathcomponents[0])
+ else:
+ item = ArvadosFile(self, pathcomponents[0])
+ self._items[pathcomponents[0]] = item
+ self.set_committed(False)
+ self.notify(ADD, self, pathcomponents[0], item)
+ return item
+ else:
+ if item is None:
+ # create new collection
+ item = Subcollection(self, pathcomponents[0])
+ self._items[pathcomponents[0]] = item
+ self.set_committed(False)
+ self.notify(ADD, self, pathcomponents[0], item)
+ if isinstance(item, RichCollectionBase):
+ return item.find_or_create(pathcomponents[1], create_type)
+ else:
+ raise IOError(errno.ENOTDIR, "Not a directory", pathcomponents[0])
+ else:
+ return self
- def flush_data(self):
- data_buffer = b''.join(self._data_buffer)
- if data_buffer:
- self._current_stream_locators.append(
- self._my_keep().put(
- data_buffer[0:config.KEEP_BLOCK_SIZE],
- copies=self.replication))
- self._data_buffer = [data_buffer[config.KEEP_BLOCK_SIZE:]]
- self._data_buffer_len = len(self._data_buffer[0])
+ @synchronized
+ def find(self, path: str) -> CollectionItem:
+ """Get the item at the given path
- def start_new_file(self, newfilename=None):
- self.finish_current_file()
- self.set_current_file_name(newfilename)
+ If `path` refers to a stream in this collection, returns a
+ corresponding `Subcollection` object. If `path` refers to a file in
+ this collection, returns a corresponding
+ `arvados.arvfile.ArvadosFile` object. If `path` does not exist in
+ this collection, then this method raises `NotADirectoryError`.
- def set_current_file_name(self, newfilename):
- if re.search(r'[\t\n]', newfilename):
- raise errors.AssertionError(
- "Manifest filenames cannot contain whitespace: %s" %
- newfilename)
- elif re.search(r'\x00', newfilename):
- raise errors.AssertionError(
- "Manifest filenames cannot contain NUL characters: %s" %
- newfilename)
- self._current_file_name = newfilename
+ Arguments:
- def current_file_name(self):
- return self._current_file_name
+ * path: str --- The path to find or create within this collection.
+ """
+ if not path:
+ raise errors.ArgumentError("Parameter 'path' is empty.")
- def finish_current_file(self):
- if self._current_file_name is None:
- if self._current_file_pos == self._current_stream_length:
- return
- raise errors.AssertionError(
- "Cannot finish an unnamed file " +
- "(%d bytes at offset %d in '%s' stream)" %
- (self._current_stream_length - self._current_file_pos,
- self._current_file_pos,
- self._current_stream_name))
- self._current_stream_files.append([
- self._current_file_pos,
- self._current_stream_length - self._current_file_pos,
- self._current_file_name])
- self._current_file_pos = self._current_stream_length
- self._current_file_name = None
+ pathcomponents = path.split("/", 1)
+ if pathcomponents[0] == '':
+ raise IOError(errno.ENOTDIR, "Not a directory", pathcomponents[0])
- def start_new_stream(self, newstreamname='.'):
- self.finish_current_stream()
- self.set_current_stream_name(newstreamname)
+ item = self._items.get(pathcomponents[0])
+ if item is None:
+ return None
+ elif len(pathcomponents) == 1:
+ return item
+ else:
+ if isinstance(item, RichCollectionBase):
+ if pathcomponents[1]:
+ return item.find(pathcomponents[1])
+ else:
+ return item
+ else:
+ raise IOError(errno.ENOTDIR, "Not a directory", pathcomponents[0])
- def set_current_stream_name(self, newstreamname):
- if re.search(r'[\t\n]', newstreamname):
- raise errors.AssertionError(
- "Manifest stream names cannot contain whitespace: '%s'" %
- (newstreamname))
- self._current_stream_name = '.' if newstreamname=='' else newstreamname
+ @synchronized
+ def mkdirs(self, path: str) -> 'Subcollection':
+ """Create and return a subcollection at `path`
- def current_stream_name(self):
- return self._current_stream_name
+ If `path` exists within this collection, raises `FileExistsError`.
+ Otherwise, creates a stream at that path and returns the
+ corresponding `Subcollection`.
+ """
+ if self.find(path) != None:
+ raise IOError(errno.EEXIST, "Directory or file exists", path)
- def finish_current_stream(self):
- self.finish_current_file()
- self.flush_data()
- if not self._current_stream_files:
- pass
- elif self._current_stream_name is None:
- raise errors.AssertionError(
- "Cannot finish an unnamed stream (%d bytes in %d files)" %
- (self._current_stream_length, len(self._current_stream_files)))
- else:
- if not self._current_stream_locators:
- self._current_stream_locators.append(config.EMPTY_BLOCK_LOCATOR)
- self._finished_streams.append([self._current_stream_name,
- self._current_stream_locators,
- self._current_stream_files])
- self._current_stream_files = []
- self._current_stream_length = 0
- self._current_stream_locators = []
- self._current_stream_name = None
- self._current_file_pos = 0
- self._current_file_name = None
+ return self.find_or_create(path, COLLECTION)
- def finish(self):
- """Store the manifest in Keep and return its locator.
+ def open(
+ self,
+ path: str,
+ mode: str="r",
+ encoding: Optional[str]=None,
+ ) -> IO:
+ """Open a file-like object within the collection
- This is useful for storing manifest fragments (task outputs)
- temporarily in Keep during a Crunch job.
+ This method returns a file-like object that can read and/or write the
+ file located at `path` within the collection. If you attempt to write
+ a `path` that does not exist, the file is created with `find_or_create`.
+ If the file cannot be opened for any other reason, this method raises
+ `OSError` with an appropriate errno.
- In other cases you should make a collection instead, by
- sending manifest_text() to the API server's "create
- collection" endpoint.
- """
- return self._my_keep().put(self.manifest_text().encode(),
- copies=self.replication)
+ Arguments:
- def portable_data_hash(self):
- stripped = self.stripped_manifest().encode()
- return '{}+{}'.format(hashlib.md5(stripped).hexdigest(), len(stripped))
+ * path: str --- The path of the file to open within this collection
- def manifest_text(self):
- self.finish_current_stream()
- manifest = ''
+ * mode: str --- The mode to open this file. Supports all the same
+ values as `builtins.open`.
- for stream in self._finished_streams:
- if not re.search(r'^\.(/.*)?$', stream[0]):
- manifest += './'
- manifest += stream[0].replace(' ', '\\040')
- manifest += ' ' + ' '.join(stream[1])
- manifest += ' ' + ' '.join("%d:%d:%s" % (sfile[0], sfile[1], sfile[2].replace(' ', '\\040')) for sfile in stream[2])
- manifest += "\n"
+ * encoding: str | None --- The text encoding of the file. Only used
+ when the file is opened in text mode. The default is
+ platform-dependent.
+ """
+ if not re.search(r'^[rwa][bt]?\+?$', mode):
+ raise errors.ArgumentError("Invalid mode {!r}".format(mode))
- return manifest
+ if mode[0] == 'r' and '+' not in mode:
+ fclass = ArvadosFileReader
+ arvfile = self.find(path)
+ elif not self.writable():
+ raise IOError(errno.EROFS, "Collection is read only")
+ else:
+ fclass = ArvadosFileWriter
+ arvfile = self.find_or_create(path, FILE)
- def data_locators(self):
- ret = []
- for name, locators, files in self._finished_streams:
- ret += locators
- return ret
+ if arvfile is None:
+ raise IOError(errno.ENOENT, "File not found", path)
+ if not isinstance(arvfile, ArvadosFile):
+ raise IOError(errno.EISDIR, "Is a directory", path)
- def save_new(self, name=None):
- return self._api_client.collections().create(
- ensure_unique_name=True,
- body={
- 'name': name,
- 'manifest_text': self.manifest_text(),
- }).execute(num_retries=self.num_retries)
+ if mode[0] == 'w':
+ arvfile.truncate(0)
+ binmode = mode[0] + 'b' + re.sub('[bt]', '', mode[1:])
+ f = fclass(arvfile, mode=binmode, num_retries=self.num_retries)
+ if 'b' not in mode:
+ bufferclass = io.BufferedRandom if f.writable() else io.BufferedReader
+ f = io.TextIOWrapper(bufferclass(WrappableFile(f)), encoding=encoding)
+ return f
-class ResumableCollectionWriter(CollectionWriter):
- """Deprecated, use Collection instead."""
+ def modified(self) -> bool:
+ """Indicate whether this collection has an API server record
- STATE_PROPS = ['_current_stream_files', '_current_stream_length',
- '_current_stream_locators', '_current_stream_name',
- '_current_file_name', '_current_file_pos', '_close_file',
- '_data_buffer', '_dependencies', '_finished_streams',
- '_queued_dirents', '_queued_trees']
+ Returns `False` if this collection corresponds to a record loaded from
+ the API server, `True` otherwise.
+ """
+ return not self.committed()
- @arvados.util._deprecated('3.0', 'arvados.collection.Collection')
- def __init__(self, api_client=None, **kwargs):
- self._dependencies = {}
- super(ResumableCollectionWriter, self).__init__(api_client, **kwargs)
+ @synchronized
+ def committed(self):
+ """Indicate whether this collection has an API server record
- @classmethod
- def from_state(cls, state, *init_args, **init_kwargs):
- # Try to build a new writer from scratch with the given state.
- # If the state is not suitable to resume (because files have changed,
- # been deleted, aren't predictable, etc.), raise a
- # StaleWriterStateError. Otherwise, return the initialized writer.
- # The caller is responsible for calling writer.do_queued_work()
- # appropriately after it's returned.
- writer = cls(*init_args, **init_kwargs)
- for attr_name in cls.STATE_PROPS:
- attr_value = state[attr_name]
- attr_class = getattr(writer, attr_name).__class__
- # Coerce the value into the same type as the initial value, if
- # needed.
- if attr_class not in (type(None), attr_value.__class__):
- attr_value = attr_class(attr_value)
- setattr(writer, attr_name, attr_value)
- # Check dependencies before we try to resume anything.
- if any(KeepLocator(ls).permission_expired()
- for ls in writer._current_stream_locators):
- raise errors.StaleWriterStateError(
- "locators include expired permission hint")
- writer.check_dependencies()
- if state['_current_file'] is not None:
- path, pos = state['_current_file']
- try:
- writer._queued_file = open(path, 'rb')
- writer._queued_file.seek(pos)
- except IOError as error:
- raise errors.StaleWriterStateError(
- u"failed to reopen active file {}: {}".format(path, error))
- return writer
+ Returns `True` if this collection corresponds to a record loaded from
+ the API server, `False` otherwise.
+ """
+ return self._committed
- def check_dependencies(self):
- for path, orig_stat in listitems(self._dependencies):
- if not S_ISREG(orig_stat[ST_MODE]):
- raise errors.StaleWriterStateError(u"{} not file".format(path))
- try:
- now_stat = tuple(os.stat(path))
- except OSError as error:
- raise errors.StaleWriterStateError(
- u"failed to stat {}: {}".format(path, error))
- if ((not S_ISREG(now_stat[ST_MODE])) or
- (orig_stat[ST_MTIME] != now_stat[ST_MTIME]) or
- (orig_stat[ST_SIZE] != now_stat[ST_SIZE])):
- raise errors.StaleWriterStateError(u"{} changed".format(path))
+ @synchronized
+ def set_committed(self, value: bool=True):
+ """Cache whether this collection has an API server record
- def dump_state(self, copy_func=lambda x: x):
- state = {attr: copy_func(getattr(self, attr))
- for attr in self.STATE_PROPS}
- if self._queued_file is None:
- state['_current_file'] = None
- else:
- state['_current_file'] = (os.path.realpath(self._queued_file.name),
- self._queued_file.tell())
- return state
+ .. ATTENTION:: Internal
+ This method is only meant to be used by other Collection methods.
- def _queue_file(self, source, filename=None):
- try:
- src_path = os.path.realpath(source)
- except Exception:
- raise errors.AssertionError(u"{} not a file path".format(source))
- try:
- path_stat = os.stat(src_path)
- except OSError as stat_error:
- path_stat = None
- super(ResumableCollectionWriter, self)._queue_file(source, filename)
- fd_stat = os.fstat(self._queued_file.fileno())
- if not S_ISREG(fd_stat.st_mode):
- # We won't be able to resume from this cache anyway, so don't
- # worry about further checks.
- self._dependencies[source] = tuple(fd_stat)
- elif path_stat is None:
- raise errors.AssertionError(
- u"could not stat {}: {}".format(source, stat_error))
- elif path_stat.st_ino != fd_stat.st_ino:
- raise errors.AssertionError(
- u"{} changed between open and stat calls".format(source))
+ Set this collection's cached "committed" flag to the given
+ value and propagates it as needed.
+ """
+ if value == self._committed:
+ return
+ if value:
+ for k,v in listitems(self._items):
+ v.set_committed(True)
+ self._committed = True
else:
- self._dependencies[src_path] = tuple(fd_stat)
+ self._committed = False
+ if self.parent is not None:
+ self.parent.set_committed(False)
- def write(self, data):
- if self._queued_file is None:
- raise errors.AssertionError(
- "resumable writer can't accept unsourced data")
- return super(ResumableCollectionWriter, self).write(data)
+ @synchronized
+ def __iter__(self) -> Iterator[str]:
+ """Iterate names of streams and files in this collection
+ This method does not recurse. It only iterates the contents of this
+ collection's corresponding stream.
+ """
+ return iter(viewkeys(self._items))
-ADD = "add"
-DEL = "del"
-MOD = "mod"
-TOK = "tok"
-FILE = "file"
-COLLECTION = "collection"
+ @synchronized
+ def __getitem__(self, k: str) -> CollectionItem:
+ """Get a `arvados.arvfile.ArvadosFile` or `Subcollection` in this collection
-class RichCollectionBase(CollectionBase):
- """Base class for Collections and Subcollections.
+ This method does not recurse. If you want to search a path, use
+ `RichCollectionBase.find` instead.
+ """
+ return self._items[k]
- Implements the majority of functionality relating to accessing items in the
- Collection.
-
- """
-
- def __init__(self, parent=None):
- self.parent = parent
- self._committed = False
- self._has_remote_blocks = False
- self._callback = None
- self._items = {}
-
- def _my_api(self):
- raise NotImplementedError()
-
- def _my_keep(self):
- raise NotImplementedError()
-
- def _my_block_manager(self):
- raise NotImplementedError()
-
- def writable(self):
- raise NotImplementedError()
-
- def root_collection(self):
- raise NotImplementedError()
-
- def notify(self, event, collection, name, item):
- raise NotImplementedError()
-
- def stream_name(self):
- raise NotImplementedError()
+ @synchronized
+ def __contains__(self, k: str) -> bool:
+ """Indicate whether this collection has an item with this name
+ This method does not recurse. It you want to check a path, use
+ `RichCollectionBase.exists` instead.
+ """
+ return k in self._items
@synchronized
- def has_remote_blocks(self):
- """Recursively check for a +R segment locator signature."""
-
- if self._has_remote_blocks:
- return True
- for item in self:
- if self[item].has_remote_blocks():
- return True
- return False
+ def __len__(self):
+ """Get the number of items directly contained in this collection
- @synchronized
- def set_has_remote_blocks(self, val):
- self._has_remote_blocks = val
- if self.parent:
- self.parent.set_has_remote_blocks(val)
+ This method does not recurse. It only counts the streams and files
+ in this collection's corresponding stream.
+ """
+ return len(self._items)
@must_be_writable
@synchronized
- def find_or_create(self, path, create_type):
- """Recursively search the specified file path.
-
- May return either a `Collection` or `ArvadosFile`. If not found, will
- create a new item at the specified path based on `create_type`. Will
- create intermediate subcollections needed to contain the final item in
- the path.
-
- :create_type:
- One of `arvados.collection.FILE` or
- `arvados.collection.COLLECTION`. If the path is not found, and value
- of create_type is FILE then create and return a new ArvadosFile for
- the last path component. If COLLECTION, then create and return a new
- Collection for the last path component.
+ def __delitem__(self, p: str) -> None:
+ """Delete an item from this collection's stream
+ This method does not recurse. If you want to remove an item by a
+ path, use `RichCollectionBase.remove` instead.
"""
-
- pathcomponents = path.split("/", 1)
- if pathcomponents[0]:
- item = self._items.get(pathcomponents[0])
- if len(pathcomponents) == 1:
- if item is None:
- # create new file
- if create_type == COLLECTION:
- item = Subcollection(self, pathcomponents[0])
- else:
- item = ArvadosFile(self, pathcomponents[0])
- self._items[pathcomponents[0]] = item
- self.set_committed(False)
- self.notify(ADD, self, pathcomponents[0], item)
- return item
- else:
- if item is None:
- # create new collection
- item = Subcollection(self, pathcomponents[0])
- self._items[pathcomponents[0]] = item
- self.set_committed(False)
- self.notify(ADD, self, pathcomponents[0], item)
- if isinstance(item, RichCollectionBase):
- return item.find_or_create(pathcomponents[1], create_type)
- else:
- raise IOError(errno.ENOTDIR, "Not a directory", pathcomponents[0])
- else:
- return self
+ del self._items[p]
+ self.set_committed(False)
+ self.notify(DEL, self, p, None)
@synchronized
- def find(self, path):
- """Recursively search the specified file path.
-
- May return either a Collection or ArvadosFile. Return None if not
- found.
- If path is invalid (ex: starts with '/'), an IOError exception will be
- raised.
+ def keys(self) -> Iterator[str]:
+ """Iterate names of streams and files in this collection
+ This method does not recurse. It only iterates the contents of this
+ collection's corresponding stream.
"""
- if not path:
- raise errors.ArgumentError("Parameter 'path' is empty.")
-
- pathcomponents = path.split("/", 1)
- if pathcomponents[0] == '':
- raise IOError(errno.ENOTDIR, "Not a directory", pathcomponents[0])
-
- item = self._items.get(pathcomponents[0])
- if item is None:
- return None
- elif len(pathcomponents) == 1:
- return item
- else:
- if isinstance(item, RichCollectionBase):
- if pathcomponents[1]:
- return item.find(pathcomponents[1])
- else:
- return item
- else:
- raise IOError(errno.ENOTDIR, "Not a directory", pathcomponents[0])
+ return self._items.keys()
@synchronized
- def mkdirs(self, path):
- """Recursive subcollection create.
-
- Like `os.makedirs()`. Will create intermediate subcollections needed
- to contain the leaf subcollection path.
-
- """
-
- if self.find(path) != None:
- raise IOError(errno.EEXIST, "Directory or file exists", path)
-
- return self.find_or_create(path, COLLECTION)
-
- def open(self, path, mode="r", encoding=None):
- """Open a file-like object for access.
-
- :path:
- path to a file in the collection
- :mode:
- a string consisting of "r", "w", or "a", optionally followed
- by "b" or "t", optionally followed by "+".
- :"b":
- binary mode: write() accepts bytes, read() returns bytes.
- :"t":
- text mode (default): write() accepts strings, read() returns strings.
- :"r":
- opens for reading
- :"r+":
- opens for reading and writing. Reads/writes share a file pointer.
- :"w", "w+":
- truncates to 0 and opens for reading and writing. Reads/writes share a file pointer.
- :"a", "a+":
- opens for reading and writing. All writes are appended to
- the end of the file. Writing does not affect the file pointer for
- reading.
+ def values(self) -> List[CollectionItem]:
+ """Get a list of objects in this collection's stream
+ The return value includes a `Subcollection` for every stream, and an
+ `arvados.arvfile.ArvadosFile` for every file, directly within this
+ collection's stream. This method does not recurse.
"""
-
- if not re.search(r'^[rwa][bt]?\+?$', mode):
- raise errors.ArgumentError("Invalid mode {!r}".format(mode))
-
- if mode[0] == 'r' and '+' not in mode:
- fclass = ArvadosFileReader
- arvfile = self.find(path)
- elif not self.writable():
- raise IOError(errno.EROFS, "Collection is read only")
- else:
- fclass = ArvadosFileWriter
- arvfile = self.find_or_create(path, FILE)
-
- if arvfile is None:
- raise IOError(errno.ENOENT, "File not found", path)
- if not isinstance(arvfile, ArvadosFile):
- raise IOError(errno.EISDIR, "Is a directory", path)
-
- if mode[0] == 'w':
- arvfile.truncate(0)
-
- binmode = mode[0] + 'b' + re.sub('[bt]', '', mode[1:])
- f = fclass(arvfile, mode=binmode, num_retries=self.num_retries)
- if 'b' not in mode:
- bufferclass = io.BufferedRandom if f.writable() else io.BufferedReader
- f = io.TextIOWrapper(bufferclass(WrappableFile(f)), encoding=encoding)
- return f
-
- def modified(self):
- """Determine if the collection has been modified since last commited."""
- return not self.committed()
-
- @synchronized
- def committed(self):
- """Determine if the collection has been committed to the API server."""
- return self._committed
+ return listvalues(self._items)
@synchronized
- def set_committed(self, value=True):
- """Recursively set committed flag.
+ def items(self) -> List[Tuple[str, CollectionItem]]:
+ """Get a list of `(name, object)` tuples from this collection's stream
- If value is True, set committed to be True for this and all children.
-
- If value is False, set committed to be False for this and all parents.
+ The return value includes a `Subcollection` for every stream, and an
+ `arvados.arvfile.ArvadosFile` for every file, directly within this
+ collection's stream. This method does not recurse.
"""
- if value == self._committed:
- return
- if value:
- for k,v in listitems(self._items):
- v.set_committed(True)
- self._committed = True
- else:
- self._committed = False
- if self.parent is not None:
- self.parent.set_committed(False)
+ return listitems(self._items)
- @synchronized
- def __iter__(self):
- """Iterate over names of files and collections contained in this collection."""
- return iter(viewkeys(self._items))
+ def exists(self, path: str) -> bool:
+ """Indicate whether this collection includes an item at `path`
- @synchronized
- def __getitem__(self, k):
- """Get a file or collection that is directly contained by this collection.
+ This method returns `True` if `path` refers to a stream or file within
+ this collection, else `False`.
- If you want to search a path, use `find()` instead.
+ Arguments:
+ * path: str --- The path to check for existence within this collection
"""
- return self._items[k]
-
- @synchronized
- def __contains__(self, k):
- """Test if there is a file or collection a directly contained by this collection."""
- return k in self._items
-
- @synchronized
- def __len__(self):
- """Get the number of items directly contained in this collection."""
- return len(self._items)
+ return self.find(path) is not None
@must_be_writable
@synchronized
- def __delitem__(self, p):
- """Delete an item by name which is directly contained by this collection."""
- del self._items[p]
- self.set_committed(False)
- self.notify(DEL, self, p, None)
-
- @synchronized
- def keys(self):
- """Get a list of names of files and collections directly contained in this collection."""
- return self._items.keys()
-
- @synchronized
- def values(self):
- """Get a list of files and collection objects directly contained in this collection."""
- return listvalues(self._items)
-
- @synchronized
- def items(self):
- """Get a list of (name, object) tuples directly contained in this collection."""
- return listitems(self._items)
+ def remove(self, path: str, recursive: bool=False) -> None:
+ """Remove the file or stream at `path`
- def exists(self, path):
- """Test if there is a file or collection at `path`."""
- return self.find(path) is not None
+ Arguments:
- @must_be_writable
- @synchronized
- def remove(self, path, recursive=False):
- """Remove the file or subcollection (directory) at `path`.
+ * path: str --- The path of the item to remove from the collection
- :recursive:
- Specify whether to remove non-empty subcollections (True), or raise an error (False).
+ * recursive: bool --- Controls the method's behavior if `path` refers
+ to a nonempty stream. If `False` (the default), this method raises
+ `OSError` with errno `ENOTEMPTY`. If `True`, this method removes all
+ items under the stream.
"""
-
if not path:
raise errors.ArgumentError("Parameter 'path' is empty.")
@@ -825,26 +556,33 @@ class RichCollectionBase(CollectionBase):
@must_be_writable
@synchronized
- def add(self, source_obj, target_name, overwrite=False, reparent=False):
- """Copy or move a file or subcollection to this collection.
+ def add(
+ self,
+ source_obj: CollectionItem,
+ target_name: str,
+ overwrite: bool=False,
+ reparent: bool=False,
+ ) -> None:
+ """Copy or move a file or subcollection object to this collection
- :source_obj:
- An ArvadosFile, or Subcollection object
+ Arguments:
- :target_name:
- Destination item name. If the target name already exists and is a
- file, this will raise an error unless you specify `overwrite=True`.
+ * source_obj: arvados.arvfile.ArvadosFile | Subcollection --- The file or subcollection
+ to add to this collection
- :overwrite:
- Whether to overwrite target file if it already exists.
+ * target_name: str --- The path inside this collection where
+ `source_obj` should be added.
- :reparent:
- If True, source_obj will be moved from its parent collection to this collection.
- If False, source_obj will be copied and the parent collection will be
- unmodified.
+ * overwrite: bool --- Controls the behavior of this method when the
+ collection already contains an object at `target_name`. If `False`
+ (the default), this method will raise `FileExistsError`. If `True`,
+ the object at `target_name` will be replaced with `source_obj`.
+ * reparent: bool --- Controls whether this method copies or moves
+ `source_obj`. If `False` (the default), `source_obj` is copied into
+ this collection. If `True`, `source_obj` is moved into this
+ collection.
"""
-
if target_name in self and not overwrite:
raise IOError(errno.EEXIST, "File already exists", target_name)
@@ -911,92 +649,117 @@ class RichCollectionBase(CollectionBase):
@must_be_writable
@synchronized
- def copy(self, source, target_path, source_collection=None, overwrite=False):
- """Copy a file or subcollection to a new path in this collection.
+ def copy(
+ self,
+ source: Union[str, CollectionItem],
+ target_path: str,
+ source_collection: Optional['RichCollectionBase']=None,
+ overwrite: bool=False,
+ ) -> None:
+ """Copy a file or subcollection object to this collection
- :source:
- A string with a path to source file or subcollection, or an actual ArvadosFile or Subcollection object.
+ Arguments:
- :target_path:
- Destination file or path. If the target path already exists and is a
- subcollection, the item will be placed inside the subcollection. If
- the target path already exists and is a file, this will raise an error
- unless you specify `overwrite=True`.
+ * source: str | arvados.arvfile.ArvadosFile |
+ arvados.collection.Subcollection --- The file or subcollection to
+ add to this collection. If `source` is a str, the object will be
+ found by looking up this path from `source_collection` (see
+ below).
- :source_collection:
- Collection to copy `source_path` from (default `self`)
+ * target_path: str --- The path inside this collection where the
+ source object should be added.
- :overwrite:
- Whether to overwrite target file if it already exists.
- """
+ * source_collection: arvados.collection.Collection | None --- The
+ collection to find the source object from when `source` is a
+ path. Defaults to the current collection (`self`).
+ * overwrite: bool --- Controls the behavior of this method when the
+ collection already contains an object at `target_path`. If `False`
+ (the default), this method will raise `FileExistsError`. If `True`,
+ the object at `target_path` will be replaced with `source_obj`.
+ """
source_obj, target_dir, target_name = self._get_src_target(source, target_path, source_collection, True)
target_dir.add(source_obj, target_name, overwrite, False)
@must_be_writable
@synchronized
- def rename(self, source, target_path, source_collection=None, overwrite=False):
- """Move a file or subcollection from `source_collection` to a new path in this collection.
+ def rename(
+ self,
+ source: Union[str, CollectionItem],
+ target_path: str,
+ source_collection: Optional['RichCollectionBase']=None,
+ overwrite: bool=False,
+ ) -> None:
+ """Move a file or subcollection object to this collection
+
+ Arguments:
- :source:
- A string with a path to source file or subcollection.
+ * source: str | arvados.arvfile.ArvadosFile |
+ arvados.collection.Subcollection --- The file or subcollection to
+ add to this collection. If `source` is a str, the object will be
+ found by looking up this path from `source_collection` (see
+ below).
- :target_path:
- Destination file or path. If the target path already exists and is a
- subcollection, the item will be placed inside the subcollection. If
- the target path already exists and is a file, this will raise an error
- unless you specify `overwrite=True`.
+ * target_path: str --- The path inside this collection where the
+ source object should be added.
- :source_collection:
- Collection to copy `source_path` from (default `self`)
+ * source_collection: arvados.collection.Collection | None --- The
+ collection to find the source object from when `source` is a
+ path. Defaults to the current collection (`self`).
- :overwrite:
- Whether to overwrite target file if it already exists.
+ * overwrite: bool --- Controls the behavior of this method when the
+ collection already contains an object at `target_path`. If `False`
+ (the default), this method will raise `FileExistsError`. If `True`,
+ the object at `target_path` will be replaced with `source_obj`.
"""
-
source_obj, target_dir, target_name = self._get_src_target(source, target_path, source_collection, False)
if not source_obj.writable():
raise IOError(errno.EROFS, "Source collection is read only", source)
target_dir.add(source_obj, target_name, overwrite, True)
- def portable_manifest_text(self, stream_name="."):
- """Get the manifest text for this collection, sub collections and files.
+ def portable_manifest_text(self, stream_name: str=".") -> str:
+ """Get the portable manifest text for this collection
- This method does not flush outstanding blocks to Keep. It will return
- a normalized manifest with access tokens stripped.
+ The portable manifest text is normalized, and does not include access
+ tokens. This method does not flush outstanding blocks to Keep.
- :stream_name:
- Name to use for this stream (directory)
+ Arguments:
+ * stream_name: str --- The name to use for this collection's stream in
+ the generated manifest. Default `'.'`.
"""
return self._get_manifest_text(stream_name, True, True)
@synchronized
- def manifest_text(self, stream_name=".", strip=False, normalize=False,
- only_committed=False):
- """Get the manifest text for this collection, sub collections and files.
-
- This method will flush outstanding blocks to Keep. By default, it will
- not normalize an unmodified manifest or strip access tokens.
+ def manifest_text(
+ self,
+ stream_name: str=".",
+ strip: bool=False,
+ normalize: bool=False,
+ only_committed: bool=False,
+ ) -> str:
+ """Get the manifest text for this collection
- :stream_name:
- Name to use for this stream (directory)
+ Arguments:
- :strip:
- If True, remove signing tokens from block locators if present.
- If False (default), block locators are left unchanged.
+ * stream_name: str --- The name to use for this collection's stream in
+ the generated manifest. Default `'.'`.
- :normalize:
- If True, always export the manifest text in normalized form
- even if the Collection is not modified. If False (default) and the collection
- is not modified, return the original manifest text even if it is not
- in normalized form.
+ * strip: bool --- Controls whether or not the returned manifest text
+ includes access tokens. If `False` (the default), the manifest text
+ will include access tokens. If `True`, the manifest text will not
+ include access tokens.
- :only_committed:
- If True, don't commit pending blocks.
+ * normalize: bool --- Controls whether or not the returned manifest
+ text is normalized. Default `False`.
+ * only_committed: bool --- Controls whether or not this method uploads
+ pending data to Keep before building and returning the manifest text.
+ If `False` (the default), this method will finish uploading all data
+ to Keep, then return the final manifest. If `True`, this method will
+ build and return a manifest that only refers to the data that has
+ finished uploading at the time this method was called.
"""
-
if not only_committed:
self._my_block_manager().commit_all()
return self._get_manifest_text(stream_name, strip, normalize,
@@ -1075,11 +838,27 @@ class RichCollectionBase(CollectionBase):
return remote_blocks
@synchronized
- def diff(self, end_collection, prefix=".", holding_collection=None):
- """Generate list of add/modify/delete actions.
+ def diff(
+ self,
+ end_collection: 'RichCollectionBase',
+ prefix: str=".",
+ holding_collection: Optional['Collection']=None,
+ ) -> ChangeList:
+ """Build a list of differences between this collection and another
+
+ Arguments:
+
+ * end_collection: arvados.collection.RichCollectionBase --- A
+ collection object with the desired end state. The returned diff
+ list will describe how to go from the current collection object
+ `self` to `end_collection`.
- When given to `apply`, will change `self` to match `end_collection`
+ * prefix: str --- The name to use for this collection's stream in
+ the diff list. Default `'.'`.
+ * holding_collection: arvados.collection.Collection | None --- A
+ collection object used to hold objects for the returned diff
+ list. By default, a new empty collection is created.
"""
changes = []
if holding_collection is None:
@@ -1101,12 +880,20 @@ class RichCollectionBase(CollectionBase):
@must_be_writable
@synchronized
- def apply(self, changes):
- """Apply changes from `diff`.
+ def apply(self, changes: ChangeList) -> None:
+ """Apply a list of changes from to this collection
- If a change conflicts with a local change, it will be saved to an
- alternate path indicating the conflict.
+ This method takes a list of changes generated by
+ `RichCollectionBase.diff` and applies it to this
+ collection. Afterward, the state of this collection object will
+ match the state of `end_collection` passed to `diff`. If a change
+ conflicts with a local change, it will be saved to an alternate path
+ indicating the conflict.
+ Arguments:
+
+ * changes: arvados.collection.ChangeList --- The list of differences
+ generated by `RichCollectionBase.diff`.
"""
if changes:
self.set_committed(False)
@@ -1148,8 +935,8 @@ class RichCollectionBase(CollectionBase):
# else, the file is modified or already removed, in either
# case we don't want to try to remove it.
- def portable_data_hash(self):
- """Get the portable data hash for this collection's manifest."""
+ def portable_data_hash(self) -> str:
+ """Get the portable data hash for this collection's manifest"""
if self._manifest_locator and self.committed():
# If the collection is already saved on the API server, and it's committed
# then return API server's PDH response.
@@ -1159,25 +946,64 @@ class RichCollectionBase(CollectionBase):
return '{}+{}'.format(hashlib.md5(stripped).hexdigest(), len(stripped))
@synchronized
- def subscribe(self, callback):
+ def subscribe(self, callback: ChangeCallback) -> None:
+ """Set a notify callback for changes to this collection
+
+ Arguments:
+
+ * callback: arvados.collection.ChangeCallback --- The callable to
+ call each time the collection is changed.
+ """
if self._callback is None:
self._callback = callback
else:
raise errors.ArgumentError("A callback is already set on this collection.")
@synchronized
- def unsubscribe(self):
+ def unsubscribe(self) -> None:
+ """Remove any notify callback set for changes to this collection"""
if self._callback is not None:
self._callback = None
@synchronized
- def notify(self, event, collection, name, item):
+ def notify(
+ self,
+ event: ChangeType,
+ collection: 'RichCollectionBase',
+ name: str,
+ item: CollectionItem,
+ ) -> None:
+ """Notify any subscribed callback about a change to this collection
+
+ .. ATTENTION:: Internal
+ This method is only meant to be used by other Collection methods.
+
+ If a callback has been registered with `RichCollectionBase.subscribe`,
+ it will be called with information about a change to this collection.
+ Then this notification will be propagated to this collection's root.
+
+ Arguments:
+
+ * event: Literal[ADD, DEL, MOD, TOK] --- The type of modification to
+ the collection.
+
+ * collection: arvados.collection.RichCollectionBase --- The
+ collection that was modified.
+
+ * name: str --- The name of the file or stream within `collection` that
+ was modified.
+
+ * item: arvados.arvfile.ArvadosFile |
+ arvados.collection.Subcollection --- The new contents at `name`
+ within `collection`.
+ """
if self._callback:
self._callback(event, collection, name, item)
self.root_collection().notify(event, collection, name, item)
@synchronized
- def __eq__(self, other):
+ def __eq__(self, other: Any) -> bool:
+ """Indicate whether this collection object is equal to another"""
if other is self:
return True
if not isinstance(other, RichCollectionBase):
@@ -1191,101 +1017,97 @@ class RichCollectionBase(CollectionBase):
return False
return True
- def __ne__(self, other):
+ def __ne__(self, other: Any) -> bool:
+ """Indicate whether this collection object is not equal to another"""
return not self.__eq__(other)
@synchronized
- def flush(self):
- """Flush bufferblocks to Keep."""
+ def flush(self) -> None:
+ """Upload any pending data to Keep"""
for e in listvalues(self):
e.flush()
class Collection(RichCollectionBase):
- """Represents the root of an Arvados Collection.
-
- This class is threadsafe. The root collection object, all subcollections
- and files are protected by a single lock (i.e. each access locks the entire
- collection).
-
- Brief summary of
- useful methods:
-
- :To read an existing file:
- `c.open("myfile", "r")`
-
- :To write a new file:
- `c.open("myfile", "w")`
-
- :To determine if a file exists:
- `c.find("myfile") is not None`
+ """Read and manipulate an Arvados collection
- :To copy a file:
- `c.copy("source", "dest")`
-
- :To delete a file:
- `c.remove("myfile")`
-
- :To save to an existing collection record:
- `c.save()`
-
- :To save a new collection record:
- `c.save_new()`
-
- :To merge remote changes into this object:
- `c.update()`
-
- Must be associated with an API server Collection record (during
- initialization, or using `save_new`) to use `save` or `update`
+ This class provides a high-level interface to create, read, and update
+ Arvados collections and their contents. Refer to the Arvados Python SDK
+ cookbook for [an introduction to using the Collection class][cookbook].
+ [cookbook]: https://doc.arvados.org/sdk/python/cookbook.html#working-with-collections
"""
- def __init__(self, manifest_locator_or_text=None,
- api_client=None,
- keep_client=None,
- num_retries=10,
- parent=None,
- apiconfig=None,
- block_manager=None,
- replication_desired=None,
- storage_classes_desired=None,
- put_threads=None):
- """Collection constructor.
-
- :manifest_locator_or_text:
- An Arvados collection UUID, portable data hash, raw manifest
- text, or (if creating an empty collection) None.
-
- :parent:
- the parent Collection, may be None.
-
- :apiconfig:
- A dict containing keys for ARVADOS_API_HOST and ARVADOS_API_TOKEN.
- Prefer this over supplying your own api_client and keep_client (except in testing).
- Will use default config settings if not specified.
+ def __init__(self, manifest_locator_or_text: Optional[str]=None,
+ api_client: Optional['arvados.api_resources.ArvadosAPIClient']=None,
+ keep_client: Optional['arvados.keep.KeepClient']=None,
+ num_retries: int=10,
+ parent: Optional['Collection']=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ block_manager: Optional['arvados.arvfile._BlockManager']=None,
+ replication_desired: Optional[int]=None,
+ storage_classes_desired: Optional[List[str]]=None,
+ put_threads: Optional[int]=None):
+ """Initialize a Collection object
- :api_client:
- The API client object to use for requests. If not specified, create one using `apiconfig`.
-
- :keep_client:
- the Keep client to use for requests. If not specified, create one using `apiconfig`.
-
- :num_retries:
- the number of retries for API and Keep requests.
-
- :block_manager:
- the block manager to use. If not specified, create one.
-
- :replication_desired:
- How many copies should Arvados maintain. If None, API server default
- configuration applies. If not None, this value will also be used
- for determining the number of block copies being written.
-
- :storage_classes_desired:
- A list of storage class names where to upload the data. If None,
- the keep client is expected to store the data into the cluster's
- default storage class(es).
+ Arguments:
+ * manifest_locator_or_text: str | None --- This string can contain a
+ collection manifest text, portable data hash, or UUID. When given a
+ portable data hash or UUID, this instance will load a collection
+ record from the API server. Otherwise, this instance will represent a
+ new collection without an API server record. The default value `None`
+ instantiates a new collection with an empty manifest.
+
+ * api_client: arvados.api_resources.ArvadosAPIClient | None --- The
+ Arvados API client object this instance uses to make requests. If
+ none is given, this instance creates its own client using the
+ settings from `apiconfig` (see below). If your client instantiates
+ many Collection objects, you can help limit memory utilization by
+ calling `arvados.api.api` to construct an
+ `arvados.safeapi.ThreadSafeApiCache`, and use that as the `api_client`
+ for every Collection.
+
+ * keep_client: arvados.keep.KeepClient | None --- The Keep client
+ object this instance uses to make requests. If none is given, this
+ instance creates its own client using its `api_client`.
+
+ * num_retries: int --- The number of times that client requests are
+ retried. Default 10.
+
+ * parent: arvados.collection.Collection | None --- The parent Collection
+ object of this instance, if any. This argument is primarily used by
+ other Collection methods; user client code shouldn't need to use it.
+
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. When no `api_client` is provided, the
+ Collection object constructs one from these settings. If no
+ mapping is provided, calls `arvados.config.settings` to get these
+ parameters from user configuration.
+
+ * block_manager: arvados.arvfile._BlockManager | None --- The
+ _BlockManager object used by this instance to coordinate reading
+ and writing Keep data blocks. If none is given, this instance
+ constructs its own. This argument is primarily used by other
+ Collection methods; user client code shouldn't need to use it.
+
+ * replication_desired: int | None --- This controls both the value of
+ the `replication_desired` field on API collection records saved by
+ this class, as well as the number of Keep services that the object
+ writes new data blocks to. If none is given, uses the default value
+ configured for the cluster.
+
+ * storage_classes_desired: list[str] | None --- This controls both
+ the value of the `storage_classes_desired` field on API collection
+ records saved by this class, as well as selecting which specific
+ Keep services the object writes new data blocks to. If none is
+ given, defaults to an empty list.
+
+ * put_threads: int | None --- The number of threads to run
+ simultaneously to upload data blocks to Keep. This value is used when
+ building a new `block_manager`. It is unused when a `block_manager`
+ is provided.
"""
if storage_classes_desired and type(storage_classes_desired) is not list:
@@ -1339,19 +1161,33 @@ class Collection(RichCollectionBase):
except errors.SyntaxError as e:
raise errors.ArgumentError("Error processing manifest text: %s", str(e)) from None
- def storage_classes_desired(self):
+ def storage_classes_desired(self) -> List[str]:
+ """Get this collection's `storage_classes_desired` value"""
return self._storage_classes_desired or []
- def root_collection(self):
+ def root_collection(self) -> 'Collection':
return self
- def get_properties(self):
+ def get_properties(self) -> Properties:
+ """Get this collection's properties
+
+ This method always returns a dict. If this collection object does not
+ have an associated API record, or that record does not have any
+ properties set, this method returns an empty dict.
+ """
if self._api_response and self._api_response["properties"]:
return self._api_response["properties"]
else:
return {}
- def get_trash_at(self):
+ def get_trash_at(self) -> Optional[datetime.datetime]:
+ """Get this collection's `trash_at` field
+
+ This method parses the `trash_at` field of the collection's API
+ record and returns a datetime from it. If that field is not set, or
+ this collection object does not have an associated API record,
+ returns None.
+ """
if self._api_response and self._api_response["trash_at"]:
try:
return ciso8601.parse_datetime(self._api_response["trash_at"])
@@ -1360,21 +1196,57 @@ class Collection(RichCollectionBase):
else:
return None
- def stream_name(self):
+ def stream_name(self) -> str:
return "."
- def writable(self):
+ def writable(self) -> bool:
return True
@synchronized
- def known_past_version(self, modified_at_and_portable_data_hash):
+ def known_past_version(
+ self,
+ modified_at_and_portable_data_hash: Tuple[Optional[str], Optional[str]]
+ ) -> bool:
+ """Indicate whether an API record for this collection has been seen before
+
+ As this collection object loads records from the API server, it records
+ their `modified_at` and `portable_data_hash` fields. This method accepts
+ a 2-tuple with values for those fields, and returns `True` if the
+ combination was previously loaded.
+ """
return modified_at_and_portable_data_hash in self._past_versions
@synchronized
@retry_method
- def update(self, other=None, num_retries=None):
- """Merge the latest collection on the API server with the current collection."""
+ def update(
+ self,
+ other: Optional['Collection']=None,
+ num_retries: Optional[int]=None,
+ ) -> None:
+ """Merge another collection's contents into this one
+
+ This method compares the manifest of this collection instance with
+ another, then updates this instance's manifest with changes from the
+ other, renaming files to flag conflicts where necessary.
+
+ When called without any arguments, this method reloads the collection's
+ API record, and updates this instance with any changes that have
+ appeared server-side. If this instance does not have a corresponding
+ API record, this method raises `arvados.errors.ArgumentError`.
+
+ Arguments:
+
+ * other: arvados.collection.Collection | None --- The collection
+ whose contents should be merged into this instance. When not
+ provided, this method reloads this collection's API record and
+ constructs a Collection object from it. If this instance does not
+ have a corresponding API record, this method raises
+ `arvados.errors.ArgumentError`.
+ * num_retries: int | None --- The number of times to retry reloading
+ the collection's API record from the API server. If not specified,
+ uses the `num_retries` provided when this instance was constructed.
+ """
if other is None:
if self._manifest_locator is None:
raise errors.ArgumentError("`other` is None but collection does not have a manifest_locator uuid")
@@ -1467,32 +1339,65 @@ class Collection(RichCollectionBase):
return self
def __exit__(self, exc_type, exc_value, traceback):
- """Support scoped auto-commit in a with: block."""
- if exc_type is None:
+ """Exit a context with this collection instance
+
+ If no exception was raised inside the context block, and this
+ collection is writable and has a corresponding API record, that
+ record will be updated to match the state of this instance at the end
+ of the block.
+ """
+ if exc_type is None:
if self.writable() and self._has_collection_uuid():
self.save()
self.stop_threads()
- def stop_threads(self):
+ def stop_threads(self) -> None:
+ """Stop background Keep upload/download threads"""
if self._block_manager is not None:
self._block_manager.stop_threads()
@synchronized
- def manifest_locator(self):
- """Get the manifest locator, if any.
-
- The manifest locator will be set when the collection is loaded from an
- API server record or the portable data hash of a manifest.
-
- The manifest locator will be None if the collection is newly created or
- was created directly from manifest text. The method `save_new()` will
- assign a manifest locator.
-
+ def manifest_locator(self) -> Optional[str]:
+ """Get this collection's manifest locator, if any
+
+ * If this collection instance is associated with an API record with a
+ UUID, return that.
+ * Otherwise, if this collection instance was loaded from an API record
+ by portable data hash, return that.
+ * Otherwise, return `None`.
"""
return self._manifest_locator
@synchronized
- def clone(self, new_parent=None, new_name=None, readonly=False, new_config=None):
+ def clone(
+ self,
+ new_parent: Optional['Collection']=None,
+ new_name: Optional[str]=None,
+ readonly: bool=False,
+ new_config: Optional[Mapping[str, str]]=None,
+ ) -> 'Collection':
+ """Create a Collection object with the same contents as this instance
+
+ This method creates a new Collection object with contents that match
+ this instance's. The new collection will not be associated with any API
+ record.
+
+ Arguments:
+
+ * new_parent: arvados.collection.Collection | None --- This value is
+ passed to the new Collection's constructor as the `parent`
+ argument.
+
+ * new_name: str | None --- This value is unused.
+
+ * readonly: bool --- If this value is true, this method constructs and
+ returns a `CollectionReader`. Otherwise, it returns a mutable
+ `Collection`. Default `False`.
+
+ * new_config: Mapping[str, str] | None --- This value is passed to the
+ new Collection's constructor as `apiconfig`. If no value is provided,
+ defaults to the configuration passed to this instance's constructor.
+ """
if new_config is None:
new_config = self._config
if readonly:
@@ -1504,31 +1409,31 @@ class Collection(RichCollectionBase):
return newcollection
@synchronized
- def api_response(self):
- """Returns information about this Collection fetched from the API server.
-
- If the Collection exists in Keep but not the API server, currently
- returns None. Future versions may provide a synthetic response.
+ def api_response(self) -> Optional[Dict[str, Any]]:
+ """Get this instance's associated API record
+ If this Collection instance has an associated API record, return it.
+ Otherwise, return `None`.
"""
return self._api_response
- def find_or_create(self, path, create_type):
- """See `RichCollectionBase.find_or_create`"""
+ def find_or_create(
+ self,
+ path: str,
+ create_type: CreateType,
+ ) -> CollectionItem:
if path == ".":
return self
else:
return super(Collection, self).find_or_create(path[2:] if path.startswith("./") else path, create_type)
- def find(self, path):
- """See `RichCollectionBase.find`"""
+ def find(self, path: str) -> CollectionItem:
if path == ".":
return self
else:
return super(Collection, self).find(path[2:] if path.startswith("./") else path)
- def remove(self, path, recursive=False):
- """See `RichCollectionBase.remove`"""
+ def remove(self, path: str, recursive: bool=False) -> None:
if path == ".":
raise errors.ArgumentError("Cannot remove '.'")
else:
@@ -1537,49 +1442,52 @@ class Collection(RichCollectionBase):
@must_be_writable
@synchronized
@retry_method
- def save(self,
- properties=None,
- storage_classes=None,
- trash_at=None,
- merge=True,
- num_retries=None,
- preserve_version=False):
- """Save collection to an existing collection record.
-
- Commit pending buffer blocks to Keep, merge with remote record (if
- merge=True, the default), and update the collection record. Returns
- the current manifest text.
-
- Will raise AssertionError if not associated with a collection record on
- the API server. If you want to save a manifest to Keep only, see
- `save_new()`.
-
- :properties:
- Additional properties of collection. This value will replace any existing
- properties of collection.
-
- :storage_classes:
- Specify desirable storage classes to be used when writing data to Keep.
-
- :trash_at:
- A collection is *expiring* when it has a *trash_at* time in the future.
- An expiring collection can be accessed as normal,
- but is scheduled to be trashed automatically at the *trash_at* time.
-
- :merge:
- Update and merge remote changes before saving. Otherwise, any
- remote changes will be ignored and overwritten.
-
- :num_retries:
- Retry count on API calls (if None, use the collection default)
-
- :preserve_version:
- If True, indicate that the collection content being saved right now
- should be preserved in a version snapshot if the collection record is
- updated in the future. Requires that the API server has
- Collections.CollectionVersioning enabled, if not, setting this will
- raise an exception.
+ def save(
+ self,
+ properties: Optional[Properties]=None,
+ storage_classes: Optional[StorageClasses]=None,
+ trash_at: Optional[datetime.datetime]=None,
+ merge: bool=True,
+ num_retries: Optional[int]=None,
+ preserve_version: bool=False,
+ ) -> str:
+ """Save collection to an existing API record
+
+ This method updates the instance's corresponding API record to match
+ the instance's state. If this instance does not have a corresponding API
+ record yet, raises `AssertionError`. (To create a new API record, use
+ `Collection.save_new`.) This method returns the saved collection
+ manifest.
+ Arguments:
+
+ * properties: dict[str, Any] | None --- If provided, the API record will
+ be updated with these properties. Note this will completely replace
+ any existing properties.
+
+ * storage_classes: list[str] | None --- If provided, the API record will
+ be updated with this value in the `storage_classes_desired` field.
+ This value will also be saved on the instance and used for any
+ changes that follow.
+
+ * trash_at: datetime.datetime | None --- If provided, the API record
+ will be updated with this value in the `trash_at` field.
+
+ * merge: bool --- If `True` (the default), this method will first
+ reload this collection's API record, and merge any new contents into
+ this instance before saving changes. See `Collection.update` for
+ details.
+
+ * num_retries: int | None --- The number of times to retry reloading
+ the collection's API record from the API server. If not specified,
+ uses the `num_retries` provided when this instance was constructed.
+
+ * preserve_version: bool --- This value will be passed to directly
+ to the underlying API call. If `True`, the Arvados API will
+ preserve the versions of this collection both immediately before
+ and after the update. If `True` when the API server is not
+ configured with collection versioning, this method raises
+ `arvados.errors.ArgumentError`.
"""
if properties and type(properties) is not dict:
raise errors.ArgumentError("properties must be dictionary type.")
@@ -1643,60 +1551,66 @@ class Collection(RichCollectionBase):
@must_be_writable
@synchronized
@retry_method
- def save_new(self, name=None,
- create_collection_record=True,
- owner_uuid=None,
- properties=None,
- storage_classes=None,
- trash_at=None,
- ensure_unique_name=False,
- num_retries=None,
- preserve_version=False):
- """Save collection to a new collection record.
-
- Commit pending buffer blocks to Keep and, when create_collection_record
- is True (default), create a new collection record. After creating a
- new collection record, this Collection object will be associated with
- the new record used by `save()`. Returns the current manifest text.
-
- :name:
- The collection name.
-
- :create_collection_record:
- If True, create a collection record on the API server.
- If False, only commit blocks to Keep and return the manifest text.
-
- :owner_uuid:
- the user, or project uuid that will own this collection.
- If None, defaults to the current user.
-
- :properties:
- Additional properties of collection. This value will replace any existing
- properties of collection.
-
- :storage_classes:
- Specify desirable storage classes to be used when writing data to Keep.
-
- :trash_at:
- A collection is *expiring* when it has a *trash_at* time in the future.
- An expiring collection can be accessed as normal,
- but is scheduled to be trashed automatically at the *trash_at* time.
-
- :ensure_unique_name:
- If True, ask the API server to rename the collection
- if it conflicts with a collection with the same name and owner. If
- False, a name conflict will result in an error.
-
- :num_retries:
- Retry count on API calls (if None, use the collection default)
-
- :preserve_version:
- If True, indicate that the collection content being saved right now
- should be preserved in a version snapshot if the collection record is
- updated in the future. Requires that the API server has
- Collections.CollectionVersioning enabled, if not, setting this will
- raise an exception.
+ def save_new(
+ self,
+ name: Optional[str]=None,
+ create_collection_record: bool=True,
+ owner_uuid: Optional[str]=None,
+ properties: Optional[Properties]=None,
+ storage_classes: Optional[StorageClasses]=None,
+ trash_at: Optional[datetime.datetime]=None,
+ ensure_unique_name: bool=False,
+ num_retries: Optional[int]=None,
+ preserve_version: bool=False,
+ ):
+ """Save collection to a new API record
+
+ This method finishes uploading new data blocks and (optionally)
+ creates a new API collection record with the provided data. If a new
+ record is created, this instance becomes associated with that record
+ for future updates like `save()`. This method returns the saved
+ collection manifest.
+
+ Arguments:
+
+ * name: str | None --- The `name` field to use on the new collection
+ record. If not specified, a generic default name is generated.
+
+ * create_collection_record: bool --- If `True` (the default), creates a
+ collection record on the API server. If `False`, the method finishes
+ all data uploads and only returns the resulting collection manifest
+ without sending it to the API server.
+
+ * owner_uuid: str | None --- The `owner_uuid` field to use on the
+ new collection record.
+ * properties: dict[str, Any] | None --- The `properties` field to use on
+ the new collection record.
+
+ * storage_classes: list[str] | None --- The
+ `storage_classes_desired` field to use on the new collection record.
+
+ * trash_at: datetime.datetime | None --- The `trash_at` field to use
+ on the new collection record.
+
+ * ensure_unique_name: bool --- This value is passed directly to the
+ Arvados API when creating the collection record. If `True`, the API
+ server may modify the submitted `name` to ensure the collection's
+ `name`+`owner_uuid` combination is unique. If `False` (the default),
+ if a collection already exists with this same `name`+`owner_uuid`
+ combination, creating a collection record will raise a validation
+ error.
+
+ * num_retries: int | None --- The number of times to retry reloading
+ the collection's API record from the API server. If not specified,
+ uses the `num_retries` provided when this instance was constructed.
+
+ * preserve_version: bool --- This value will be passed to directly
+ to the underlying API call. If `True`, the Arvados API will
+ preserve the versions of this collection both immediately before
+ and after the update. If `True` when the API server is not
+ configured with collection versioning, this method raises
+ `arvados.errors.ArgumentError`.
"""
if properties and type(properties) is not dict:
raise errors.ArgumentError("properties must be dictionary type.")
@@ -1834,17 +1748,24 @@ class Collection(RichCollectionBase):
self.set_committed(True)
@synchronized
- def notify(self, event, collection, name, item):
+ def notify(
+ self,
+ event: ChangeType,
+ collection: 'RichCollectionBase',
+ name: str,
+ item: CollectionItem,
+ ) -> None:
if self._callback:
self._callback(event, collection, name, item)
class Subcollection(RichCollectionBase):
- """This is a subdirectory within a collection that doesn't have its own API
- server record.
-
- Subcollection locking falls under the umbrella lock of its root collection.
+ """Read and manipulate a stream/directory within an Arvados collection
+ This class represents a single stream (like a directory) within an Arvados
+ `Collection`. It is returned by `Collection.find` and provides the same API.
+ Operations that work on the API collection record propagate to the parent
+ `Collection` object.
"""
def __init__(self, parent, name):
@@ -1854,10 +1775,10 @@ class Subcollection(RichCollectionBase):
self.name = name
self.num_retries = parent.num_retries
- def root_collection(self):
+ def root_collection(self) -> 'Collection':
return self.parent.root_collection()
- def writable(self):
+ def writable(self) -> bool:
return self.root_collection().writable()
def _my_api(self):
@@ -1869,11 +1790,15 @@ class Subcollection(RichCollectionBase):
def _my_block_manager(self):
return self.root_collection()._my_block_manager()
- def stream_name(self):
+ def stream_name(self) -> str:
return os.path.join(self.parent.stream_name(), self.name)
@synchronized
- def clone(self, new_parent, new_name):
+ def clone(
+ self,
+ new_parent: Optional['Collection']=None,
+ new_name: Optional[str]=None,
+ ) -> 'Subcollection':
c = Subcollection(new_parent, new_name)
c._clonefrom(self)
return c
@@ -1900,11 +1825,11 @@ class Subcollection(RichCollectionBase):
class CollectionReader(Collection):
- """A read-only collection object.
-
- Initialize from a collection UUID or portable data hash, or raw
- manifest text. See `Collection` constructor for detailed options.
+ """Read-only `Collection` subclass
+ This class will never create or update any API collection records. You can
+ use this class for additional code safety when you only need to read
+ existing collections.
"""
def __init__(self, manifest_locator_or_text, *args, **kwargs):
self._in_init = True
@@ -1918,7 +1843,7 @@ class CollectionReader(Collection):
# all_streams() and all_files()
self._streams = None
- def writable(self):
+ def writable(self) -> bool:
return self._in_init
def _populate_streams(orig_func):
@@ -1935,16 +1860,10 @@ class CollectionReader(Collection):
return orig_func(self, *args, **kwargs)
return populate_streams_wrapper
+ @arvados.util._deprecated('3.0', 'Collection iteration')
@_populate_streams
def normalize(self):
- """Normalize the streams returned by `all_streams`.
-
- This method is kept for backwards compatability and only affects the
- behavior of `all_streams()` and `all_files()`
-
- """
-
- # Rearrange streams
+ """Normalize the streams returned by `all_streams`"""
streams = {}
for s in self.all_streams():
for f in s.all_files():
@@ -1971,3 +1890,423 @@ class CollectionReader(Collection):
for s in self.all_streams():
for f in s.all_files():
yield f
+
+
+class CollectionWriter(CollectionBase):
+ """Create a new collection from scratch
+
+ .. WARNING:: Deprecated
+ This class is deprecated. Prefer `arvados.collection.Collection`
+ instead.
+ """
+
+ @arvados.util._deprecated('3.0', 'arvados.collection.Collection')
+ def __init__(self, api_client=None, num_retries=0, replication=None):
+ """Instantiate a CollectionWriter.
+
+ CollectionWriter lets you build a new Arvados Collection from scratch.
+ Write files to it. The CollectionWriter will upload data to Keep as
+ appropriate, and provide you with the Collection manifest text when
+ you're finished.
+
+ Arguments:
+ * api_client: The API client to use to look up Collections. If not
+ provided, CollectionReader will build one from available Arvados
+ configuration.
+ * num_retries: The default number of times to retry failed
+ service requests. Default 0. You may change this value
+ after instantiation, but note those changes may not
+ propagate to related objects like the Keep client.
+ * replication: The number of copies of each block to store.
+ If this argument is None or not supplied, replication is
+ the server-provided default if available, otherwise 2.
+ """
+ self._api_client = api_client
+ self.num_retries = num_retries
+ self.replication = (2 if replication is None else replication)
+ self._keep_client = None
+ self._data_buffer = []
+ self._data_buffer_len = 0
+ self._current_stream_files = []
+ self._current_stream_length = 0
+ self._current_stream_locators = []
+ self._current_stream_name = '.'
+ self._current_file_name = None
+ self._current_file_pos = 0
+ self._finished_streams = []
+ self._close_file = None
+ self._queued_file = None
+ self._queued_dirents = deque()
+ self._queued_trees = deque()
+ self._last_open = None
+
+ def __exit__(self, exc_type, exc_value, traceback):
+ if exc_type is None:
+ self.finish()
+
+ def do_queued_work(self):
+ # The work queue consists of three pieces:
+ # * _queued_file: The file object we're currently writing to the
+ # Collection.
+ # * _queued_dirents: Entries under the current directory
+ # (_queued_trees[0]) that we want to write or recurse through.
+ # This may contain files from subdirectories if
+ # max_manifest_depth == 0 for this directory.
+ # * _queued_trees: Directories that should be written as separate
+ # streams to the Collection.
+ # This function handles the smallest piece of work currently queued
+ # (current file, then current directory, then next directory) until
+ # no work remains. The _work_THING methods each do a unit of work on
+ # THING. _queue_THING methods add a THING to the work queue.
+ while True:
+ if self._queued_file:
+ self._work_file()
+ elif self._queued_dirents:
+ self._work_dirents()
+ elif self._queued_trees:
+ self._work_trees()
+ else:
+ break
+
+ def _work_file(self):
+ while True:
+ buf = self._queued_file.read(config.KEEP_BLOCK_SIZE)
+ if not buf:
+ break
+ self.write(buf)
+ self.finish_current_file()
+ if self._close_file:
+ self._queued_file.close()
+ self._close_file = None
+ self._queued_file = None
+
+ def _work_dirents(self):
+ path, stream_name, max_manifest_depth = self._queued_trees[0]
+ if stream_name != self.current_stream_name():
+ self.start_new_stream(stream_name)
+ while self._queued_dirents:
+ dirent = self._queued_dirents.popleft()
+ target = os.path.join(path, dirent)
+ if os.path.isdir(target):
+ self._queue_tree(target,
+ os.path.join(stream_name, dirent),
+ max_manifest_depth - 1)
+ else:
+ self._queue_file(target, dirent)
+ break
+ if not self._queued_dirents:
+ self._queued_trees.popleft()
+
+ def _work_trees(self):
+ path, stream_name, max_manifest_depth = self._queued_trees[0]
+ d = arvados.util.listdir_recursive(
+ path, max_depth = (None if max_manifest_depth == 0 else 0))
+ if d:
+ self._queue_dirents(stream_name, d)
+ else:
+ self._queued_trees.popleft()
+
+ def _queue_file(self, source, filename=None):
+ assert (self._queued_file is None), "tried to queue more than one file"
+ if not hasattr(source, 'read'):
+ source = open(source, 'rb')
+ self._close_file = True
+ else:
+ self._close_file = False
+ if filename is None:
+ filename = os.path.basename(source.name)
+ self.start_new_file(filename)
+ self._queued_file = source
+
+ def _queue_dirents(self, stream_name, dirents):
+ assert (not self._queued_dirents), "tried to queue more than one tree"
+ self._queued_dirents = deque(sorted(dirents))
+
+ def _queue_tree(self, path, stream_name, max_manifest_depth):
+ self._queued_trees.append((path, stream_name, max_manifest_depth))
+
+ def write_file(self, source, filename=None):
+ self._queue_file(source, filename)
+ self.do_queued_work()
+
+ def write_directory_tree(self,
+ path, stream_name='.', max_manifest_depth=-1):
+ self._queue_tree(path, stream_name, max_manifest_depth)
+ self.do_queued_work()
+
+ def write(self, newdata):
+ if isinstance(newdata, bytes):
+ pass
+ elif isinstance(newdata, str):
+ newdata = newdata.encode()
+ elif hasattr(newdata, '__iter__'):
+ for s in newdata:
+ self.write(s)
+ return
+ self._data_buffer.append(newdata)
+ self._data_buffer_len += len(newdata)
+ self._current_stream_length += len(newdata)
+ while self._data_buffer_len >= config.KEEP_BLOCK_SIZE:
+ self.flush_data()
+
+ def open(self, streampath, filename=None):
+ """open(streampath[, filename]) -> file-like object
+
+ Pass in the path of a file to write to the Collection, either as a
+ single string or as two separate stream name and file name arguments.
+ This method returns a file-like object you can write to add it to the
+ Collection.
+
+ You may only have one file object from the Collection open at a time,
+ so be sure to close the object when you're done. Using the object in
+ a with statement makes that easy:
+
+ with cwriter.open('./doc/page1.txt') as outfile:
+ outfile.write(page1_data)
+ with cwriter.open('./doc/page2.txt') as outfile:
+ outfile.write(page2_data)
+ """
+ if filename is None:
+ streampath, filename = split(streampath)
+ if self._last_open and not self._last_open.closed:
+ raise errors.AssertionError(
+ u"can't open '{}' when '{}' is still open".format(
+ filename, self._last_open.name))
+ if streampath != self.current_stream_name():
+ self.start_new_stream(streampath)
+ self.set_current_file_name(filename)
+ self._last_open = _WriterFile(self, filename)
+ return self._last_open
+
+ def flush_data(self):
+ data_buffer = b''.join(self._data_buffer)
+ if data_buffer:
+ self._current_stream_locators.append(
+ self._my_keep().put(
+ data_buffer[0:config.KEEP_BLOCK_SIZE],
+ copies=self.replication))
+ self._data_buffer = [data_buffer[config.KEEP_BLOCK_SIZE:]]
+ self._data_buffer_len = len(self._data_buffer[0])
+
+ def start_new_file(self, newfilename=None):
+ self.finish_current_file()
+ self.set_current_file_name(newfilename)
+
+ def set_current_file_name(self, newfilename):
+ if re.search(r'[\t\n]', newfilename):
+ raise errors.AssertionError(
+ "Manifest filenames cannot contain whitespace: %s" %
+ newfilename)
+ elif re.search(r'\x00', newfilename):
+ raise errors.AssertionError(
+ "Manifest filenames cannot contain NUL characters: %s" %
+ newfilename)
+ self._current_file_name = newfilename
+
+ def current_file_name(self):
+ return self._current_file_name
+
+ def finish_current_file(self):
+ if self._current_file_name is None:
+ if self._current_file_pos == self._current_stream_length:
+ return
+ raise errors.AssertionError(
+ "Cannot finish an unnamed file " +
+ "(%d bytes at offset %d in '%s' stream)" %
+ (self._current_stream_length - self._current_file_pos,
+ self._current_file_pos,
+ self._current_stream_name))
+ self._current_stream_files.append([
+ self._current_file_pos,
+ self._current_stream_length - self._current_file_pos,
+ self._current_file_name])
+ self._current_file_pos = self._current_stream_length
+ self._current_file_name = None
+
+ def start_new_stream(self, newstreamname='.'):
+ self.finish_current_stream()
+ self.set_current_stream_name(newstreamname)
+
+ def set_current_stream_name(self, newstreamname):
+ if re.search(r'[\t\n]', newstreamname):
+ raise errors.AssertionError(
+ "Manifest stream names cannot contain whitespace: '%s'" %
+ (newstreamname))
+ self._current_stream_name = '.' if newstreamname=='' else newstreamname
+
+ def current_stream_name(self):
+ return self._current_stream_name
+
+ def finish_current_stream(self):
+ self.finish_current_file()
+ self.flush_data()
+ if not self._current_stream_files:
+ pass
+ elif self._current_stream_name is None:
+ raise errors.AssertionError(
+ "Cannot finish an unnamed stream (%d bytes in %d files)" %
+ (self._current_stream_length, len(self._current_stream_files)))
+ else:
+ if not self._current_stream_locators:
+ self._current_stream_locators.append(config.EMPTY_BLOCK_LOCATOR)
+ self._finished_streams.append([self._current_stream_name,
+ self._current_stream_locators,
+ self._current_stream_files])
+ self._current_stream_files = []
+ self._current_stream_length = 0
+ self._current_stream_locators = []
+ self._current_stream_name = None
+ self._current_file_pos = 0
+ self._current_file_name = None
+
+ def finish(self):
+ """Store the manifest in Keep and return its locator.
+
+ This is useful for storing manifest fragments (task outputs)
+ temporarily in Keep during a Crunch job.
+
+ In other cases you should make a collection instead, by
+ sending manifest_text() to the API server's "create
+ collection" endpoint.
+ """
+ return self._my_keep().put(self.manifest_text().encode(),
+ copies=self.replication)
+
+ def portable_data_hash(self):
+ stripped = self.stripped_manifest().encode()
+ return '{}+{}'.format(hashlib.md5(stripped).hexdigest(), len(stripped))
+
+ def manifest_text(self):
+ self.finish_current_stream()
+ manifest = ''
+
+ for stream in self._finished_streams:
+ if not re.search(r'^\.(/.*)?$', stream[0]):
+ manifest += './'
+ manifest += stream[0].replace(' ', '\\040')
+ manifest += ' ' + ' '.join(stream[1])
+ manifest += ' ' + ' '.join("%d:%d:%s" % (sfile[0], sfile[1], sfile[2].replace(' ', '\\040')) for sfile in stream[2])
+ manifest += "\n"
+
+ return manifest
+
+ def data_locators(self):
+ ret = []
+ for name, locators, files in self._finished_streams:
+ ret += locators
+ return ret
+
+ def save_new(self, name=None):
+ return self._api_client.collections().create(
+ ensure_unique_name=True,
+ body={
+ 'name': name,
+ 'manifest_text': self.manifest_text(),
+ }).execute(num_retries=self.num_retries)
+
+
+class ResumableCollectionWriter(CollectionWriter):
+ """CollectionWriter that can serialize internal state to disk
+
+ .. WARNING:: Deprecated
+ This class is deprecated. Prefer `arvados.collection.Collection`
+ instead.
+ """
+
+ STATE_PROPS = ['_current_stream_files', '_current_stream_length',
+ '_current_stream_locators', '_current_stream_name',
+ '_current_file_name', '_current_file_pos', '_close_file',
+ '_data_buffer', '_dependencies', '_finished_streams',
+ '_queued_dirents', '_queued_trees']
+
+ @arvados.util._deprecated('3.0', 'arvados.collection.Collection')
+ def __init__(self, api_client=None, **kwargs):
+ self._dependencies = {}
+ super(ResumableCollectionWriter, self).__init__(api_client, **kwargs)
+
+ @classmethod
+ def from_state(cls, state, *init_args, **init_kwargs):
+ # Try to build a new writer from scratch with the given state.
+ # If the state is not suitable to resume (because files have changed,
+ # been deleted, aren't predictable, etc.), raise a
+ # StaleWriterStateError. Otherwise, return the initialized writer.
+ # The caller is responsible for calling writer.do_queued_work()
+ # appropriately after it's returned.
+ writer = cls(*init_args, **init_kwargs)
+ for attr_name in cls.STATE_PROPS:
+ attr_value = state[attr_name]
+ attr_class = getattr(writer, attr_name).__class__
+ # Coerce the value into the same type as the initial value, if
+ # needed.
+ if attr_class not in (type(None), attr_value.__class__):
+ attr_value = attr_class(attr_value)
+ setattr(writer, attr_name, attr_value)
+ # Check dependencies before we try to resume anything.
+ if any(KeepLocator(ls).permission_expired()
+ for ls in writer._current_stream_locators):
+ raise errors.StaleWriterStateError(
+ "locators include expired permission hint")
+ writer.check_dependencies()
+ if state['_current_file'] is not None:
+ path, pos = state['_current_file']
+ try:
+ writer._queued_file = open(path, 'rb')
+ writer._queued_file.seek(pos)
+ except IOError as error:
+ raise errors.StaleWriterStateError(
+ u"failed to reopen active file {}: {}".format(path, error))
+ return writer
+
+ def check_dependencies(self):
+ for path, orig_stat in listitems(self._dependencies):
+ if not S_ISREG(orig_stat[ST_MODE]):
+ raise errors.StaleWriterStateError(u"{} not file".format(path))
+ try:
+ now_stat = tuple(os.stat(path))
+ except OSError as error:
+ raise errors.StaleWriterStateError(
+ u"failed to stat {}: {}".format(path, error))
+ if ((not S_ISREG(now_stat[ST_MODE])) or
+ (orig_stat[ST_MTIME] != now_stat[ST_MTIME]) or
+ (orig_stat[ST_SIZE] != now_stat[ST_SIZE])):
+ raise errors.StaleWriterStateError(u"{} changed".format(path))
+
+ def dump_state(self, copy_func=lambda x: x):
+ state = {attr: copy_func(getattr(self, attr))
+ for attr in self.STATE_PROPS}
+ if self._queued_file is None:
+ state['_current_file'] = None
+ else:
+ state['_current_file'] = (os.path.realpath(self._queued_file.name),
+ self._queued_file.tell())
+ return state
+
+ def _queue_file(self, source, filename=None):
+ try:
+ src_path = os.path.realpath(source)
+ except Exception:
+ raise errors.AssertionError(u"{} not a file path".format(source))
+ try:
+ path_stat = os.stat(src_path)
+ except OSError as stat_error:
+ path_stat = None
+ super(ResumableCollectionWriter, self)._queue_file(source, filename)
+ fd_stat = os.fstat(self._queued_file.fileno())
+ if not S_ISREG(fd_stat.st_mode):
+ # We won't be able to resume from this cache anyway, so don't
+ # worry about further checks.
+ self._dependencies[source] = tuple(fd_stat)
+ elif path_stat is None:
+ raise errors.AssertionError(
+ u"could not stat {}: {}".format(source, stat_error))
+ elif path_stat.st_ino != fd_stat.st_ino:
+ raise errors.AssertionError(
+ u"{} changed between open and stat calls".format(source))
+ else:
+ self._dependencies[src_path] = tuple(fd_stat)
+
+ def write(self, data):
+ if self._queued_file is None:
+ raise errors.AssertionError(
+ "resumable writer can't accept unsourced data")
+ return super(ResumableCollectionWriter, self).write(data)
commit 1fbd0ad57137ef23a6ec957746c05127ab8cc8c5
Author: Brett Smith <brett.smith at curii.com>
Date: Wed Nov 29 08:54:10 2023 -0500
Merge branch '21211-pysdk-annotations'
Closes #21211.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index ca9f17f866..8a17e42fcb 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -9,12 +9,7 @@ niceties such as caching, X-Request-Id header for tracking, and more. The main
client constructors are `api` and `api_from_config`.
"""
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
import collections
-import http.client
import httplib2
import json
import logging
@@ -28,6 +23,14 @@ import threading
import time
import types
+from typing import (
+ Any,
+ Dict,
+ List,
+ Mapping,
+ Optional,
+)
+
import apiclient
import apiclient.http
from apiclient import discovery as apiclient_discovery
@@ -152,7 +155,7 @@ def _new_http_error(cls, *args, **kwargs):
errors.ApiError, *args, **kwargs)
apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
-def http_cache(data_type):
+def http_cache(data_type: str) -> cache.SafeHTTPCache:
"""Set up an HTTP file cache
This function constructs and returns an `arvados.cache.SafeHTTPCache`
@@ -177,18 +180,18 @@ def http_cache(data_type):
return cache.SafeHTTPCache(str(path), max_age=60*60*24*2)
def api_client(
- version,
- discoveryServiceUrl,
- token,
+ version: str,
+ discoveryServiceUrl: str,
+ token: str,
*,
- cache=True,
- http=None,
- insecure=False,
- num_retries=10,
- request_id=None,
- timeout=5*60,
- **kwargs,
-):
+ cache: bool=True,
+ http: Optional[httplib2.Http]=None,
+ insecure: bool=False,
+ num_retries: int=10,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ **kwargs: Any,
+) -> apiclient_discovery.Resource:
"""Build an Arvados API client
This function returns a `googleapiclient.discovery.Resource` object
@@ -232,7 +235,6 @@ def api_client(
Additional keyword arguments will be passed directly to
`googleapiclient.discovery.build`.
-
"""
if http is None:
http = httplib2.Http(
@@ -294,12 +296,12 @@ def api_client(
return svc
def normalize_api_kwargs(
- version=None,
- discoveryServiceUrl=None,
- host=None,
- token=None,
- **kwargs,
-):
+ version: Optional[str]=None,
+ discoveryServiceUrl: Optional[str]=None,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ **kwargs: Any,
+) -> Dict[str, Any]:
"""Validate kwargs from `api` and build kwargs for `api_client`
This method takes high-level keyword arguments passed to the `api`
@@ -352,7 +354,11 @@ def normalize_api_kwargs(
**kwargs,
}
-def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
+def api_kwargs_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> Dict[str, Any]:
"""Build `api_client` keyword arguments from configuration
This function accepts a mapping with Arvados configuration settings like
@@ -395,9 +401,18 @@ def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
**kwargs,
)
-def api(version=None, cache=True, host=None, token=None, insecure=False,
- request_id=None, timeout=5*60, *,
- discoveryServiceUrl=None, **kwargs):
+def api(
+ version: Optional[str]=None,
+ cache: bool=True,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ insecure: bool=False,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ *,
+ discoveryServiceUrl: Optional[str]=None,
+ **kwargs: Any,
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Dynamically build an Arvados API client
This function provides a high-level "do what I mean" interface to build an
@@ -449,7 +464,11 @@ def api(version=None, cache=True, host=None, token=None, insecure=False,
from .safeapi import ThreadSafeApiCache
return ThreadSafeApiCache({}, {}, kwargs, version)
-def api_from_config(version=None, apiconfig=None, **kwargs):
+def api_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Build an Arvados API client from a configuration mapping
This function builds an Arvados API client from a mapping with user
diff --git a/sdk/python/arvados/retry.py b/sdk/python/arvados/retry.py
index ea8a6f65af..e9e574f5df 100644
--- a/sdk/python/arvados/retry.py
+++ b/sdk/python/arvados/retry.py
@@ -15,21 +15,28 @@ It also provides utility functions for common operations with `RetryLoop`:
#
# SPDX-License-Identifier: Apache-2.0
-from builtins import range
-from builtins import object
import functools
import inspect
import pycurl
import time
from collections import deque
+from typing import (
+ Callable,
+ Generic,
+ Optional,
+ TypeVar,
+)
import arvados.errors
_HTTP_SUCCESSES = set(range(200, 300))
_HTTP_CAN_RETRY = set([408, 409, 423, 500, 502, 503, 504])
-class RetryLoop(object):
+CT = TypeVar('CT', bound=Callable)
+T = TypeVar('T')
+
+class RetryLoop(Generic[T]):
"""Coordinate limited retries of code.
`RetryLoop` coordinates a loop that runs until it records a
@@ -53,12 +60,12 @@ class RetryLoop(object):
it doesn't succeed. This means the loop body could run at most
`num_retries + 1` times.
- * success_check: Callable --- This is a function that will be called
- each time the loop saves a result. The function should return `True`
- if the result indicates the code succeeded, `False` if it represents a
- permanent failure, and `None` if it represents a temporary failure.
- If no function is provided, the loop will end after any result is
- saved.
+ * success_check: Callable[[T], bool | None] --- This is a function that
+ will be called each time the loop saves a result. The function should
+ return `True` if the result indicates the code succeeded, `False` if
+ it represents a permanent failure, and `None` if it represents a
+ temporary failure. If no function is provided, the loop will end
+ after any result is saved.
* backoff_start: float --- The number of seconds that must pass before
the loop's second iteration. Default 0, which disables all waiting.
@@ -73,9 +80,15 @@ class RetryLoop(object):
* max_wait: float --- Maximum number of seconds to wait between
retries. Default 60.
"""
- def __init__(self, num_retries, success_check=lambda r: True,
- backoff_start=0, backoff_growth=2, save_results=1,
- max_wait=60):
+ def __init__(
+ self,
+ num_retries: int,
+ success_check: Callable[[T], Optional[bool]]=lambda r: True,
+ backoff_start: float=0,
+ backoff_growth: float=2,
+ save_results: int=1,
+ max_wait: float=60
+ ) -> None:
self.tries_left = num_retries + 1
self.check_result = success_check
self.backoff_wait = backoff_start
@@ -87,11 +100,11 @@ class RetryLoop(object):
self._running = None
self._success = None
- def __iter__(self):
+ def __iter__(self) -> 'RetryLoop':
"""Return an iterator of retries."""
return self
- def running(self):
+ def running(self) -> Optional[bool]:
"""Return whether this loop is running.
Returns `None` if the loop has never run, `True` if it is still running,
@@ -100,7 +113,7 @@ class RetryLoop(object):
"""
return self._running and (self._success is None)
- def __next__(self):
+ def __next__(self) -> int:
"""Record a loop attempt.
If the loop is still running, decrements the number of tries left and
@@ -121,7 +134,7 @@ class RetryLoop(object):
self.tries_left -= 1
return self.tries_left
- def save_result(self, result):
+ def save_result(self, result: T) -> None:
"""Record a loop result.
Save the given result, and end the loop if it indicates
@@ -133,8 +146,7 @@ class RetryLoop(object):
Arguments:
- * result: Any --- The result from this loop attempt to check and
- save.
+ * result: T --- The result from this loop attempt to check and save.
"""
if not self.running():
raise arvados.errors.AssertionError(
@@ -143,7 +155,7 @@ class RetryLoop(object):
self._success = self.check_result(result)
self._attempts += 1
- def success(self):
+ def success(self) -> Optional[bool]:
"""Return the loop's end state.
Returns `True` if the loop recorded a successful result, `False` if it
@@ -151,7 +163,7 @@ class RetryLoop(object):
"""
return self._success
- def last_result(self):
+ def last_result(self) -> T:
"""Return the most recent result the loop saved.
Raises `arvados.errors.AssertionError` if called before any result has
@@ -163,7 +175,7 @@ class RetryLoop(object):
raise arvados.errors.AssertionError(
"queried loop results before any were recorded")
- def attempts(self):
+ def attempts(self) -> int:
"""Return the number of results that have been saved.
This count includes all kinds of results: success, permanent failure,
@@ -171,7 +183,7 @@ class RetryLoop(object):
"""
return self._attempts
- def attempts_str(self):
+ def attempts_str(self) -> str:
"""Return a human-friendly string counting saved results.
This method returns '1 attempt' or 'N attempts', where the number
@@ -183,7 +195,7 @@ class RetryLoop(object):
return '{} attempts'.format(self._attempts)
-def check_http_response_success(status_code):
+def check_http_response_success(status_code: int) -> Optional[bool]:
"""Convert a numeric HTTP status code to a loop control flag.
This method takes a numeric HTTP status code and returns `True` if
@@ -213,7 +225,7 @@ def check_http_response_success(status_code):
else:
return None # Get well soon, server.
-def retry_method(orig_func):
+def retry_method(orig_func: CT) -> CT:
"""Provide a default value for a method's num_retries argument.
This is a decorator for instance and class methods that accept a
diff --git a/sdk/python/arvados/safeapi.py b/sdk/python/arvados/safeapi.py
index 3ecc72a950..56b92e8f08 100644
--- a/sdk/python/arvados/safeapi.py
+++ b/sdk/python/arvados/safeapi.py
@@ -7,12 +7,15 @@ This module provides `ThreadSafeApiCache`, a thread-safe, API-compatible
Arvados API client.
"""
-from __future__ import absolute_import
-
-from builtins import object
import sys
import threading
+from typing import (
+ Any,
+ Mapping,
+ Optional,
+)
+
from . import config
from . import keep
from . import util
@@ -30,27 +33,31 @@ class ThreadSafeApiCache(object):
Arguments:
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`,
- and optionally `ARVADOS_API_HOST_INSECURE`. If not provided, uses
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, uses
`arvados.config.settings` to get these parameters from user
configuration. You can pass an empty mapping to build the client
solely from `api_params`.
- keep_params: Mapping[str, Any]
- : Keyword arguments used to construct an associated
- `arvados.keep.KeepClient`.
+ * keep_params: Mapping[str, Any] --- Keyword arguments used to construct
+ an associated `arvados.keep.KeepClient`.
- api_params: Mapping[str, Any]
- : Keyword arguments used to construct each thread's API client. These
- have the same meaning as in the `arvados.api.api` function.
+ * api_params: Mapping[str, Any] --- Keyword arguments used to construct
+ each thread's API client. These have the same meaning as in the
+ `arvados.api.api` function.
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ `'v1'`.
"""
-
- def __init__(self, apiconfig=None, keep_params={}, api_params={}, version=None):
+ def __init__(
+ self,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ keep_params: Optional[Mapping[str, Any]]={},
+ api_params: Optional[Mapping[str, Any]]={},
+ version: Optional[str]=None,
+ ) -> None:
if apiconfig or apiconfig is None:
self._api_kwargs = api.api_kwargs_from_config(version, apiconfig, **api_params)
else:
@@ -60,7 +67,7 @@ class ThreadSafeApiCache(object):
self.local = threading.local()
self.keep = keep.KeepClient(api_client=self, **keep_params)
- def localapi(self):
+ def localapi(self) -> 'googleapiclient.discovery.Resource':
try:
client = self.local.api
except AttributeError:
@@ -69,6 +76,6 @@ class ThreadSafeApiCache(object):
self.local.api = client
return client
- def __getattr__(self, name):
+ def __getattr__(self, name: str) -> Any:
# Proxy nonexistent attributes to the thread-local API client.
return getattr(self.localapi(), name)
commit 3e5c9648ad4e9d5d9c320558b30a226a4702343c
Author: Brett Smith <brett.smith at curii.com>
Date: Wed Nov 29 08:54:46 2023 -0500
Merge branch '19830-pysdk-util-docs'
Closes #19830.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 21ca72c4bd..e90f381298 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -1,32 +1,30 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Arvados Python SDK
+
+This module provides the entire Python SDK for Arvados. The most useful modules
+include:
+
+* arvados.api - After you `import arvados`, you can call `arvados.api.api` as
+ `arvados.api` to construct a client object.
+
+* arvados.collection - The `arvados.collection.Collection` class provides a
+ high-level interface to read and write collections. It coordinates sending
+ data to and from Keep, and synchronizing updates with the collection object.
+
+* arvados.util - Utility functions to use mostly in conjunction with the API
+ client object and the results it returns.
+
+Other submodules provide lower-level functionality.
+"""
-from __future__ import print_function
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import object
-import bz2
-import fcntl
-import hashlib
-import http.client
-import httplib2
-import json
import logging as stdliblog
import os
-import pprint
-import re
-import string
import sys
-import time
import types
-import zlib
-if sys.version_info >= (3, 0):
- from collections import UserDict
-else:
- from UserDict import UserDict
+from collections import UserDict
from .api import api, api_from_config, http_cache
from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 88adc8879b..050c67f68d 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -1,10 +1,13 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Arvados utilities
-from __future__ import division
-from builtins import range
+This module provides functions and constants that are useful across a variety
+of Arvados resource types, or extend the Arvados API client (see `arvados.api`).
+"""
+import errno
import fcntl
import functools
import hashlib
@@ -13,30 +16,63 @@ import os
import random
import re
import subprocess
-import errno
import sys
import warnings
import arvados.errors
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterator,
+ TypeVar,
+ Union,
+)
+
+T = TypeVar('T')
+
HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
+"""Regular expression to match a hexadecimal string (case-insensitive)"""
CR_UNCOMMITTED = 'Uncommitted'
+"""Constant `state` value for uncommited container requests"""
CR_COMMITTED = 'Committed'
+"""Constant `state` value for committed container requests"""
CR_FINAL = 'Final'
+"""Constant `state` value for finalized container requests"""
keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*')
+"""Regular expression to match any Keep block locator"""
signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*\+A\S+(\+\S+)*')
+"""Regular expression to match any Keep block locator with an access token hint"""
portable_data_hash_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+')
+"""Regular expression to match any collection portable data hash"""
+manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
+"""Regular expression to match an Arvados collection manifest text"""
+keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a file path from a collection identified by portable data hash"""
+keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a `keep:` URI with a collection identified by portable data hash"""
+
uuid_pattern = re.compile(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}')
+"""Regular expression to match any Arvados object UUID"""
collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
+"""Regular expression to match any Arvados collection UUID"""
+container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
+"""Regular expression to match any Arvados container UUID"""
group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
-user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados group UUID"""
link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
+"""Regular expression to match any Arvados link UUID"""
+user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados user UUID"""
job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
-container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
-manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
-keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
-keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match any Arvados job UUID
+
+.. WARNING:: Deprecated
+ Arvados job resources are deprecated and will be removed in a future
+ release. Prefer the containers API instead.
+"""
def _deprecated(version=None, preferred=None):
"""Mark a callable as deprecated in the SDK
@@ -47,12 +83,11 @@ def _deprecated(version=None, preferred=None):
If the following arguments are given, they'll be included in the
notices:
- preferred: str | None
- : The name of an alternative that users should use instead.
+ * preferred: str | None --- The name of an alternative that users should
+ use instead.
- version: str | None
- : The version of Arvados when the callable is scheduled to be
- removed.
+ * version: str | None --- The version of Arvados when the callable is
+ scheduled to be removed.
"""
if version is None:
version = ''
@@ -91,6 +126,276 @@ def _deprecated(version=None, preferred=None):
return deprecated_wrapper
return deprecated_decorator
+def is_hex(s: str, *length_args: int) -> bool:
+ """Indicate whether a string is a hexadecimal number
+
+ This method returns true if all characters in the string are hexadecimal
+ digits. It is case-insensitive.
+
+ You can also pass optional length arguments to check that the string has
+ the expected number of digits. If you pass one integer, the string must
+ have that length exactly, otherwise the method returns False. If you
+ pass two integers, the string's length must fall within that minimum and
+ maximum (inclusive), otherwise the method returns False.
+
+ Arguments:
+
+ * s: str --- The string to check
+
+ * length_args: int --- Optional length limit(s) for the string to check
+ """
+ num_length_args = len(length_args)
+ if num_length_args > 2:
+ raise arvados.errors.ArgumentError(
+ "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
+ elif num_length_args == 2:
+ good_len = (length_args[0] <= len(s) <= length_args[1])
+ elif num_length_args == 1:
+ good_len = (len(s) == length_args[0])
+ else:
+ good_len = True
+ return bool(good_len and HEX_RE.match(s))
+
+def keyset_list_all(
+ fn: Callable[..., 'arvados.api_resources.ArvadosAPIRequest'],
+ order_key: str="created_at",
+ num_retries: int=0,
+ ascending: bool=True,
+ **kwargs: Any,
+) -> Iterator[Dict[str, Any]]:
+ """Iterate all Arvados resources from an API list call
+
+ This method takes a method that represents an Arvados API list call, and
+ iterates the objects returned by the API server. It can make multiple API
+ calls to retrieve and iterate all objects available from the API server.
+
+ Arguments:
+
+ * fn: Callable[..., arvados.api_resources.ArvadosAPIRequest] --- A
+ function that wraps an Arvados API method that returns a list of
+ objects. If you have an Arvados API client named `arv`, examples
+ include `arv.collections().list` and `arv.groups().contents`. Note
+ that you should pass the function *without* calling it.
+
+ * order_key: str --- The name of the primary object field that objects
+ should be sorted by. This name is used to build an `order` argument
+ for `fn`. Default `'created_at'`.
+
+ * num_retries: int --- This argument is passed through to
+ `arvados.api_resources.ArvadosAPIRequest.execute` for each API call. See
+ that method's docstring for details. Default 0 (meaning API calls will
+ use the `num_retries` value set when the Arvados API client was
+ constructed).
+
+ * ascending: bool --- Used to build an `order` argument for `fn`. If True,
+ all fields will be sorted in `'asc'` (ascending) order. Otherwise, all
+ fields will be sorted in `'desc'` (descending) order.
+
+ Additional keyword arguments will be passed directly to `fn` for each API
+ call. Note that this function sets `count`, `limit`, and `order` as part of
+ its work.
+ """
+ pagesize = 1000
+ kwargs["limit"] = pagesize
+ kwargs["count"] = 'none'
+ asc = "asc" if ascending else "desc"
+ kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
+ other_filters = kwargs.get("filters", [])
+
+ try:
+ select = set(kwargs['select'])
+ except KeyError:
+ pass
+ else:
+ select.add(order_key)
+ select.add('uuid')
+ kwargs['select'] = list(select)
+
+ nextpage = []
+ tot = 0
+ expect_full_page = True
+ seen_prevpage = set()
+ seen_thispage = set()
+ lastitem = None
+ prev_page_all_same_order_key = False
+
+ while True:
+ kwargs["filters"] = nextpage+other_filters
+ items = fn(**kwargs).execute(num_retries=num_retries)
+
+ if len(items["items"]) == 0:
+ if prev_page_all_same_order_key:
+ nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
+ prev_page_all_same_order_key = False
+ continue
+ else:
+ return
+
+ seen_prevpage = seen_thispage
+ seen_thispage = set()
+
+ for i in items["items"]:
+ # In cases where there's more than one record with the
+ # same order key, the result could include records we
+ # already saw in the last page. Skip them.
+ if i["uuid"] in seen_prevpage:
+ continue
+ seen_thispage.add(i["uuid"])
+ yield i
+
+ firstitem = items["items"][0]
+ lastitem = items["items"][-1]
+
+ if firstitem[order_key] == lastitem[order_key]:
+ # Got a page where every item has the same order key.
+ # Switch to using uuid for paging.
+ nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
+ prev_page_all_same_order_key = True
+ else:
+ # Start from the last order key seen, but skip the last
+ # known uuid to avoid retrieving the same row twice. If
+ # there are multiple rows with the same order key it is
+ # still likely we'll end up retrieving duplicate rows.
+ # That's handled by tracking the "seen" rows for each page
+ # so they can be skipped if they show up on the next page.
+ nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
+ prev_page_all_same_order_key = False
+
+def ca_certs_path(fallback: T=httplib2.CA_CERTS) -> Union[str, T]:
+ """Return the path of the best available source of CA certificates
+
+ This function checks various known paths that provide trusted CA
+ certificates, and returns the first one that exists. It checks:
+
+ * the path in the `SSL_CERT_FILE` environment variable (used by OpenSSL)
+ * `/etc/arvados/ca-certificates.crt`, respected by all Arvados software
+ * `/etc/ssl/certs/ca-certificates.crt`, the default store on Debian-based
+ distributions
+ * `/etc/pki/tls/certs/ca-bundle.crt`, the default store on Red Hat-based
+ distributions
+
+ If none of these paths exist, this function returns the value of `fallback`.
+
+ Arguments:
+
+ * fallback: T --- The value to return if none of the known paths exist.
+ The default value is the certificate store of Mozilla's trusted CAs
+ included with the Python [certifi][] package.
+
+ [certifi]: https://pypi.org/project/certifi/
+ """
+ for ca_certs_path in [
+ # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
+ # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
+ os.environ.get('SSL_CERT_FILE'),
+ # Arvados specific:
+ '/etc/arvados/ca-certificates.crt',
+ # Debian:
+ '/etc/ssl/certs/ca-certificates.crt',
+ # Red Hat:
+ '/etc/pki/tls/certs/ca-bundle.crt',
+ ]:
+ if ca_certs_path and os.path.exists(ca_certs_path):
+ return ca_certs_path
+ return fallback
+
+def new_request_id() -> str:
+ """Return a random request ID
+
+ This function generates and returns a random string suitable for use as a
+ `X-Request-Id` header value in the Arvados API.
+ """
+ rid = "req-"
+ # 2**104 > 36**20 > 2**103
+ n = random.getrandbits(104)
+ for _ in range(20):
+ c = n % 36
+ if c < 10:
+ rid += chr(c+ord('0'))
+ else:
+ rid += chr(c+ord('a')-10)
+ n = n // 36
+ return rid
+
+def get_config_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+ """Return an Arvados cluster's configuration, with caching
+
+ This function gets and returns the Arvados configuration from the API
+ server. It caches the result on the client object and reuses it on any
+ future calls.
+
+ Arguments:
+
+ * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+ object to use to retrieve and cache the Arvados cluster configuration.
+ """
+ if not svc._rootDesc.get('resources').get('configs', False):
+ # Old API server version, no config export endpoint
+ return {}
+ if not hasattr(svc, '_cached_config'):
+ svc._cached_config = svc.configs().get().execute()
+ return svc._cached_config
+
+def get_vocabulary_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+ """Return an Arvados cluster's vocabulary, with caching
+
+ This function gets and returns the Arvados vocabulary from the API
+ server. It caches the result on the client object and reuses it on any
+ future calls.
+
+ .. HINT:: Low-level method
+ This is a relatively low-level wrapper around the Arvados API. Most
+ users will prefer to use `arvados.vocabulary.load_vocabulary`.
+
+ Arguments:
+
+ * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+ object to use to retrieve and cache the Arvados cluster vocabulary.
+ """
+ if not svc._rootDesc.get('resources').get('vocabularies', False):
+ # Old API server version, no vocabulary export endpoint
+ return {}
+ if not hasattr(svc, '_cached_vocabulary'):
+ svc._cached_vocabulary = svc.vocabularies().get().execute()
+ return svc._cached_vocabulary
+
+def trim_name(collectionname: str) -> str:
+ """Limit the length of a name to fit within Arvados API limits
+
+ This function ensures that a string is short enough to use as an object
+ name in the Arvados API, leaving room for text that may be added by the
+ `ensure_unique_name` argument. If the source name is short enough, it is
+ returned unchanged. Otherwise, this function returns a string with excess
+ characters removed from the middle of the source string and replaced with
+ an ellipsis.
+
+ Arguments:
+
+ * collectionname: str --- The desired source name
+ """
+ max_name_len = 254 - 28
+
+ if len(collectionname) > max_name_len:
+ over = len(collectionname) - max_name_len
+ split = int(max_name_len/2)
+ collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
+
+ return collectionname
+
+ at _deprecated('3.0', 'arvados.util.keyset_list_all')
+def list_all(fn, num_retries=0, **kwargs):
+ # Default limit to (effectively) api server's MAX_LIMIT
+ kwargs.setdefault('limit', sys.maxsize)
+ items = []
+ offset = 0
+ items_available = sys.maxsize
+ while len(items) < items_available:
+ c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
+ items += c['items']
+ items_available = c['items_available']
+ offset = c['offset'] + len(c['items'])
+ return items
+
@_deprecated('3.0')
def clear_tmpdir(path=None):
"""
@@ -428,174 +733,3 @@ def listdir_recursive(dirname, base=None, max_depth=None):
else:
allfiles += [ent_base]
return allfiles
-
-def is_hex(s, *length_args):
- """is_hex(s[, length[, max_length]]) -> boolean
-
- Return True if s is a string of hexadecimal digits.
- If one length argument is given, the string must contain exactly
- that number of digits.
- If two length arguments are given, the string must contain a number of
- digits between those two lengths, inclusive.
- Return False otherwise.
- """
- num_length_args = len(length_args)
- if num_length_args > 2:
- raise arvados.errors.ArgumentError(
- "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
- elif num_length_args == 2:
- good_len = (length_args[0] <= len(s) <= length_args[1])
- elif num_length_args == 1:
- good_len = (len(s) == length_args[0])
- else:
- good_len = True
- return bool(good_len and HEX_RE.match(s))
-
- at _deprecated('3.0', 'arvados.util.keyset_list_all')
-def list_all(fn, num_retries=0, **kwargs):
- # Default limit to (effectively) api server's MAX_LIMIT
- kwargs.setdefault('limit', sys.maxsize)
- items = []
- offset = 0
- items_available = sys.maxsize
- while len(items) < items_available:
- c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
- items += c['items']
- items_available = c['items_available']
- offset = c['offset'] + len(c['items'])
- return items
-
-def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
- pagesize = 1000
- kwargs["limit"] = pagesize
- kwargs["count"] = 'none'
- asc = "asc" if ascending else "desc"
- kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
- other_filters = kwargs.get("filters", [])
-
- try:
- select = set(kwargs['select'])
- except KeyError:
- pass
- else:
- select.add(order_key)
- select.add('uuid')
- kwargs['select'] = list(select)
-
- nextpage = []
- tot = 0
- expect_full_page = True
- seen_prevpage = set()
- seen_thispage = set()
- lastitem = None
- prev_page_all_same_order_key = False
-
- while True:
- kwargs["filters"] = nextpage+other_filters
- items = fn(**kwargs).execute(num_retries=num_retries)
-
- if len(items["items"]) == 0:
- if prev_page_all_same_order_key:
- nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
- prev_page_all_same_order_key = False
- continue
- else:
- return
-
- seen_prevpage = seen_thispage
- seen_thispage = set()
-
- for i in items["items"]:
- # In cases where there's more than one record with the
- # same order key, the result could include records we
- # already saw in the last page. Skip them.
- if i["uuid"] in seen_prevpage:
- continue
- seen_thispage.add(i["uuid"])
- yield i
-
- firstitem = items["items"][0]
- lastitem = items["items"][-1]
-
- if firstitem[order_key] == lastitem[order_key]:
- # Got a page where every item has the same order key.
- # Switch to using uuid for paging.
- nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
- prev_page_all_same_order_key = True
- else:
- # Start from the last order key seen, but skip the last
- # known uuid to avoid retrieving the same row twice. If
- # there are multiple rows with the same order key it is
- # still likely we'll end up retrieving duplicate rows.
- # That's handled by tracking the "seen" rows for each page
- # so they can be skipped if they show up on the next page.
- nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
- prev_page_all_same_order_key = False
-
-def ca_certs_path(fallback=httplib2.CA_CERTS):
- """Return the path of the best available CA certs source.
-
- This function searches for various distribution sources of CA
- certificates, and returns the first it finds. If it doesn't find any,
- it returns the value of `fallback` (httplib2's CA certs by default).
- """
- for ca_certs_path in [
- # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
- # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
- os.environ.get('SSL_CERT_FILE'),
- # Arvados specific:
- '/etc/arvados/ca-certificates.crt',
- # Debian:
- '/etc/ssl/certs/ca-certificates.crt',
- # Red Hat:
- '/etc/pki/tls/certs/ca-bundle.crt',
- ]:
- if ca_certs_path and os.path.exists(ca_certs_path):
- return ca_certs_path
- return fallback
-
-def new_request_id():
- rid = "req-"
- # 2**104 > 36**20 > 2**103
- n = random.getrandbits(104)
- for _ in range(20):
- c = n % 36
- if c < 10:
- rid += chr(c+ord('0'))
- else:
- rid += chr(c+ord('a')-10)
- n = n // 36
- return rid
-
-def get_config_once(svc):
- if not svc._rootDesc.get('resources').get('configs', False):
- # Old API server version, no config export endpoint
- return {}
- if not hasattr(svc, '_cached_config'):
- svc._cached_config = svc.configs().get().execute()
- return svc._cached_config
-
-def get_vocabulary_once(svc):
- if not svc._rootDesc.get('resources').get('vocabularies', False):
- # Old API server version, no vocabulary export endpoint
- return {}
- if not hasattr(svc, '_cached_vocabulary'):
- svc._cached_vocabulary = svc.vocabularies().get().execute()
- return svc._cached_vocabulary
-
-def trim_name(collectionname):
- """
- trim_name takes a record name (collection name, project name, etc)
- and trims it to fit the 255 character name limit, with additional
- space for the timestamp added by ensure_unique_name, by removing
- excess characters from the middle and inserting an ellipse
- """
-
- max_name_len = 254 - 28
-
- if len(collectionname) > max_name_len:
- over = len(collectionname) - max_name_len
- split = int(max_name_len/2)
- collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
-
- return collectionname
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index eb2784c714..99c5af8a27 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -2,10 +2,6 @@
#
# SPDX-License-Identifier: Apache-2.0
-from __future__ import print_function
-from __future__ import division
-from builtins import str
-from builtins import range
import argparse
import atexit
import errno
@@ -18,7 +14,6 @@ import shlex
import shutil
import signal
import socket
-import string
import subprocess
import sys
import tempfile
@@ -26,10 +21,7 @@ import time
import unittest
import yaml
-try:
- from urllib.parse import urlparse
-except ImportError:
- from urlparse import urlparse
+from urllib.parse import urlparse
MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
if __name__ == '__main__' and os.path.exists(
commit 29aeb9634b700eec2e5866782f08a09450a20dfd
Author: Brett Smith <brett.smith at curii.com>
Date: Tue Nov 21 17:29:13 2023 -0500
Merge branch '21132-api-resources-fixes'
Closes #21132, #21136.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/discovery2pydoc.py b/sdk/python/discovery2pydoc.py
index 6ca3aafeb6..70a51371ac 100755
--- a/sdk/python/discovery2pydoc.py
+++ b/sdk/python/discovery2pydoc.py
@@ -77,12 +77,19 @@ If you work with this raw object, the keys of the dictionary are documented
below, along with their types. The `items` key maps to a list of matching
`{cls_name}` objects.
'''
-_MODULE_PYDOC = '''Arvados API client documentation skeleton
-
-This module documents the methods and return types provided by the Arvados API
-client. Start with `ArvadosAPIClient`, which documents the methods available
-from the API client objects constructed by `arvados.api`. The implementation is
-generated dynamically at runtime when the client object is built.
+_MODULE_PYDOC = '''Arvados API client reference documentation
+
+This module provides reference documentation for the interface of the
+Arvados API client, including method signatures and type information for
+returned objects. However, the functions in `arvados.api` will return
+different classes at runtime that are generated dynamically from the Arvados
+API discovery document. The classes in this module do not have any
+implementation, and you should not instantiate them in your code.
+
+If you're just starting out, `ArvadosAPIClient` documents the methods
+available from the client object. From there, you can follow the trail into
+resource methods, request objects, and finally the data dictionaries returned
+by the API server.
'''
_SCHEMA_PYDOC = '''
@@ -95,25 +102,62 @@ to list the specific keys you need. Refer to the API documentation for details.
'''
_MODULE_PRELUDE = '''
+import googleapiclient.discovery
+import googleapiclient.http
+import httplib2
import sys
+from typing import Any, Dict, Generic, List, Optional, TypeVar
if sys.version_info < (3, 8):
- from typing import Any
from typing_extensions import TypedDict
else:
- from typing import Any, TypedDict
+ from typing import TypedDict
+
+# ST represents an API response type
+ST = TypeVar('ST', bound=TypedDict)
'''
+_REQUEST_CLASS = '''
+class ArvadosAPIRequest(googleapiclient.http.HttpRequest, Generic[ST]):
+ """Generic API request object
+
+ When you call an API method in the Arvados Python SDK, it returns a
+ request object. You usually call `execute()` on this object to submit the
+ request to your Arvados API server and retrieve the response. `execute()`
+ will return the type of object annotated in the subscript of
+ `ArvadosAPIRequest`.
+ """
+
+ def execute(self, http: Optional[httplib2.Http]=None, num_retries: int=0) -> ST:
+ """Execute this request and return the response
+
+ Arguments:
+
+ * http: httplib2.Http | None --- The HTTP client object to use to
+ execute the request. If not specified, uses the HTTP client object
+ created with the API client object.
+
+ * num_retries: int --- The maximum number of times to retry this
+ request if the server returns a retryable failure. The API client
+ object also has a maximum number of retries specified when it is
+ instantiated (see `arvados.api.api_client`). This request is run
+ with the larger of that number and this argument. Default 0.
+ """
-_TYPE_MAP = {
+'''
+
+# Annotation represents a valid Python type annotation. Future development
+# could expand this to include other valid types like `type`.
+Annotation = str
+_TYPE_MAP: Mapping[str, Annotation] = {
# Map the API's JavaScript-based type names to Python annotations.
# Some of these may disappear after Arvados issue #19795 is fixed.
- 'Array': 'list',
- 'array': 'list',
+ 'Array': 'List',
+ 'array': 'List',
'boolean': 'bool',
# datetime fields are strings in ISO 8601 format.
'datetime': 'str',
- 'Hash': 'dict[str, Any]',
+ 'Hash': 'Dict[str, Any]',
'integer': 'int',
- 'object': 'dict[str, Any]',
+ 'object': 'Dict[str, Any]',
'string': 'str',
'text': 'str',
}
@@ -197,9 +241,15 @@ class Parameter(inspect.Parameter):
class Method:
- def __init__(self, name: str, spec: Mapping[str, Any]) -> None:
+ def __init__(
+ self,
+ name: str,
+ spec: Mapping[str, Any],
+ annotate: Callable[[Annotation], Annotation]=str,
+ ) -> None:
self.name = name
self._spec = spec
+ self._annotate = annotate
self._required_params = []
self._optional_params = []
for param_name, param_spec in spec['parameters'].items():
@@ -221,7 +271,8 @@ class Method:
try:
returns = get_type_annotation(self._spec['response']['$ref'])
except KeyError:
- returns = 'dict[str, Any]'
+ returns = 'Dict[str, Any]'
+ returns = self._annotate(returns)
return inspect.Signature(parameters, return_annotation=returns)
def doc(self, doc_slice: slice=slice(None)) -> str:
@@ -285,7 +336,7 @@ def document_resource(name: str, spec: Mapping[str, Any]) -> str:
if class_name in _DEPRECATED_RESOURCES:
docstring += _DEPRECATED_NOTICE
methods = [
- Method(key, meth_spec)
+ Method(key, meth_spec, 'ArvadosAPIRequest[{}]'.format)
for key, meth_spec in spec['methods'].items()
if key not in _ALIASED_METHODS
]
@@ -354,7 +405,11 @@ def main(arglist: Optional[Sequence[str]]=None) -> int:
for name, resource_spec in resources:
print(document_resource(name, resource_spec), file=args.out_file)
- print('''class ArvadosAPIClient:''', file=args.out_file)
+ print(
+ _REQUEST_CLASS,
+ '''class ArvadosAPIClient(googleapiclient.discovery.Resource):''',
+ sep='\n', file=args.out_file,
+ )
for name, _ in resources:
class_name = classify_name(name)
docstring = f"Return an instance of `{class_name}` to call methods via this client"
commit b198eda19443c899602fb11d800a999ccd3521a8
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 28 09:24:27 2023 -0500
Merge branch '20831-user-table-locks' refs #20831
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index c65e142924..c5facdc7d9 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -225,7 +225,11 @@ func (conn *Conn) ConfigGet(ctx context.Context) (json.RawMessage, error) {
}
func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
- return conn.chooseBackend(conn.cluster.ClusterID).VocabularyGet(ctx)
+ return conn.local.VocabularyGet(ctx)
+}
+
+func (conn *Conn) DiscoveryDocument(ctx context.Context) (arvados.DiscoveryDocument, error) {
+ return conn.local.DiscoveryDocument(ctx)
}
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
@@ -627,6 +631,7 @@ var userAttrsCachedFromLoginCluster = map[string]bool{
"first_name": true,
"is_active": true,
"is_admin": true,
+ "is_invited": true,
"last_name": true,
"modified_at": true,
"prefs": true,
@@ -636,7 +641,6 @@ var userAttrsCachedFromLoginCluster = map[string]bool{
"etag": false,
"full_name": false,
"identity_url": false,
- "is_invited": false,
"modified_by_client_uuid": false,
"modified_by_user_uuid": false,
"owner_uuid": false,
@@ -648,7 +652,8 @@ var userAttrsCachedFromLoginCluster = map[string]bool{
func (conn *Conn) batchUpdateUsers(ctx context.Context,
options arvados.ListOptions,
- items []arvados.User) (err error) {
+ items []arvados.User,
+ includeAdminAndInvited bool) (err error) {
id := conn.cluster.Login.LoginCluster
logger := ctxlog.FromContext(ctx)
@@ -695,6 +700,11 @@ func (conn *Conn) batchUpdateUsers(ctx context.Context,
}
}
}
+ if !includeAdminAndInvited {
+ // make sure we don't send these fields.
+ delete(updates, "is_admin")
+ delete(updates, "is_invited")
+ }
batchOpts.Updates[user.UUID] = updates
}
if len(batchOpts.Updates) > 0 {
@@ -707,13 +717,47 @@ func (conn *Conn) batchUpdateUsers(ctx context.Context,
return nil
}
+func (conn *Conn) includeAdminAndInvitedInBatchUpdate(ctx context.Context, be backend, updateUserUUID string) (bool, error) {
+ // API versions prior to 20231117 would only include the
+ // is_invited and is_admin fields if the current user is an
+ // admin, or is requesting their own user record. If those
+ // fields aren't actually valid then we don't want to
+ // send them in the batch update.
+ dd, err := be.DiscoveryDocument(ctx)
+ if err != nil {
+ // couldn't get discovery document
+ return false, err
+ }
+ if dd.Revision >= "20231117" {
+ // newer version, fields are valid.
+ return true, nil
+ }
+ selfuser, err := be.UserGetCurrent(ctx, arvados.GetOptions{})
+ if err != nil {
+ // couldn't get our user record
+ return false, err
+ }
+ if selfuser.IsAdmin || selfuser.UUID == updateUserUUID {
+ // we are an admin, or the current user is the same as
+ // the user that we are updating.
+ return true, nil
+ }
+ // Better safe than sorry.
+ return false, nil
+}
+
func (conn *Conn) UserList(ctx context.Context, options arvados.ListOptions) (arvados.UserList, error) {
if id := conn.cluster.Login.LoginCluster; id != "" && id != conn.cluster.ClusterID && !options.BypassFederation {
- resp, err := conn.chooseBackend(id).UserList(ctx, options)
+ be := conn.chooseBackend(id)
+ resp, err := be.UserList(ctx, options)
if err != nil {
return resp, err
}
- err = conn.batchUpdateUsers(ctx, options, resp.Items)
+ includeAdminAndInvited, err := conn.includeAdminAndInvitedInBatchUpdate(ctx, be, "")
+ if err != nil {
+ return arvados.UserList{}, err
+ }
+ err = conn.batchUpdateUsers(ctx, options, resp.Items, includeAdminAndInvited)
if err != nil {
return arvados.UserList{}, err
}
@@ -730,13 +774,18 @@ func (conn *Conn) UserUpdate(ctx context.Context, options arvados.UpdateOptions)
if options.BypassFederation {
return conn.local.UserUpdate(ctx, options)
}
- resp, err := conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ be := conn.chooseBackend(options.UUID)
+ resp, err := be.UserUpdate(ctx, options)
if err != nil {
return resp, err
}
if !strings.HasPrefix(options.UUID, conn.cluster.ClusterID) {
+ includeAdminAndInvited, err := conn.includeAdminAndInvitedInBatchUpdate(ctx, be, options.UUID)
+ if err != nil {
+ return arvados.User{}, err
+ }
// Copy the updated user record to the local cluster
- err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp})
+ err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp}, includeAdminAndInvited)
if err != nil {
return arvados.User{}, err
}
@@ -783,7 +832,8 @@ func (conn *Conn) UserUnsetup(ctx context.Context, options arvados.GetOptions) (
}
func (conn *Conn) UserGet(ctx context.Context, options arvados.GetOptions) (arvados.User, error) {
- resp, err := conn.chooseBackend(options.UUID).UserGet(ctx, options)
+ be := conn.chooseBackend(options.UUID)
+ resp, err := be.UserGet(ctx, options)
if err != nil {
return resp, err
}
@@ -791,7 +841,11 @@ func (conn *Conn) UserGet(ctx context.Context, options arvados.GetOptions) (arva
return arvados.User{}, httpErrorf(http.StatusBadGateway, "Had requested %v but response was for %v", options.UUID, resp.UUID)
}
if options.UUID[:5] != conn.cluster.ClusterID {
- err = conn.batchUpdateUsers(ctx, arvados.ListOptions{Select: options.Select}, []arvados.User{resp})
+ includeAdminAndInvited, err := conn.includeAdminAndInvitedInBatchUpdate(ctx, be, options.UUID)
+ if err != nil {
+ return arvados.User{}, err
+ }
+ err = conn.batchUpdateUsers(ctx, arvados.ListOptions{Select: options.Select}, []arvados.User{resp}, includeAdminAndInvited)
if err != nil {
return arvados.User{}, err
}
diff --git a/lib/controller/federation/user_test.go b/lib/controller/federation/user_test.go
index 1bd1bd2f18..33bc95d0ea 100644
--- a/lib/controller/federation/user_test.go
+++ b/lib/controller/federation/user_test.go
@@ -78,7 +78,7 @@ func (s *UserSuite) TestLoginClusterUserList(c *check.C) {
"identity_url": false,
// virtual attrs
"full_name": false,
- "is_invited": false,
+ "is_invited": true,
}
if opts.Select != nil {
// Only the selected
@@ -146,7 +146,7 @@ func (s *UserSuite) TestLoginClusterUserGet(c *check.C) {
"identity_url": false,
// virtual attrs
"full_name": false,
- "is_invited": false,
+ "is_invited": true,
}
if opts.Select != nil {
// Only the selected
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index e15e4c4702..a8ecc57bba 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -20,6 +20,7 @@ import (
"net/url"
"strconv"
"strings"
+ "sync"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
@@ -44,10 +45,13 @@ type Conn struct {
SendHeader http.Header
RedactHostInErrors bool
- clusterID string
- httpClient http.Client
- baseURL url.URL
- tokenProvider TokenProvider
+ clusterID string
+ httpClient http.Client
+ baseURL url.URL
+ tokenProvider TokenProvider
+ discoveryDocument *arvados.DiscoveryDocument
+ discoveryDocumentMtx sync.Mutex
+ discoveryDocumentExpires time.Time
}
func NewConn(clusterID string, url *url.URL, insecure bool, tp TokenProvider) *Conn {
@@ -146,10 +150,13 @@ func (conn *Conn) requestAndDecode(ctx context.Context, dst interface{}, ep arva
}
if len(tokens) > 1 {
+ if params == nil {
+ params = make(map[string]interface{})
+ }
params["reader_tokens"] = tokens[1:]
}
path := ep.Path
- if strings.Contains(ep.Path, "/{uuid}") {
+ if strings.Contains(ep.Path, "/{uuid}") && params != nil {
uuid, _ := params["uuid"].(string)
path = strings.Replace(path, "/{uuid}", "/"+uuid, 1)
delete(params, "uuid")
@@ -189,6 +196,22 @@ func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error)
return resp, err
}
+func (conn *Conn) DiscoveryDocument(ctx context.Context) (arvados.DiscoveryDocument, error) {
+ conn.discoveryDocumentMtx.Lock()
+ defer conn.discoveryDocumentMtx.Unlock()
+ if conn.discoveryDocument != nil && time.Now().Before(conn.discoveryDocumentExpires) {
+ return *conn.discoveryDocument, nil
+ }
+ var dd arvados.DiscoveryDocument
+ err := conn.requestAndDecode(ctx, &dd, arvados.EndpointDiscoveryDocument, nil, nil)
+ if err != nil {
+ return dd, err
+ }
+ conn.discoveryDocument = &dd
+ conn.discoveryDocumentExpires = time.Now().Add(time.Hour)
+ return *conn.discoveryDocument, nil
+}
+
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
ep := arvados.EndpointLogin
var resp arvados.LoginResponse
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index f4ac1ab3c4..bff01eeda5 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -25,6 +25,7 @@ type APIEndpoint struct {
var (
EndpointConfigGet = APIEndpoint{"GET", "arvados/v1/config", ""}
EndpointVocabularyGet = APIEndpoint{"GET", "arvados/v1/vocabulary", ""}
+ EndpointDiscoveryDocument = APIEndpoint{"GET", "discovery/v1/apis/arvados/v1/rest", ""}
EndpointLogin = APIEndpoint{"GET", "login", ""}
EndpointLogout = APIEndpoint{"GET", "logout", ""}
EndpointAuthorizedKeyCreate = APIEndpoint{"POST", "arvados/v1/authorized_keys", "authorized_key"}
@@ -347,4 +348,5 @@ type API interface {
APIClientAuthorizationDelete(ctx context.Context, options DeleteOptions) (APIClientAuthorization, error)
APIClientAuthorizationUpdate(ctx context.Context, options UpdateOptions) (APIClientAuthorization, error)
APIClientAuthorizationGet(ctx context.Context, options GetOptions) (APIClientAuthorization, error)
+ DiscoveryDocument(ctx context.Context) (DiscoveryDocument, error)
}
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index d71ade8a81..735a44d24c 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -630,6 +630,7 @@ type DiscoveryDocument struct {
GitURL string `json:"gitUrl"`
Schemas map[string]Schema `json:"schemas"`
Resources map[string]Resource `json:"resources"`
+ Revision string `json:"revision"`
}
type Resource struct {
diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go
index 4e214414d7..b5f8e962dc 100644
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@ -40,6 +40,10 @@ func (as *APIStub) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error
as.appendCall(ctx, as.VocabularyGet, nil)
return arvados.Vocabulary{}, as.Error
}
+func (as *APIStub) DiscoveryDocument(ctx context.Context) (arvados.DiscoveryDocument, error) {
+ as.appendCall(ctx, as.DiscoveryDocument, nil)
+ return arvados.DiscoveryDocument{}, as.Error
+}
func (as *APIStub) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
as.appendCall(ctx, as.Login, options)
return arvados.LoginResponse{}, as.Error
diff --git a/sdk/python/arvados-v1-discovery.json b/sdk/python/arvados-v1-discovery.json
index 5d4666bf94..93c692a191 100644
--- a/sdk/python/arvados-v1-discovery.json
+++ b/sdk/python/arvados-v1-discovery.json
@@ -9094,7 +9094,7 @@
}
}
},
- "revision": "20220510",
+ "revision": "20231117",
"schemas": {
"JobList": {
"id": "JobList",
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 4d15cb1215..a6809c6293 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -36,7 +36,7 @@ class Arvados::V1::SchemaController < ApplicationController
# format is YYYYMMDD, must be fixed width (needs to be lexically
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20220510",
+ revision: "20231117",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index 88377dbaf6..7b2b40c3a8 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -16,42 +16,13 @@ class Arvados::V1::UsersController < ApplicationController
# records from LoginCluster.
def batch_update
@objects = []
- params[:updates].andand.each do |uuid, attrs|
- begin
- u = User.find_or_create_by(uuid: uuid)
- rescue ActiveRecord::RecordNotUnique
- retry
- end
- needupdate = {}
- nullify_attrs(attrs).each do |k,v|
- if !v.nil? && u.send(k) != v
- needupdate[k] = v
- end
- end
- if needupdate.length > 0
- begin
- u.update_attributes!(needupdate)
- rescue ActiveRecord::RecordInvalid
- loginCluster = Rails.configuration.Login.LoginCluster
- if u.uuid[0..4] == loginCluster && !needupdate[:username].nil?
- local_user = User.find_by_username(needupdate[:username])
- # The username of this record conflicts with an existing,
- # different user record. This can happen because the
- # username changed upstream on the login cluster, or
- # because we're federated with another cluster with a user
- # by the same username. The login cluster is the source
- # of truth, so change the username on the conflicting
- # record and retry the update operation.
- if local_user.uuid != u.uuid
- new_username = "#{needupdate[:username]}#{rand(99999999)}"
- Rails.logger.warn("cached username '#{needupdate[:username]}' collision with user '#{local_user.uuid}' - renaming to '#{new_username}' before retrying")
- local_user.update_attributes!({username: new_username})
- retry
- end
- end
- raise # Not the issue we're handling above
- end
- end
+ # update_remote_user takes a row lock on the User record, so sort
+ # the keys so we always lock them in the same order.
+ sorted = params[:updates].keys.sort
+ sorted.each do |uuid|
+ attrs = params[:updates][uuid]
+ attrs[:uuid] = uuid
+ u = User.update_remote_user nullify_attrs(attrs)
@objects << u
end
@offset = 0
@@ -279,7 +250,7 @@ class Arvados::V1::UsersController < ApplicationController
return super if @read_users.any?(&:is_admin)
if params[:uuid] != current_user.andand.uuid
# Non-admin index/show returns very basic information about readable users.
- safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name", "username", "can_write", "can_manage", "kind"]
+ safe_attrs = ["uuid", "is_active", "is_admin", "is_invited", "email", "first_name", "last_name", "username", "can_write", "can_manage", "kind"]
if @select
@select = @select & safe_attrs
else
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index c149ffc329..a03ce4f899 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -363,69 +363,17 @@ class ApiClientAuthorization < ArvadosModel
if user.nil? and remote_user.nil?
Rails.logger.warn "remote token #{token.inspect} rejected: cannot get owner #{remote_user_uuid} from database or remote cluster"
return nil
+ end
+
# Invariant: remote_user_prefix == upstream_cluster_id
# therefore: remote_user_prefix != Rails.configuration.ClusterID
# Add or update user and token in local database so we can
# validate subsequent requests faster.
- elsif user.nil?
- # Create a new record for this user.
- user = User.new(uuid: remote_user['uuid'],
- is_active: false,
- is_admin: false,
- email: remote_user['email'],
- owner_uuid: system_user_uuid)
- user.set_initial_username(requested: remote_user['username'])
- end
- # Sync user record if we loaded a remote user.
act_as_system_user do
- if remote_user
- %w[first_name last_name email prefs].each do |attr|
- user.send(attr+'=', remote_user[attr])
- end
-
- begin
- user.save!
- rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
- Rails.logger.debug("remote user #{remote_user['uuid']} already exists, retrying...")
- # Some other request won the race: retry fetching the user record.
- user = User.find_by_uuid(remote_user['uuid'])
- if !user
- Rails.logger.warn("cannot find or create remote user #{remote_user['uuid']}")
- return nil
- end
- end
-
- if user.is_invited && !remote_user['is_invited']
- # Remote user is not "invited" state, they should be unsetup, which
- # also makes them inactive.
- user.unsetup
- else
- if !user.is_invited && remote_user['is_invited'] and
- (remote_user_prefix == Rails.configuration.Login.LoginCluster or
- Rails.configuration.Users.AutoSetupNewUsers or
- Rails.configuration.Users.NewUsersAreActive or
- Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
- user.setup
- end
-
- if !user.is_active && remote_user['is_active'] && user.is_invited and
- (remote_user_prefix == Rails.configuration.Login.LoginCluster or
- Rails.configuration.Users.NewUsersAreActive or
- Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
- user.update_attributes!(is_active: true)
- elsif user.is_active && !remote_user['is_active']
- user.update_attributes!(is_active: false)
- end
-
- if remote_user_prefix == Rails.configuration.Login.LoginCluster and
- user.is_active and
- user.is_admin != remote_user['is_admin']
- # Remote cluster controls our user database, including the
- # admin flag.
- user.update_attributes!(is_admin: remote_user['is_admin'])
- end
- end
+ if remote_user && remote_user_uuid != anonymous_user_uuid
+ # Sync user record if we loaded a remote user.
+ user = User.update_remote_user remote_user
end
# If stored_secret is set, we save stored_secret in the database
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index bbdd9c2843..d4d4a82458 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -34,6 +34,7 @@ class User < ArvadosModel
before_create :set_initial_username, :if => Proc.new {
username.nil? and email
}
+ before_create :active_is_not_nil
after_create :after_ownership_change
after_create :setup_on_activate
after_create :add_system_group_permission_link
@@ -104,6 +105,10 @@ class User < ArvadosModel
self.groups_i_can(:read).select { |x| x.match(/-f+$/) }.first)
end
+ def self.ignored_select_attributes
+ super + ["full_name", "is_invited"]
+ end
+
def groups_i_can(verb)
my_groups = self.group_permissions(VAL_FOR_PERM[verb]).keys
if verb == :read
@@ -382,7 +387,7 @@ SELECT target_uuid, perm_level
end
def set_initial_username(requested: false)
- if !requested.is_a?(String) || requested.empty?
+ if (!requested.is_a?(String) || requested.empty?) and email
email_parts = email.partition("@")
local_parts = email_parts.first.partition("+")
if email_parts.any?(&:empty?)
@@ -393,13 +398,20 @@ SELECT target_uuid, perm_level
requested = email_parts.first
end
end
- requested.sub!(/^[^A-Za-z]+/, "")
- requested.gsub!(/[^A-Za-z0-9]/, "")
- unless requested.empty?
+ if requested
+ requested.sub!(/^[^A-Za-z]+/, "")
+ requested.gsub!(/[^A-Za-z0-9]/, "")
+ end
+ unless !requested || requested.empty?
self.username = find_usable_username_from(requested)
end
end
+ def active_is_not_nil
+ self.is_active = false if self.is_active.nil?
+ self.is_admin = false if self.is_admin.nil?
+ end
+
# Move this user's (i.e., self's) owned items to new_owner_uuid and
# new_user_uuid (for things normally owned directly by the user).
#
@@ -592,6 +604,98 @@ SELECT target_uuid, perm_level
primary_user
end
+ def self.update_remote_user remote_user
+ remote_user = remote_user.symbolize_keys
+ begin
+ user = User.find_or_create_by(uuid: remote_user[:uuid])
+ rescue ActiveRecord::RecordNotUnique
+ retry
+ end
+
+ remote_user_prefix = user.uuid[0..4]
+ user.with_lock do
+ needupdate = {}
+ [:email, :username, :first_name, :last_name, :prefs].each do |k|
+ v = remote_user[k]
+ if !v.nil? && user.send(k) != v
+ needupdate[k] = v
+ end
+ end
+
+ user.email = needupdate[:email] if needupdate[:email]
+
+ loginCluster = Rails.configuration.Login.LoginCluster
+ if user.username.nil? || user.username == ""
+ # Don't have a username yet, set one
+ needupdate[:username] = user.set_initial_username(requested: remote_user[:username])
+ elsif remote_user_prefix != loginCluster
+ # Upstream is not login cluster, don't try to change the
+ # username once set.
+ needupdate.delete :username
+ end
+
+ if needupdate.length > 0
+ begin
+ user.update!(needupdate)
+ rescue ActiveRecord::RecordInvalid
+ if remote_user_prefix == loginCluster && !needupdate[:username].nil?
+ local_user = User.find_by_username(needupdate[:username])
+ # The username of this record conflicts with an existing,
+ # different user record. This can happen because the
+ # username changed upstream on the login cluster, or
+ # because we're federated with another cluster with a user
+ # by the same username. The login cluster is the source
+ # of truth, so change the username on the conflicting
+ # record and retry the update operation.
+ if local_user.uuid != user.uuid
+ new_username = "#{needupdate[:username]}#{rand(99999999)}"
+ Rails.logger.warn("cached username '#{needupdate[:username]}' collision with user '#{local_user.uuid}' - renaming to '#{new_username}' before retrying")
+ local_user.update!({username: new_username})
+ retry
+ end
+ end
+ raise # Not the issue we're handling above
+ end
+ end
+
+ if user.is_invited && remote_user[:is_invited] == false
+ # Remote user is not "invited" state, they should be unsetup, which
+ # also makes them inactive.
+ user.unsetup
+ else
+ if !user.is_invited && remote_user[:is_invited] and
+ (remote_user_prefix == Rails.configuration.Login.LoginCluster or
+ Rails.configuration.Users.AutoSetupNewUsers or
+ Rails.configuration.Users.NewUsersAreActive or
+ Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
+ # Remote user is 'invited' and should be set up
+ user.setup
+ end
+
+ if !user.is_active && remote_user[:is_active] && user.is_invited and
+ (remote_user_prefix == Rails.configuration.Login.LoginCluster or
+ Rails.configuration.Users.NewUsersAreActive or
+ Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
+ # remote user is active and invited, we need to activate them
+ user.update!(is_active: true)
+ elsif user.is_active && remote_user[:is_active] == false
+ # remote user is not active, we need to de-activate them
+ user.update!(is_active: false)
+ end
+
+ if remote_user_prefix == Rails.configuration.Login.LoginCluster and
+ user.is_active and
+ !remote_user[:is_admin].nil? and
+ user.is_admin != remote_user[:is_admin]
+ # Remote cluster controls our user database, including the
+ # admin flag.
+ user.update!(is_admin: remote_user[:is_admin])
+ end
+ end
+ end
+ user
+ end
+
protected
def self.attributes_required_columns
diff --git a/services/api/test/fixtures/links.yml b/services/api/test/fixtures/links.yml
index 99b97510db..00d5971534 100644
--- a/services/api/test/fixtures/links.yml
+++ b/services/api/test/fixtures/links.yml
@@ -1139,3 +1139,17 @@ public_favorites_permission_link:
name: can_read
head_uuid: zzzzz-j7d0g-publicfavorites
properties: {}
+
+future_project_user_member_of_all_users_group:
+ uuid: zzzzz-o0j2j-cdnq6627g0h0r2a
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2015-07-28T21:34:41.361747000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2015-07-28T21:34:41.361747000Z
+ updated_at: 2015-07-28T21:34:41.361747000Z
+ tail_uuid: zzzzz-tpzed-futureprojview2
+ link_class: permission
+ name: can_write
+ head_uuid: zzzzz-j7d0g-fffffffffffffff
+ properties: {}
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index 50d1606a08..aeb7e6a88e 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -1063,23 +1063,28 @@ The Arvados team.
'is_active' => true,
'is_admin' => true,
'prefs' => {'foo' => 'bar'},
+ 'is_invited' => true
},
newuuid => {
'first_name' => 'noot',
'email' => 'root at remot.example.com',
'username' => '',
+ 'is_invited' => true
},
unchanginguuid => {
'email' => 'root at unchanging.example.com',
'prefs' => {'foo' => {'bar' => 'baz'}},
+ 'is_invited' => true
},
conflictinguuid1 => {
'email' => 'root at conflictingname1.example.com',
- 'username' => 'active'
+ 'username' => 'active',
+ 'is_invited' => true
},
conflictinguuid2 => {
'email' => 'root at conflictingname2.example.com',
- 'username' => 'federatedactive'
+ 'username' => 'federatedactive',
+ 'is_invited' => true
},
}})
assert_response(:success)
@@ -1096,7 +1101,7 @@ The Arvados team.
assert_equal(1, Log.where(object_uuid: unchanginguuid).count)
end
- NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
+ NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "is_admin", "is_invited", "email", "first_name",
"last_name", "username", "can_write", "can_manage"].sort
def check_non_admin_index
diff --git a/services/api/test/integration/users_test.rb b/services/api/test/integration/users_test.rb
index ca14336389..f8956b21e2 100644
--- a/services/api/test/integration/users_test.rb
+++ b/services/api/test/integration/users_test.rb
@@ -303,15 +303,15 @@ class UsersTest < ActionDispatch::IntegrationTest
assert_response :success
rp = json_response
assert_not_nil rp["uuid"]
- assert_not_nil rp["is_active"]
- assert_nil rp["is_admin"]
+ assert_equal true, rp["is_active"]
+ assert_equal false, rp["is_admin"]
get "/arvados/v1/users/#{rp['uuid']}",
params: {format: 'json'},
headers: auth(:admin)
assert_response :success
assert_equal rp["uuid"], json_response['uuid']
- assert_nil json_response['is_admin']
+ assert_equal false, json_response['is_admin']
assert_equal true, json_response['is_active']
assert_equal 'foo at example.com', json_response['email']
assert_equal 'barney', json_response['username']
diff --git a/services/api/test/unit/user_test.rb b/services/api/test/unit/user_test.rb
index 7e19ad5821..8a41fb3976 100644
--- a/services/api/test/unit/user_test.rb
+++ b/services/api/test/unit/user_test.rb
@@ -153,12 +153,12 @@ class UserTest < ActiveSupport::TestCase
assert_equal("active/foo", repositories(:foo).name)
end
- [[false, 'foo at example.com', true, nil],
- [false, 'bar at example.com', nil, true],
- [true, 'foo at example.com', true, nil],
+ [[false, 'foo at example.com', true, false],
+ [false, 'bar at example.com', false, true],
+ [true, 'foo at example.com', true, false],
[true, 'bar at example.com', true, true],
- [false, '', nil, nil],
- [true, '', true, nil]
+ [false, '', false, false],
+ [true, '', true, false]
].each do |auto_admin_first_user_config, auto_admin_user_config, foo_should_be_admin, bar_should_be_admin|
# In each case, 'foo' is created first, then 'bar', then 'bar2', then 'baz'.
test "auto admin with auto_admin_first=#{auto_admin_first_user_config} auto_admin=#{auto_admin_user_config}" do
commit 0f6951dae0640b14f95df30f4ce65e11db90be26
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Nov 27 11:35:05 2023 -0500
Merge branch '21137-rp-initiated-logout'
Closes #21137.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/lib/controller/localdb/login_oidc.go b/lib/controller/localdb/login_oidc.go
index 65e2e250e5..946cfe0927 100644
--- a/lib/controller/localdb/login_oidc.go
+++ b/lib/controller/localdb/login_oidc.go
@@ -68,10 +68,11 @@ type oidcLoginController struct {
// https://people.googleapis.com/)
peopleAPIBasePath string
- provider *oidc.Provider // initialized by setup()
- oauth2conf *oauth2.Config // initialized by setup()
- verifier *oidc.IDTokenVerifier // initialized by setup()
- mu sync.Mutex // protects setup()
+ provider *oidc.Provider // initialized by setup()
+ endSessionURL *url.URL // initialized by setup()
+ oauth2conf *oauth2.Config // initialized by setup()
+ verifier *oidc.IDTokenVerifier // initialized by setup()
+ mu sync.Mutex // protects setup()
}
// Initialize ctrl.provider and ctrl.oauth2conf.
@@ -101,11 +102,46 @@ func (ctrl *oidcLoginController) setup() error {
ClientID: ctrl.ClientID,
})
ctrl.provider = provider
+ var claims struct {
+ EndSessionEndpoint string `json:"end_session_endpoint"`
+ }
+ err = provider.Claims(&claims)
+ if err != nil {
+ return fmt.Errorf("error parsing OIDC discovery metadata: %v", err)
+ } else if claims.EndSessionEndpoint == "" {
+ ctrl.endSessionURL = nil
+ } else {
+ u, err := url.Parse(claims.EndSessionEndpoint)
+ if err != nil {
+ return fmt.Errorf("OIDC end_session_endpoint is not a valid URL: %v", err)
+ } else if u.Scheme != "https" {
+ return fmt.Errorf("OIDC end_session_endpoint MUST use HTTPS but does not: %v", u.String())
+ } else {
+ ctrl.endSessionURL = u
+ }
+ }
return nil
}
func (ctrl *oidcLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
- return logout(ctx, ctrl.Cluster, opts)
+ err := ctrl.setup()
+ if err != nil {
+ return arvados.LogoutResponse{}, fmt.Errorf("error setting up OpenID Connect provider: %s", err)
+ }
+ resp, err := logout(ctx, ctrl.Cluster, opts)
+ if err != nil {
+ return arvados.LogoutResponse{}, err
+ }
+ creds, credsOK := auth.FromContext(ctx)
+ if ctrl.endSessionURL != nil && credsOK && len(creds.Tokens) > 0 {
+ values := ctrl.endSessionURL.Query()
+ values.Set("client_id", ctrl.ClientID)
+ values.Set("post_logout_redirect_uri", resp.RedirectLocation)
+ u := *ctrl.endSessionURL
+ u.RawQuery = values.Encode()
+ resp.RedirectLocation = u.String()
+ }
+ return resp, err
}
func (ctrl *oidcLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
diff --git a/lib/controller/localdb/login_oidc_test.go b/lib/controller/localdb/login_oidc_test.go
index 9469fdfd31..f505f5bc49 100644
--- a/lib/controller/localdb/login_oidc_test.go
+++ b/lib/controller/localdb/login_oidc_test.go
@@ -15,6 +15,7 @@ import (
"net/http"
"net/http/httptest"
"net/url"
+ "regexp"
"sort"
"strings"
"sync"
@@ -97,6 +98,75 @@ func (s *OIDCLoginSuite) TestGoogleLogout(c *check.C) {
c.Check(resp.RedirectLocation, check.Equals, "https://192.168.1.1/bar")
}
+func (s *OIDCLoginSuite) checkRPInitiatedLogout(c *check.C, returnTo string) {
+ if !c.Check(s.fakeProvider.EndSessionEndpoint, check.NotNil,
+ check.Commentf("buggy test: EndSessionEndpoint not configured")) {
+ return
+ }
+ expURL, err := url.Parse(s.fakeProvider.Issuer.URL)
+ if !c.Check(err, check.IsNil, check.Commentf("error parsing expected URL")) {
+ return
+ }
+ expURL.Path = expURL.Path + s.fakeProvider.EndSessionEndpoint.Path
+
+ accessToken := s.fakeProvider.ValidAccessToken()
+ ctx := ctrlctx.NewWithToken(s.ctx, s.cluster, accessToken)
+ resp, err := s.localdb.Logout(ctx, arvados.LogoutOptions{ReturnTo: returnTo})
+ if !c.Check(err, check.IsNil) {
+ return
+ }
+ loc, err := url.Parse(resp.RedirectLocation)
+ if !c.Check(err, check.IsNil, check.Commentf("error parsing response URL")) {
+ return
+ }
+
+ c.Check(loc.Scheme, check.Equals, "https")
+ c.Check(loc.Host, check.Equals, expURL.Host)
+ c.Check(loc.Path, check.Equals, expURL.Path)
+
+ var expReturn string
+ switch returnTo {
+ case "":
+ expReturn = s.cluster.Services.Workbench2.ExternalURL.String()
+ default:
+ expReturn = returnTo
+ }
+ values := loc.Query()
+ c.Check(values.Get("client_id"), check.Equals, s.cluster.Login.Google.ClientID)
+ c.Check(values.Get("post_logout_redirect_uri"), check.Equals, expReturn)
+}
+
+func (s *OIDCLoginSuite) TestRPInitiatedLogoutWithoutReturnTo(c *check.C) {
+ s.fakeProvider.EndSessionEndpoint = &url.URL{Path: "/logout/fromRP"}
+ s.checkRPInitiatedLogout(c, "")
+}
+
+func (s *OIDCLoginSuite) TestRPInitiatedLogoutWithReturnTo(c *check.C) {
+ s.fakeProvider.EndSessionEndpoint = &url.URL{Path: "/rp_logout"}
+ u := arvados.URL{Scheme: "https", Host: "foo.example", Path: "/"}
+ s.cluster.Login.TrustedClients[u] = struct{}{}
+ s.checkRPInitiatedLogout(c, u.String())
+}
+
+func (s *OIDCLoginSuite) TestEndSessionEndpointBadScheme(c *check.C) {
+ // RP-Initiated Logout 1.0 says: "This URL MUST use the https scheme..."
+ u := url.URL{Scheme: "http", Host: "example.com"}
+ s.fakeProvider.EndSessionEndpoint = &u
+ _, err := s.localdb.Logout(s.ctx, arvados.LogoutOptions{})
+ c.Check(err, check.ErrorMatches,
+ `.*\bend_session_endpoint MUST use HTTPS but does not: `+regexp.QuoteMeta(u.String()))
+}
+
+func (s *OIDCLoginSuite) TestNoRPInitiatedLogoutWithoutToken(c *check.C) {
+ endPath := "/TestNoRPInitiatedLogoutWithoutToken"
+ s.fakeProvider.EndSessionEndpoint = &url.URL{Path: endPath}
+ resp, _ := s.localdb.Logout(s.ctx, arvados.LogoutOptions{})
+ u, err := url.Parse(resp.RedirectLocation)
+ c.Check(err, check.IsNil)
+ c.Check(strings.HasSuffix(u.Path, endPath), check.Equals, false,
+ check.Commentf("logout redirected to end_session_endpoint without token"))
+}
+
func (s *OIDCLoginSuite) TestGoogleLogin_Start_Bogus(c *check.C) {
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{})
c.Check(err, check.IsNil)
diff --git a/sdk/go/arvadostest/oidc_provider.go b/sdk/go/arvadostest/oidc_provider.go
index 529c1dca12..31a2667122 100644
--- a/sdk/go/arvadostest/oidc_provider.go
+++ b/sdk/go/arvadostest/oidc_provider.go
@@ -33,6 +33,10 @@ type OIDCProvider struct {
AuthGivenName string
AuthFamilyName string
AccessTokenPayload map[string]interface{}
+ // end_session_endpoint metadata URL.
+ // If nil or empty, not included in discovery.
+ // If relative, built from Issuer.URL.
+ EndSessionEndpoint *url.URL
PeopleAPIResponse map[string]interface{}
@@ -71,13 +75,26 @@ func (p *OIDCProvider) serveOIDC(w http.ResponseWriter, req *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch req.URL.Path {
case "/.well-known/openid-configuration":
- json.NewEncoder(w).Encode(map[string]interface{}{
+ configuration := map[string]interface{}{
"issuer": p.Issuer.URL,
"authorization_endpoint": p.Issuer.URL + "/auth",
"token_endpoint": p.Issuer.URL + "/token",
"jwks_uri": p.Issuer.URL + "/jwks",
"userinfo_endpoint": p.Issuer.URL + "/userinfo",
- })
+ }
+ if p.EndSessionEndpoint == nil {
+ // Not included in configuration
+ } else if p.EndSessionEndpoint.Scheme != "" {
+ configuration["end_session_endpoint"] = p.EndSessionEndpoint.String()
+ } else {
+ u, err := url.Parse(p.Issuer.URL)
+ p.c.Check(err, check.IsNil,
+ check.Commentf("error parsing IssuerURL for EndSessionEndpoint"))
+ u.Scheme = "https"
+ u.Path = u.Path + p.EndSessionEndpoint.Path
+ configuration["end_session_endpoint"] = u.String()
+ }
+ json.NewEncoder(w).Encode(configuration)
case "/token":
var clientID, clientSecret string
auth, _ := base64.StdEncoding.DecodeString(strings.TrimPrefix(req.Header.Get("Authorization"), "Basic "))
commit f7a870dde952902343656e367b26ad550fe3d63c
Author: Tom Clegg <tom at curii.com>
Date: Mon Nov 27 11:48:10 2023 -0500
Merge branch '21126-trash-when-ro'
closes #21126
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/Gemfile.lock b/doc/Gemfile.lock
index 420e13146f..a6c85319c8 100644
--- a/doc/Gemfile.lock
+++ b/doc/Gemfile.lock
@@ -29,4 +29,4 @@ DEPENDENCIES
zenweb
BUNDLED WITH
- 2.1.4
+ 2.2.19
diff --git a/doc/_config.yml b/doc/_config.yml
index cbb082aefb..04ff5caa9b 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -195,6 +195,7 @@ navbar:
- admin/storage-classes.html.textile.liquid
- admin/keep-recovering-data.html.textile.liquid
- admin/keep-measuring-deduplication.html.textile.liquid
+ - admin/keep-faster-gc-s3.html.textile.liquid
- Cloud:
- admin/spot-instances.html.textile.liquid
- admin/cloudtest.html.textile.liquid
diff --git a/doc/admin/keep-balance.html.textile.liquid b/doc/admin/keep-balance.html.textile.liquid
index 2785930de8..4d18307cf7 100644
--- a/doc/admin/keep-balance.html.textile.liquid
+++ b/doc/admin/keep-balance.html.textile.liquid
@@ -30,14 +30,12 @@ The @Collections.BalancePeriod@ value in @/etc/arvados/config.yml@ determines th
Keep-balance can also be run with the @-once@ flag to do a single scan/balance operation and then exit. The exit code will be zero if the operation was successful.
-h3. Committing
-
-Keep-balance computes and reports changes but does not implement them by sending pull and trash lists to the Keep services unless the @-commit-pull@ and @-commit-trash@ flags are used.
-
h3. Additional configuration
For configuring resource usage tuning and lost block reporting, please see the @Collections.BlobMissingReport@, @Collections.BalanceCollectionBatch@, @Collections.BalanceCollectionBuffers@ option in the "default config.yml file":{{site.baseurl}}/admin/config.html.
+The @Collections.BalancePullLimit@ and @Collections.BalanceTrashLimit@ configuration entries determine the maximum number of pull and trash operations keep-balance will attempt to apply on each keepstore server. If both values are zero, keep-balance will operate in "dry run" mode, where all changes are computed but none are committed.
+
h3. Limitations
Keep-balance does not attempt to discover whether committed pull and trash requests ever get carried out -- only that they are accepted by the Keep services. If some services are full, new copies of under-replicated blocks might never get made, only repeatedly requested.
diff --git a/doc/admin/keep-faster-gc-s3.html.textile.liquid b/doc/admin/keep-faster-gc-s3.html.textile.liquid
new file mode 100644
index 0000000000..2230ac2491
--- /dev/null
+++ b/doc/admin/keep-faster-gc-s3.html.textile.liquid
@@ -0,0 +1,41 @@
+---
+layout: default
+navsection: admin
+title: "Faster garbage collection in S3"
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+When there is a large number of unneeded blocks stored in an S3 bucket, particularly when using @PrefixLength: 0@, the speed of garbage collection can be severely limited by AWS API rate limits and Arvados's multi-step trash/delete process.
+
+The multi-step trash/delete process can be short-circuited by setting @BlobTrashLifetime@ to zero and enabling @UnsafeDelete@ on S3-backed volumes. However, on an actively used cluster such a configuration *can result in data loss* in the rare case where a given block is trashed and then rewritten soon afterward, and S3 processes the write and delete requests in the opposite order.
+
+The following steps can be used to temporarily disable writes on an S3 bucket to enable faster garbage collection without data loss or service interruption. Note that garbage collection on other S3 volumes will be temporarily disabled during this procedure.
+# Create a new S3 bucket and configure it as an additional volume (this step may be skipped if the configuration already has enough writable volumes that clients will still be able to write blocks while the target volume is read-only). We recommend using @PrefixLength: 3@ for the new volume because this results in a much higher rate limit for I/O and garbage collection operations compared to the default @PrefixLength: 0 at . If the target volume configuration specifies @StorageClasses@, use the same values for the new volume.
+# Shut down the @keep-balance@ service.
+# Update your configuration as follows: <notextile><pre>
+ Collections:
+ BlobTrashLifetime: 0
+ BalancePullLimit: 0
+ [...]
+ Volumes:
+ <span class="userinput">target-volume-uuid</span>:
+ ReadOnly: true
+ AllowTrashWhileReadOnly: true
+ DriverParameters:
+ UnsafeDelete: true
+</pre></notextile> Note that @BlobTrashLifetime: 0@ instructs keepstore to delete unneeded blocks outright (bypassing the recoverable trash phase); however, in this mode it will normally not trash any blocks at all on an S3 volume due to the safety issue mentioned above, unless the volume is configured with @UnsafeDelete: true at .
+# Restart all @keepstore@ services with the updated configuration.
+# Start the @keep-balance@ service.
+# Objects will be deleted immediately instead of being first copied to trash on the S3 volume, which should significantly speed up cleanup of trashed objects. Monitor progress by watching @keep-balance@ logs and metrics. When garbage collection is complete, keep-balance logs will show an empty changeset: <notextile><pre><code>zzzzz-bi6l4-0123456789abcdef (keep0.zzzzz.arvadosapi.com:25107, disk): ChangeSet{Pulls:0, Trashes:0}</code></pre></notextile>
+# Remove the @UnsafeDelete@ configuration entry on the target volume.
+# Remove the @BlobTrashLifetime@ configuration entry (or restore it to its previous value).
+# If the target volume has @PrefixLength: 0@ and the new volume has @PrefixLength: 3@, skip the next two steps: new data will be stored on the new volume, some existing data will be moved automatically to other volumes, and some will be left on the target volume as long as it's needed.
+# If you want to resume writing new data to the target volume, revert to @ReadOnly: false@ and @AllowTrashWhileReadOnly: false@ on the target volume.
+# If you want to stop writing new data to the newly created volume, set @ReadOnly: true@ and @AllowTrashWhileReadOnly: true@ on the new volume.
+# Remove the @BalancePullLimit@ configuration entry (or restore its previous value), and restart @keep-balance at .
+# Restart all @keepstore@ services with the updated configuration.
diff --git a/doc/install/install-keep-balance.html.textile.liquid b/doc/install/install-keep-balance.html.textile.liquid
index bb4ae7b3d8..05d27b7cb4 100644
--- a/doc/install/install-keep-balance.html.textile.liquid
+++ b/doc/install/install-keep-balance.html.textile.liquid
@@ -24,7 +24,7 @@ Keep-balance can be installed anywhere with network access to Keep services, arv
{% include 'notebox_begin' %}
-If you are installing keep-balance on an existing system with valuable data, you can run keep-balance in "dry run" mode first and review its logs as a precaution. To do this, edit your keep-balance startup script to use the flags @-commit-pulls=false -commit-trash=false -commit-confirmed-fields=false at .
+If you are installing keep-balance on an existing system with valuable data, you can run keep-balance in "dry run" mode first and review its logs as a precaution. To do this, set the @Collections.BalancePullLimit@ and @Collections.BalanceTrashLimit@ configuration entries to zero.
{% include 'notebox_end' %}
commit e371b3b0ef8f44a6c7bcd49a96f6d9f872b479b5
Author: Tom Clegg <tom at curii.com>
Date: Thu Nov 2 14:40:37 2023 -0400
Merge branch '21126-trash-when-ro'
refs #21126
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 9386c8b344..844e67bfcc 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -1714,6 +1714,11 @@ Clusters:
ReadOnly: false
"http://host1.example:25107": {}
ReadOnly: false
+ # AllowTrashWhenReadOnly enables unused and overreplicated
+ # blocks to be trashed/deleted even when ReadOnly is
+ # true. Normally, this is false and ReadOnly prevents all
+ # trash/delete operations as well as writes.
+ AllowTrashWhenReadOnly: false
Replication: 1
StorageClasses:
# If you have configured storage classes (see StorageClasses
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 6a71078bb1..e2ad7b089d 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -319,12 +319,13 @@ type StorageClassConfig struct {
}
type Volume struct {
- AccessViaHosts map[URL]VolumeAccess
- ReadOnly bool
- Replication int
- StorageClasses map[string]bool
- Driver string
- DriverParameters json.RawMessage
+ AccessViaHosts map[URL]VolumeAccess
+ ReadOnly bool
+ AllowTrashWhenReadOnly bool
+ Replication int
+ StorageClasses map[string]bool
+ Driver string
+ DriverParameters json.RawMessage
}
type S3VolumeDriverParameters struct {
diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go
index 5b6d71a4fb..85750d8cfc 100644
--- a/sdk/go/arvados/keep_service.go
+++ b/sdk/go/arvados/keep_service.go
@@ -33,7 +33,8 @@ type KeepService struct {
type KeepMount struct {
UUID string `json:"uuid"`
DeviceID string `json:"device_id"`
- ReadOnly bool `json:"read_only"`
+ AllowWrite bool `json:"allow_write"`
+ AllowTrash bool `json:"allow_trash"`
Replication int `json:"replication"`
StorageClasses map[string]bool `json:"storage_classes"`
}
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index e4bf0f0d61..e71eb07efa 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -228,7 +228,7 @@ func (bal *Balancer) cleanupMounts() {
rwdev := map[string]*KeepService{}
for _, srv := range bal.KeepServices {
for _, mnt := range srv.mounts {
- if !mnt.ReadOnly {
+ if mnt.AllowWrite {
rwdev[mnt.UUID] = srv
}
}
@@ -238,7 +238,7 @@ func (bal *Balancer) cleanupMounts() {
for _, srv := range bal.KeepServices {
var dedup []*KeepMount
for _, mnt := range srv.mounts {
- if mnt.ReadOnly && rwdev[mnt.UUID] != nil {
+ if !mnt.AllowWrite && rwdev[mnt.UUID] != nil {
bal.logf("skipping srv %s readonly mount %q because same volume is mounted read-write on srv %s", srv, mnt.UUID, rwdev[mnt.UUID])
} else {
dedup = append(dedup, mnt)
@@ -587,9 +587,11 @@ func (bal *Balancer) setupLookupTables(cluster *arvados.Cluster) {
for _, mnt := range srv.mounts {
bal.mounts++
- // All mounts on a read-only service are
- // effectively read-only.
- mnt.ReadOnly = mnt.ReadOnly || srv.ReadOnly
+ if srv.ReadOnly {
+ // All mounts on a read-only service
+ // are effectively read-only.
+ mnt.AllowWrite = false
+ }
for class := range mnt.StorageClasses {
if mbc := bal.mountsByClass[class]; mbc == nil {
@@ -674,7 +676,7 @@ func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) ba
slots = append(slots, slot{
mnt: mnt,
repl: repl,
- want: repl != nil && mnt.ReadOnly,
+ want: repl != nil && !mnt.AllowTrash,
})
}
}
@@ -763,7 +765,7 @@ func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) ba
protMnt[slot.mnt] = true
replProt += slot.mnt.Replication
}
- if replWant < desired && (slot.repl != nil || !slot.mnt.ReadOnly) {
+ if replWant < desired && (slot.repl != nil || slot.mnt.AllowWrite) {
slots[i].want = true
wantSrv[slot.mnt.KeepService] = true
wantMnt[slot.mnt] = true
@@ -882,7 +884,7 @@ func (bal *Balancer) balanceBlock(blkid arvados.SizedDigest, blk *BlockState) ba
case slot.repl == nil && slot.want && len(blk.Replicas) == 0:
lost = true
change = changeNone
- case slot.repl == nil && slot.want && !slot.mnt.ReadOnly:
+ case slot.repl == nil && slot.want && slot.mnt.AllowWrite:
slot.mnt.KeepService.AddPull(Pull{
SizedDigest: blkid,
From: blk.Replicas[0].KeepMount.KeepService,
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index ea2bfa3f0b..b7b3fb6123 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -89,21 +89,29 @@ var stubMounts = map[string][]arvados.KeepMount{
UUID: "zzzzz-ivpuk-000000000000000",
DeviceID: "keep0-vol0",
StorageClasses: map[string]bool{"default": true},
+ AllowWrite: true,
+ AllowTrash: true,
}},
"keep1.zzzzz.arvadosapi.com:25107": {{
UUID: "zzzzz-ivpuk-100000000000000",
DeviceID: "keep1-vol0",
StorageClasses: map[string]bool{"default": true},
+ AllowWrite: true,
+ AllowTrash: true,
}},
"keep2.zzzzz.arvadosapi.com:25107": {{
UUID: "zzzzz-ivpuk-200000000000000",
DeviceID: "keep2-vol0",
StorageClasses: map[string]bool{"default": true},
+ AllowWrite: true,
+ AllowTrash: true,
}},
"keep3.zzzzz.arvadosapi.com:25107": {{
UUID: "zzzzz-ivpuk-300000000000000",
DeviceID: "keep3-vol0",
StorageClasses: map[string]bool{"default": true},
+ AllowWrite: true,
+ AllowTrash: true,
}},
}
diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index cf92a7cabf..85d4ff8b5d 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -94,6 +94,8 @@ func (bal *balancerSuite) SetUpTest(c *check.C) {
KeepMount: arvados.KeepMount{
UUID: fmt.Sprintf("zzzzz-mount-%015x", i),
StorageClasses: map[string]bool{"default": true},
+ AllowWrite: true,
+ AllowTrash: true,
},
KeepService: srv,
}}
@@ -160,15 +162,53 @@ func (bal *balancerSuite) TestSkipReadonly(c *check.C) {
}})
}
+func (bal *balancerSuite) TestAllowTrashWhenReadOnly(c *check.C) {
+ srvs := bal.srvList(0, slots{3})
+ srvs[0].mounts[0].KeepMount.AllowWrite = false
+ srvs[0].mounts[0].KeepMount.AllowTrash = true
+ // can't pull to slot 3, so pull to slot 4 instead
+ bal.try(c, tester{
+ desired: map[string]int{"default": 4},
+ current: slots{0, 1},
+ shouldPull: slots{2, 4},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ pulling: 2,
+ }})
+ // expect to be able to trash slot 3 in future, so pull to
+ // slot 1
+ bal.try(c, tester{
+ desired: map[string]int{"default": 2},
+ current: slots{0, 3},
+ shouldPull: slots{1},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ pulling: 1,
+ }})
+ // trash excess from slot 3
+ bal.try(c, tester{
+ desired: map[string]int{"default": 2},
+ current: slots{0, 1, 3},
+ shouldTrash: slots{3},
+ expectBlockState: &balancedBlockState{
+ needed: 2,
+ unneeded: 1,
+ }})
+}
+
func (bal *balancerSuite) TestMultipleViewsReadOnly(c *check.C) {
- bal.testMultipleViews(c, true)
+ bal.testMultipleViews(c, false, false)
+}
+
+func (bal *balancerSuite) TestMultipleViewsReadOnlyAllowTrash(c *check.C) {
+ bal.testMultipleViews(c, false, true)
}
func (bal *balancerSuite) TestMultipleViews(c *check.C) {
- bal.testMultipleViews(c, false)
+ bal.testMultipleViews(c, true, true)
}
-func (bal *balancerSuite) testMultipleViews(c *check.C, readonly bool) {
+func (bal *balancerSuite) testMultipleViews(c *check.C, allowWrite, allowTrash bool) {
for i, srv := range bal.srvs {
// Add a mount to each service
srv.mounts[0].KeepMount.DeviceID = fmt.Sprintf("writable-by-srv-%x", i)
@@ -176,7 +216,8 @@ func (bal *balancerSuite) testMultipleViews(c *check.C, readonly bool) {
KeepMount: arvados.KeepMount{
DeviceID: bal.srvs[(i+1)%len(bal.srvs)].mounts[0].KeepMount.DeviceID,
UUID: bal.srvs[(i+1)%len(bal.srvs)].mounts[0].KeepMount.UUID,
- ReadOnly: readonly,
+ AllowWrite: allowWrite,
+ AllowTrash: allowTrash,
Replication: 1,
StorageClasses: map[string]bool{"default": true},
},
@@ -195,11 +236,12 @@ func (bal *balancerSuite) testMultipleViews(c *check.C, readonly bool) {
desired: map[string]int{"default": 1},
current: slots{0, i, i},
shouldTrash: slots{i}})
- } else if readonly {
+ } else if !allowTrash {
// Timestamps are all different, and the third
// replica can't be trashed because it's on a
- // read-only mount, so the first two replicas
- // should be trashed.
+ // read-only mount (with
+ // AllowTrashWhenReadOnly=false), so the first
+ // two replicas should be trashed.
bal.try(c, tester{
desired: map[string]int{"default": 1},
current: slots{0, i, i},
@@ -381,7 +423,7 @@ func (bal *balancerSuite) TestDecreaseReplBlockTooNew(c *check.C) {
}
func (bal *balancerSuite) TestCleanupMounts(c *check.C) {
- bal.srvs[3].mounts[0].KeepMount.ReadOnly = true
+ bal.srvs[3].mounts[0].KeepMount.AllowWrite = false
bal.srvs[3].mounts[0].KeepMount.DeviceID = "abcdef"
bal.srvs[14].mounts[0].KeepMount.UUID = bal.srvs[3].mounts[0].KeepMount.UUID
bal.srvs[14].mounts[0].KeepMount.DeviceID = "abcdef"
@@ -590,6 +632,8 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
// classes=[special,special2].
bal.srvs[9].mounts = []*KeepMount{{
KeepMount: arvados.KeepMount{
+ AllowWrite: true,
+ AllowTrash: true,
Replication: 1,
StorageClasses: map[string]bool{"special": true},
UUID: "zzzzz-mount-special00000009",
@@ -598,6 +642,8 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
KeepService: bal.srvs[9],
}, {
KeepMount: arvados.KeepMount{
+ AllowWrite: true,
+ AllowTrash: true,
Replication: 1,
StorageClasses: map[string]bool{"special": true, "special2": true},
UUID: "zzzzz-mount-special20000009",
@@ -610,6 +656,8 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
// classes=[special3], one with classes=[default].
bal.srvs[13].mounts = []*KeepMount{{
KeepMount: arvados.KeepMount{
+ AllowWrite: true,
+ AllowTrash: true,
Replication: 1,
StorageClasses: map[string]bool{"special2": true},
UUID: "zzzzz-mount-special2000000d",
@@ -618,6 +666,8 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
KeepService: bal.srvs[13],
}, {
KeepMount: arvados.KeepMount{
+ AllowWrite: true,
+ AllowTrash: true,
Replication: 1,
StorageClasses: map[string]bool{"default": true},
UUID: "zzzzz-mount-00000000000000d",
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index f9b383e70e..56a52c913a 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -480,10 +480,9 @@ func (v *AzureBlobVolume) listBlobs(page int, params storage.ListBlobsParameters
// Trash a Keep block.
func (v *AzureBlobVolume) Trash(loc string) error {
- if v.volume.ReadOnly {
+ if v.volume.ReadOnly && !v.volume.AllowTrashWhenReadOnly {
return MethodDisabledError
}
-
// Ideally we would use If-Unmodified-Since, but that
// particular condition seems to be ignored by Azure. Instead,
// we get the Etag before checking Mtime, and use If-Match to
@@ -575,10 +574,6 @@ func (v *AzureBlobVolume) isKeepBlock(s string) bool {
// EmptyTrash looks for trashed blocks that exceeded BlobTrashLifetime
// and deletes them from the volume.
func (v *AzureBlobVolume) EmptyTrash() {
- if v.cluster.Collections.BlobDeleteConcurrency < 1 {
- return
- }
-
var bytesDeleted, bytesInTrash int64
var blocksDeleted, blocksInTrash int64
diff --git a/services/keepstore/command.go b/services/keepstore/command.go
index 555f16dfe1..db387f494b 100644
--- a/services/keepstore/command.go
+++ b/services/keepstore/command.go
@@ -186,7 +186,7 @@ func (h *handler) setup(ctx context.Context, cluster *arvados.Cluster, token str
// Initialize the trashq and workers
h.trashq = NewWorkQueue()
- for i := 0; i < 1 || i < h.Cluster.Collections.BlobTrashConcurrency; i++ {
+ for i := 0; i < h.Cluster.Collections.BlobTrashConcurrency; i++ {
go RunTrashWorker(h.volmgr, h.Logger, h.Cluster, h.trashq)
}
@@ -208,8 +208,10 @@ func (h *handler) setup(ctx context.Context, cluster *arvados.Cluster, token str
}
h.keepClient.Arvados.ApiToken = fmt.Sprintf("%x", rand.Int63())
- if d := h.Cluster.Collections.BlobTrashCheckInterval.Duration(); d > 0 {
- go emptyTrash(h.volmgr.writables, d)
+ if d := h.Cluster.Collections.BlobTrashCheckInterval.Duration(); d > 0 &&
+ h.Cluster.Collections.BlobTrash &&
+ h.Cluster.Collections.BlobDeleteConcurrency > 0 {
+ go emptyTrash(h.volmgr.mounts, d)
}
return nil
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 60fdde89c7..abeb20fe86 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -475,8 +475,10 @@ func (rtr *router) handleDELETE(resp http.ResponseWriter, req *http.Request) {
Deleted int `json:"copies_deleted"`
Failed int `json:"copies_failed"`
}
- for _, vol := range rtr.volmgr.AllWritable() {
- if err := vol.Trash(hash); err == nil {
+ for _, vol := range rtr.volmgr.Mounts() {
+ if !vol.KeepMount.AllowTrash {
+ continue
+ } else if err := vol.Trash(hash); err == nil {
result.Deleted++
} else if os.IsNotExist(err) {
continue
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index f83ba603d2..953aa047cb 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -49,7 +49,9 @@ func (e *KeepError) Error() string {
func emptyTrash(mounts []*VolumeMount, interval time.Duration) {
for range time.NewTicker(interval).C {
for _, v := range mounts {
- v.EmptyTrash()
+ if v.KeepMount.AllowTrash {
+ v.EmptyTrash()
+ }
}
}
}
diff --git a/services/keepstore/s3aws_volume.go b/services/keepstore/s3aws_volume.go
index 8f2c275391..18b30f4638 100644
--- a/services/keepstore/s3aws_volume.go
+++ b/services/keepstore/s3aws_volume.go
@@ -295,10 +295,6 @@ func (v *S3AWSVolume) Compare(ctx context.Context, loc string, expect []byte) er
// EmptyTrash looks for trashed blocks that exceeded BlobTrashLifetime
// and deletes them from the volume.
func (v *S3AWSVolume) EmptyTrash() {
- if v.cluster.Collections.BlobDeleteConcurrency < 1 {
- return
- }
-
var bytesInTrash, blocksInTrash, bytesDeleted, blocksDeleted int64
// Define "ready to delete" as "...when EmptyTrash started".
@@ -850,7 +846,7 @@ func (b *s3AWSbucket) Del(path string) error {
// Trash a Keep block.
func (v *S3AWSVolume) Trash(loc string) error {
- if v.volume.ReadOnly {
+ if v.volume.ReadOnly && !v.volume.AllowTrashWhenReadOnly {
return MethodDisabledError
}
if t, err := v.Mtime(loc); err != nil {
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index d7c73127eb..5e8a5a963c 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -36,10 +36,12 @@ func TrashItem(volmgr *RRVolumeManager, logger logrus.FieldLogger, cluster *arva
var volumes []*VolumeMount
if uuid := trashRequest.MountUUID; uuid == "" {
- volumes = volmgr.AllWritable()
- } else if mnt := volmgr.Lookup(uuid, true); mnt == nil {
+ volumes = volmgr.Mounts()
+ } else if mnt := volmgr.Lookup(uuid, false); mnt == nil {
logger.Warnf("trash request for nonexistent mount: %v", trashRequest)
return
+ } else if !mnt.KeepMount.AllowTrash {
+ logger.Warnf("trash request for mount with ReadOnly=true, AllowTrashWhenReadOnly=false: %v", trashRequest)
} else {
volumes = []*VolumeMount{mnt}
}
diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go
index a08b7de01a..dee4bdc1c1 100644
--- a/services/keepstore/unix_volume.go
+++ b/services/keepstore/unix_volume.go
@@ -442,8 +442,7 @@ func (v *UnixVolume) Trash(loc string) error {
// be re-written), or (b) Touch() will update the file's timestamp and
// Trash() will read the correct up-to-date timestamp and choose not to
// trash the file.
-
- if v.volume.ReadOnly || !v.cluster.Collections.BlobTrash {
+ if v.volume.ReadOnly && !v.volume.AllowTrashWhenReadOnly {
return MethodDisabledError
}
if err := v.lock(context.TODO()); err != nil {
@@ -647,10 +646,6 @@ var unixTrashLocRegexp = regexp.MustCompile(`/([0-9a-f]{32})\.trash\.(\d+)$`)
// EmptyTrash walks hierarchy looking for {hash}.trash.*
// and deletes those with deadline < now.
func (v *UnixVolume) EmptyTrash() {
- if v.cluster.Collections.BlobDeleteConcurrency < 1 {
- return
- }
-
var bytesDeleted, bytesInTrash int64
var blocksDeleted, blocksInTrash int64
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index c3b8cd6283..f597ff5781 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -316,8 +316,6 @@ func makeRRVolumeManager(logger logrus.FieldLogger, cluster *arvados.Cluster, my
if err != nil {
return nil, fmt.Errorf("error initializing volume %s: %s", uuid, err)
}
- logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly || va.ReadOnly)
-
sc := cfgvol.StorageClasses
if len(sc) == 0 {
sc = map[string]bool{"default": true}
@@ -330,7 +328,8 @@ func makeRRVolumeManager(logger logrus.FieldLogger, cluster *arvados.Cluster, my
KeepMount: arvados.KeepMount{
UUID: uuid,
DeviceID: vol.GetDeviceID(),
- ReadOnly: cfgvol.ReadOnly || va.ReadOnly,
+ AllowWrite: !va.ReadOnly && !cfgvol.ReadOnly,
+ AllowTrash: !va.ReadOnly && (!cfgvol.ReadOnly || cfgvol.AllowTrashWhenReadOnly),
Replication: repl,
StorageClasses: sc,
},
@@ -340,9 +339,10 @@ func makeRRVolumeManager(logger logrus.FieldLogger, cluster *arvados.Cluster, my
vm.mounts = append(vm.mounts, mnt)
vm.mountMap[uuid] = mnt
vm.readables = append(vm.readables, mnt)
- if !mnt.KeepMount.ReadOnly {
+ if mnt.KeepMount.AllowWrite {
vm.writables = append(vm.writables, mnt)
}
+ logger.Printf("started volume %s (%s), AllowWrite=%v, AllowTrash=%v", uuid, vol, mnt.AllowWrite, mnt.AllowTrash)
}
// pri(mnt): return highest priority of any storage class
// offered by mnt
@@ -382,7 +382,7 @@ func (vm *RRVolumeManager) Mounts() []*VolumeMount {
}
func (vm *RRVolumeManager) Lookup(uuid string, needWrite bool) *VolumeMount {
- if mnt, ok := vm.mountMap[uuid]; ok && (!needWrite || !mnt.ReadOnly) {
+ if mnt, ok := vm.mountMap[uuid]; ok && (!needWrite || mnt.AllowWrite) {
return mnt
}
return nil
commit 9405216718cc96971347d2ad6f13cb175923388e
Author: Tom Clegg <tom at curii.com>
Date: Mon Nov 20 14:04:26 2023 -0500
Merge branch '21189-changeset-limit'
refs #21189
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/cmd_test.go b/cmd/arvados-client/cmd_test.go
index cbbc7b1f95..911375c655 100644
--- a/cmd/arvados-client/cmd_test.go
+++ b/cmd/arvados-client/cmd_test.go
@@ -9,6 +9,7 @@ import (
"io/ioutil"
"testing"
+ "git.arvados.org/arvados.git/lib/cmd"
check "gopkg.in/check.v1"
)
@@ -23,12 +24,12 @@ type ClientSuite struct{}
func (s *ClientSuite) TestBadCommand(c *check.C) {
exited := handler.RunCommand("arvados-client", []string{"no such command"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard)
- c.Check(exited, check.Equals, 2)
+ c.Check(exited, check.Equals, cmd.EXIT_INVALIDARGUMENT)
}
func (s *ClientSuite) TestBadSubcommandArgs(c *check.C) {
exited := handler.RunCommand("arvados-client", []string{"get"}, bytes.NewReader(nil), ioutil.Discard, ioutil.Discard)
- c.Check(exited, check.Equals, 2)
+ c.Check(exited, check.Equals, cmd.EXIT_INVALIDARGUMENT)
}
func (s *ClientSuite) TestVersion(c *check.C) {
diff --git a/cmd/arvados-server/arvados-controller.service b/cmd/arvados-server/arvados-controller.service
index 420cbb035a..f96532de5e 100644
--- a/cmd/arvados-server/arvados-controller.service
+++ b/cmd/arvados-server/arvados-controller.service
@@ -19,6 +19,7 @@ ExecStart=/usr/bin/arvados-controller
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/arvados-dispatch-cloud.service b/cmd/arvados-server/arvados-dispatch-cloud.service
index 8d57e8a161..11887b8f8c 100644
--- a/cmd/arvados-server/arvados-dispatch-cloud.service
+++ b/cmd/arvados-server/arvados-dispatch-cloud.service
@@ -19,6 +19,7 @@ ExecStart=/usr/bin/arvados-dispatch-cloud
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/arvados-dispatch-lsf.service b/cmd/arvados-server/arvados-dispatch-lsf.service
index 65d8786670..f90cd9033d 100644
--- a/cmd/arvados-server/arvados-dispatch-lsf.service
+++ b/cmd/arvados-server/arvados-dispatch-lsf.service
@@ -19,6 +19,7 @@ ExecStart=/usr/bin/arvados-dispatch-lsf
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/arvados-git-httpd.service b/cmd/arvados-server/arvados-git-httpd.service
index b45587ffc0..6e5b0dc8e2 100644
--- a/cmd/arvados-server/arvados-git-httpd.service
+++ b/cmd/arvados-server/arvados-git-httpd.service
@@ -19,6 +19,7 @@ ExecStart=/usr/bin/arvados-git-httpd
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/arvados-health.service b/cmd/arvados-server/arvados-health.service
index cf246b0ee2..ef145e26eb 100644
--- a/cmd/arvados-server/arvados-health.service
+++ b/cmd/arvados-server/arvados-health.service
@@ -19,6 +19,7 @@ ExecStart=/usr/bin/arvados-health
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/arvados-ws.service b/cmd/arvados-server/arvados-ws.service
index f73db5d080..2e88449599 100644
--- a/cmd/arvados-server/arvados-ws.service
+++ b/cmd/arvados-server/arvados-ws.service
@@ -18,6 +18,7 @@ ExecStart=/usr/bin/arvados-ws
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/crunch-dispatch-slurm.service b/cmd/arvados-server/crunch-dispatch-slurm.service
index 51b4e58c35..d2a2fb39d9 100644
--- a/cmd/arvados-server/crunch-dispatch-slurm.service
+++ b/cmd/arvados-server/crunch-dispatch-slurm.service
@@ -19,6 +19,7 @@ ExecStart=/usr/bin/crunch-dispatch-slurm
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/keep-balance.service b/cmd/arvados-server/keep-balance.service
index 1c5808288b..f282f0a650 100644
--- a/cmd/arvados-server/keep-balance.service
+++ b/cmd/arvados-server/keep-balance.service
@@ -14,12 +14,13 @@ StartLimitIntervalSec=0
[Service]
Type=notify
EnvironmentFile=-/etc/arvados/environment
-ExecStart=/usr/bin/keep-balance -commit-pulls -commit-trash
+ExecStart=/usr/bin/keep-balance
# Set a reasonable default for the open file limit
LimitNOFILE=65536
Restart=always
RestartSec=10s
Nice=19
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/keep-web.service b/cmd/arvados-server/keep-web.service
index c0e193d6d8..4ecd0b4978 100644
--- a/cmd/arvados-server/keep-web.service
+++ b/cmd/arvados-server/keep-web.service
@@ -19,6 +19,7 @@ ExecStart=/usr/bin/keep-web
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/keepproxy.service b/cmd/arvados-server/keepproxy.service
index 7d4d092677..139df1c3fa 100644
--- a/cmd/arvados-server/keepproxy.service
+++ b/cmd/arvados-server/keepproxy.service
@@ -19,6 +19,7 @@ ExecStart=/usr/bin/keepproxy
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/cmd/arvados-server/keepstore.service b/cmd/arvados-server/keepstore.service
index bcfde3a788..de0fd1dbd7 100644
--- a/cmd/arvados-server/keepstore.service
+++ b/cmd/arvados-server/keepstore.service
@@ -23,6 +23,7 @@ ExecStart=/usr/bin/keepstore
LimitNOFILE=65536
Restart=always
RestartSec=1
+RestartPreventExitStatus=2
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
diff --git a/lib/cli/get.go b/lib/cli/get.go
index 9625214e22..352e7b9af6 100644
--- a/lib/cli/get.go
+++ b/lib/cli/get.go
@@ -30,12 +30,12 @@ func (getCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, st
flags.SetOutput(stderr)
err = flags.Parse(args)
if err != nil {
- return 2
+ return cmd.EXIT_INVALIDARGUMENT
}
if len(flags.Args()) != 1 {
fmt.Fprintf(stderr, "usage of %s:\n", prog)
flags.PrintDefaults()
- return 2
+ return cmd.EXIT_INVALIDARGUMENT
}
if opts.Short {
opts.Format = "uuid"
diff --git a/lib/cmd/cmd.go b/lib/cmd/cmd.go
index 3d4092e6b8..cb2b4455d2 100644
--- a/lib/cmd/cmd.go
+++ b/lib/cmd/cmd.go
@@ -20,6 +20,8 @@ import (
"github.com/sirupsen/logrus"
)
+const EXIT_INVALIDARGUMENT = 2
+
type Handler interface {
RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int
}
@@ -86,13 +88,13 @@ func (m Multi) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
} else if len(args) < 1 {
fmt.Fprintf(stderr, "usage: %s command [args]\n", prog)
m.Usage(stderr)
- return 2
+ return EXIT_INVALIDARGUMENT
} else if cmd, ok = m[args[0]]; ok {
return cmd.RunCommand(prog+" "+args[0], args[1:], stdin, stdout, stderr)
} else {
fmt.Fprintf(stderr, "%s: unrecognized command %q\n", prog, args[0])
m.Usage(stderr)
- return 2
+ return EXIT_INVALIDARGUMENT
}
}
diff --git a/lib/cmd/parseflags.go b/lib/cmd/parseflags.go
index 707cacbf52..275e063f31 100644
--- a/lib/cmd/parseflags.go
+++ b/lib/cmd/parseflags.go
@@ -35,7 +35,7 @@ func ParseFlags(f FlagSet, prog string, args []string, positional string, stderr
case nil:
if f.NArg() > 0 && positional == "" {
fmt.Fprintf(stderr, "unrecognized command line arguments: %v (try -help)\n", f.Args())
- return false, 2
+ return false, EXIT_INVALIDARGUMENT
}
return true, 0
case flag.ErrHelp:
@@ -55,6 +55,6 @@ func ParseFlags(f FlagSet, prog string, args []string, positional string, stderr
return false, 0
default:
fmt.Fprintf(stderr, "error parsing command line arguments: %s (try -help)\n", err)
- return false, 2
+ return false, EXIT_INVALIDARGUMENT
}
}
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index 9503a54d2d..53f677796d 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -33,7 +33,7 @@ func (s *CommandSuite) SetUpSuite(c *check.C) {
func (s *CommandSuite) TestDump_BadArg(c *check.C) {
var stderr bytes.Buffer
code := DumpCommand.RunCommand("arvados config-dump", []string{"-badarg"}, bytes.NewBuffer(nil), bytes.NewBuffer(nil), &stderr)
- c.Check(code, check.Equals, 2)
+ c.Check(code, check.Equals, cmd.EXIT_INVALIDARGUMENT)
c.Check(stderr.String(), check.Equals, "error parsing command line arguments: flag provided but not defined: -badarg (try -help)\n")
}
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index a633216be7..9386c8b344 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -638,6 +638,15 @@ Clusters:
# once.
BalanceUpdateLimit: 100000
+ # Maximum number of "pull block from other server" and "trash
+ # block" requests to send to each keepstore server at a
+ # time. Smaller values use less memory in keepstore and
+ # keep-balance. Larger values allow more progress per
+ # keep-balance iteration. A zero value computes all of the
+ # needed changes but does not apply any.
+ BalancePullLimit: 100000
+ BalanceTrashLimit: 100000
+
# Default lifetime for ephemeral collections: 2 weeks. This must not
# be less than BlobSigningTTL.
DefaultTrashLifetime: 336h
diff --git a/lib/config/export.go b/lib/config/export.go
index cbd9ff6d7f..674b37473e 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -93,7 +93,9 @@ var whitelist = map[string]bool{
"Collections.BalanceCollectionBatch": false,
"Collections.BalanceCollectionBuffers": false,
"Collections.BalancePeriod": false,
+ "Collections.BalancePullLimit": false,
"Collections.BalanceTimeout": false,
+ "Collections.BalanceTrashLimit": false,
"Collections.BalanceUpdateLimit": false,
"Collections.BlobDeleteConcurrency": false,
"Collections.BlobMissingReport": false,
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 64424fc3e9..6a71078bb1 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -150,6 +150,8 @@ type Cluster struct {
BalanceCollectionBuffers int
BalanceTimeout Duration
BalanceUpdateLimit int
+ BalancePullLimit int
+ BalanceTrashLimit int
WebDAVCache WebDAVCacheConfig
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 215c5e1b5b..e4bf0f0d61 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -137,7 +137,7 @@ func (bal *Balancer) Run(ctx context.Context, client *arvados.Client, cluster *a
client.Timeout = 0
rs := bal.rendezvousState()
- if runOptions.CommitTrash && rs != runOptions.SafeRendezvousState {
+ if cluster.Collections.BalanceTrashLimit > 0 && rs != runOptions.SafeRendezvousState {
if runOptions.SafeRendezvousState != "" {
bal.logf("notice: KeepServices list has changed since last run")
}
@@ -155,6 +155,7 @@ func (bal *Balancer) Run(ctx context.Context, client *arvados.Client, cluster *a
if err = bal.GetCurrentState(ctx, client, cluster.Collections.BalanceCollectionBatch, cluster.Collections.BalanceCollectionBuffers); err != nil {
return
}
+ bal.setupLookupTables(cluster)
bal.ComputeChangeSets()
bal.PrintStatistics()
if err = bal.CheckSanityLate(); err != nil {
@@ -171,14 +172,14 @@ func (bal *Balancer) Run(ctx context.Context, client *arvados.Client, cluster *a
}
lbFile = nil
}
- if runOptions.CommitPulls {
+ if cluster.Collections.BalancePullLimit > 0 {
err = bal.CommitPulls(ctx, client)
if err != nil {
// Skip trash if we can't pull. (Too cautious?)
return
}
}
- if runOptions.CommitTrash {
+ if cluster.Collections.BalanceTrashLimit > 0 {
err = bal.CommitTrash(ctx, client)
if err != nil {
return
@@ -542,7 +543,6 @@ func (bal *Balancer) ComputeChangeSets() {
// This just calls balanceBlock() once for each block, using a
// pool of worker goroutines.
defer bal.time("changeset_compute", "wall clock time to compute changesets")()
- bal.setupLookupTables()
type balanceTask struct {
blkid arvados.SizedDigest
@@ -577,7 +577,7 @@ func (bal *Balancer) ComputeChangeSets() {
bal.collectStatistics(results)
}
-func (bal *Balancer) setupLookupTables() {
+func (bal *Balancer) setupLookupTables(cluster *arvados.Cluster) {
bal.serviceRoots = make(map[string]string)
bal.classes = defaultClasses
bal.mountsByClass = map[string]map[*KeepMount]bool{"default": {}}
@@ -607,6 +607,13 @@ func (bal *Balancer) setupLookupTables() {
// class" case in balanceBlock depends on the order classes
// are considered.
sort.Strings(bal.classes)
+
+ for _, srv := range bal.KeepServices {
+ srv.ChangeSet = &ChangeSet{
+ PullLimit: cluster.Collections.BalancePullLimit,
+ TrashLimit: cluster.Collections.BalanceTrashLimit,
+ }
+ }
}
const (
@@ -955,19 +962,21 @@ type replicationStats struct {
}
type balancerStats struct {
- lost blocksNBytes
- overrep blocksNBytes
- unref blocksNBytes
- garbage blocksNBytes
- underrep blocksNBytes
- unachievable blocksNBytes
- justright blocksNBytes
- desired blocksNBytes
- current blocksNBytes
- pulls int
- trashes int
- replHistogram []int
- classStats map[string]replicationStats
+ lost blocksNBytes
+ overrep blocksNBytes
+ unref blocksNBytes
+ garbage blocksNBytes
+ underrep blocksNBytes
+ unachievable blocksNBytes
+ justright blocksNBytes
+ desired blocksNBytes
+ current blocksNBytes
+ pulls int
+ pullsDeferred int
+ trashes int
+ trashesDeferred int
+ replHistogram []int
+ classStats map[string]replicationStats
// collectionBytes / collectionBlockBytes = deduplication ratio
collectionBytes int64 // sum(bytes in referenced blocks) across all collections
@@ -1090,7 +1099,9 @@ func (bal *Balancer) collectStatistics(results <-chan balanceResult) {
}
for _, srv := range bal.KeepServices {
s.pulls += len(srv.ChangeSet.Pulls)
+ s.pullsDeferred += srv.ChangeSet.PullsDeferred
s.trashes += len(srv.ChangeSet.Trashes)
+ s.trashesDeferred += srv.ChangeSet.TrashesDeferred
}
bal.stats = s
bal.Metrics.UpdateStats(s)
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index db9a668481..ea2bfa3f0b 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -388,9 +388,7 @@ func (s *runSuite) TestRefuseZeroCollections(c *check.C) {
_, err := s.db.Exec(`delete from collections`)
c.Assert(err, check.IsNil)
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
- Logger: ctxlog.TestLogger(c),
+ Logger: ctxlog.TestLogger(c),
}
s.stub.serveCurrentUserAdmin()
s.stub.serveZeroCollections()
@@ -408,8 +406,6 @@ func (s *runSuite) TestRefuseZeroCollections(c *check.C) {
func (s *runSuite) TestRefuseBadIndex(c *check.C) {
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
ChunkPrefix: "abc",
Logger: ctxlog.TestLogger(c),
}
@@ -431,9 +427,7 @@ func (s *runSuite) TestRefuseBadIndex(c *check.C) {
func (s *runSuite) TestRefuseNonAdmin(c *check.C) {
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
- Logger: ctxlog.TestLogger(c),
+ Logger: ctxlog.TestLogger(c),
}
s.stub.serveCurrentUserNotAdmin()
s.stub.serveZeroCollections()
@@ -460,8 +454,6 @@ func (s *runSuite) TestInvalidChunkPrefix(c *check.C) {
s.SetUpTest(c)
c.Logf("trying invalid prefix %q", trial.prefix)
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
ChunkPrefix: trial.prefix,
Logger: ctxlog.TestLogger(c),
}
@@ -481,9 +473,7 @@ func (s *runSuite) TestInvalidChunkPrefix(c *check.C) {
func (s *runSuite) TestRefuseSameDeviceDifferentVolumes(c *check.C) {
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
- Logger: ctxlog.TestLogger(c),
+ Logger: ctxlog.TestLogger(c),
}
s.stub.serveCurrentUserAdmin()
s.stub.serveZeroCollections()
@@ -511,9 +501,7 @@ func (s *runSuite) TestWriteLostBlocks(c *check.C) {
s.config.Collections.BlobMissingReport = lostf.Name()
defer os.Remove(lostf.Name())
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
- Logger: ctxlog.TestLogger(c),
+ Logger: ctxlog.TestLogger(c),
}
s.stub.serveCurrentUserAdmin()
s.stub.serveFooBarFileCollections()
@@ -532,10 +520,10 @@ func (s *runSuite) TestWriteLostBlocks(c *check.C) {
}
func (s *runSuite) TestDryRun(c *check.C) {
+ s.config.Collections.BalanceTrashLimit = 0
+ s.config.Collections.BalancePullLimit = 0
opts := RunOptions{
- CommitPulls: false,
- CommitTrash: false,
- Logger: ctxlog.TestLogger(c),
+ Logger: ctxlog.TestLogger(c),
}
s.stub.serveCurrentUserAdmin()
collReqs := s.stub.serveFooBarFileCollections()
@@ -553,7 +541,10 @@ func (s *runSuite) TestDryRun(c *check.C) {
}
c.Check(trashReqs.Count(), check.Equals, 0)
c.Check(pullReqs.Count(), check.Equals, 0)
- c.Check(bal.stats.pulls, check.Not(check.Equals), 0)
+ c.Check(bal.stats.pulls, check.Equals, 0)
+ c.Check(bal.stats.pullsDeferred, check.Not(check.Equals), 0)
+ c.Check(bal.stats.trashes, check.Equals, 0)
+ c.Check(bal.stats.trashesDeferred, check.Not(check.Equals), 0)
c.Check(bal.stats.underrep.replicas, check.Not(check.Equals), 0)
c.Check(bal.stats.overrep.replicas, check.Not(check.Equals), 0)
}
@@ -562,10 +553,8 @@ func (s *runSuite) TestCommit(c *check.C) {
s.config.Collections.BlobMissingReport = c.MkDir() + "/keep-balance-lost-blocks-test-"
s.config.ManagementToken = "xyzzy"
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
- Logger: ctxlog.TestLogger(c),
- Dumper: ctxlog.TestLogger(c),
+ Logger: ctxlog.TestLogger(c),
+ Dumper: ctxlog.TestLogger(c),
}
s.stub.serveCurrentUserAdmin()
s.stub.serveFooBarFileCollections()
@@ -600,8 +589,6 @@ func (s *runSuite) TestCommit(c *check.C) {
func (s *runSuite) TestChunkPrefix(c *check.C) {
s.config.Collections.BlobMissingReport = c.MkDir() + "/keep-balance-lost-blocks-test-"
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
ChunkPrefix: "ac", // catch "foo" but not "bar"
Logger: ctxlog.TestLogger(c),
Dumper: ctxlog.TestLogger(c),
@@ -631,10 +618,8 @@ func (s *runSuite) TestChunkPrefix(c *check.C) {
func (s *runSuite) TestRunForever(c *check.C) {
s.config.ManagementToken = "xyzzy"
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
- Logger: ctxlog.TestLogger(c),
- Dumper: ctxlog.TestLogger(c),
+ Logger: ctxlog.TestLogger(c),
+ Dumper: ctxlog.TestLogger(c),
}
s.stub.serveCurrentUserAdmin()
s.stub.serveFooBarFileCollections()
diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index f9fca1431b..cf92a7cabf 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -12,6 +12,7 @@ import (
"testing"
"time"
+ "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
check "gopkg.in/check.v1"
@@ -26,6 +27,7 @@ var _ = check.Suite(&balancerSuite{})
type balancerSuite struct {
Balancer
+ config *arvados.Cluster
srvs []*KeepService
blks map[string]tester
knownRendezvous [][]int
@@ -72,6 +74,11 @@ func (bal *balancerSuite) SetUpSuite(c *check.C) {
bal.signatureTTL = 3600
bal.Logger = ctxlog.TestLogger(c)
+
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.Equals, nil)
+ bal.config, err = cfg.GetCluster("")
+ c.Assert(err, check.Equals, nil)
}
func (bal *balancerSuite) SetUpTest(c *check.C) {
@@ -693,7 +700,7 @@ func (bal *balancerSuite) TestChangeStorageClasses(c *check.C) {
// the appropriate changes for that block have been added to the
// changesets.
func (bal *balancerSuite) try(c *check.C, t tester) {
- bal.setupLookupTables()
+ bal.setupLookupTables(bal.config)
blk := &BlockState{
Replicas: bal.replList(t.known, t.current),
Desired: t.desired,
@@ -701,9 +708,6 @@ func (bal *balancerSuite) try(c *check.C, t tester) {
for i, t := range t.timestamps {
blk.Replicas[i].Mtime = t
}
- for _, srv := range bal.srvs {
- srv.ChangeSet = &ChangeSet{}
- }
result := bal.balanceBlock(knownBlkid(t.known), blk)
var didPull, didTrash slots
diff --git a/services/keep-balance/change_set.go b/services/keep-balance/change_set.go
index 8e0ba028ac..c3579556bb 100644
--- a/services/keep-balance/change_set.go
+++ b/services/keep-balance/change_set.go
@@ -60,22 +60,35 @@ func (t Trash) MarshalJSON() ([]byte, error) {
// ChangeSet is a set of change requests that will be sent to a
// keepstore server.
type ChangeSet struct {
- Pulls []Pull
- Trashes []Trash
- mutex sync.Mutex
+ PullLimit int
+ TrashLimit int
+
+ Pulls []Pull
+ PullsDeferred int // number that weren't added because of PullLimit
+ Trashes []Trash
+ TrashesDeferred int // number that weren't added because of TrashLimit
+ mutex sync.Mutex
}
// AddPull adds a Pull operation.
func (cs *ChangeSet) AddPull(p Pull) {
cs.mutex.Lock()
- cs.Pulls = append(cs.Pulls, p)
+ if len(cs.Pulls) < cs.PullLimit {
+ cs.Pulls = append(cs.Pulls, p)
+ } else {
+ cs.PullsDeferred++
+ }
cs.mutex.Unlock()
}
// AddTrash adds a Trash operation
func (cs *ChangeSet) AddTrash(t Trash) {
cs.mutex.Lock()
- cs.Trashes = append(cs.Trashes, t)
+ if len(cs.Trashes) < cs.TrashLimit {
+ cs.Trashes = append(cs.Trashes, t)
+ } else {
+ cs.TrashesDeferred++
+ }
cs.mutex.Unlock()
}
@@ -83,5 +96,5 @@ func (cs *ChangeSet) AddTrash(t Trash) {
func (cs *ChangeSet) String() string {
cs.mutex.Lock()
defer cs.mutex.Unlock()
- return fmt.Sprintf("ChangeSet{Pulls:%d, Trashes:%d}", len(cs.Pulls), len(cs.Trashes))
+ return fmt.Sprintf("ChangeSet{Pulls:%d, Trashes:%d} Deferred{Pulls:%d Trashes:%d}", len(cs.Pulls), len(cs.Trashes), cs.PullsDeferred, cs.TrashesDeferred)
}
diff --git a/services/keep-balance/integration_test.go b/services/keep-balance/integration_test.go
index 42463a002a..2e353c92be 100644
--- a/services/keep-balance/integration_test.go
+++ b/services/keep-balance/integration_test.go
@@ -87,8 +87,6 @@ func (s *integrationSuite) TestBalanceAPIFixtures(c *check.C) {
logger := logrus.New()
logger.Out = io.MultiWriter(&logBuf, os.Stderr)
opts := RunOptions{
- CommitPulls: true,
- CommitTrash: true,
CommitConfirmedFields: true,
Logger: logger,
}
@@ -101,7 +99,6 @@ func (s *integrationSuite) TestBalanceAPIFixtures(c *check.C) {
nextOpts, err := bal.Run(context.Background(), s.client, s.config, opts)
c.Check(err, check.IsNil)
c.Check(nextOpts.SafeRendezvousState, check.Not(check.Equals), "")
- c.Check(nextOpts.CommitPulls, check.Equals, true)
if iter == 0 {
c.Check(logBuf.String(), check.Matches, `(?ms).*ChangeSet{Pulls:1.*`)
c.Check(logBuf.String(), check.Not(check.Matches), `(?ms).*ChangeSet{.*Trashes:[^0]}*`)
diff --git a/services/keep-balance/main.go b/services/keep-balance/main.go
index 6bc9989589..ec1cb18ee1 100644
--- a/services/keep-balance/main.go
+++ b/services/keep-balance/main.go
@@ -32,10 +32,10 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
flags.BoolVar(&options.Once, "once", false,
"balance once and then exit")
- flags.BoolVar(&options.CommitPulls, "commit-pulls", false,
- "send pull requests (make more replicas of blocks that are underreplicated or are not in optimal rendezvous probe order)")
- flags.BoolVar(&options.CommitTrash, "commit-trash", false,
- "send trash requests (delete unreferenced old blocks, and excess replicas of overreplicated blocks)")
+ deprCommitPulls := flags.Bool("commit-pulls", true,
+ "send pull requests (must be true -- configure Collections.BalancePullLimit = 0 to disable.)")
+ deprCommitTrash := flags.Bool("commit-trash", true,
+ "send trash requests (must be true -- configure Collections.BalanceTrashLimit = 0 to disable.)")
flags.BoolVar(&options.CommitConfirmedFields, "commit-confirmed-fields", true,
"update collection fields (replicas_confirmed, storage_classes_confirmed, etc.)")
flags.StringVar(&options.ChunkPrefix, "chunk-prefix", "",
@@ -55,6 +55,13 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
return code
}
+ if !*deprCommitPulls || !*deprCommitTrash {
+ fmt.Fprint(stderr,
+ "Usage error: the -commit-pulls or -commit-trash command line flags are no longer supported.\n",
+ "Use Collections.BalancePullLimit and Collections.BalanceTrashLimit instead.\n")
+ return cmd.EXIT_INVALIDARGUMENT
+ }
+
// Drop our custom args that would be rejected by the generic
// service.Command
args = nil
diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index 9bcaec43d8..b20144e3af 100644
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -27,8 +27,6 @@ import (
// RunOptions fields are controlled by command line flags.
type RunOptions struct {
Once bool
- CommitPulls bool
- CommitTrash bool
CommitConfirmedFields bool
ChunkPrefix string
Logger logrus.FieldLogger
@@ -112,9 +110,9 @@ func (srv *Server) runForever(ctx context.Context) error {
logger.Printf("starting up: will scan every %v and on SIGUSR1", srv.Cluster.Collections.BalancePeriod)
for {
- if !srv.RunOptions.CommitPulls && !srv.RunOptions.CommitTrash {
+ if srv.Cluster.Collections.BalancePullLimit < 1 && srv.Cluster.Collections.BalanceTrashLimit < 1 {
logger.Print("WARNING: Will scan periodically, but no changes will be committed.")
- logger.Print("======= Consider using -commit-pulls and -commit-trash flags.")
+ logger.Print("======= To commit changes, set BalancePullLimit and BalanceTrashLimit values greater than zero.")
}
if !dblock.KeepBalanceService.Check() {
commit d3cd6f7557ded90258d86a25aa755328f702e488
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Nov 13 14:35:52 2023 -0500
Merge branch '21021-controller-logout'
Closes #21021.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index b4b7476440..c65e142924 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -14,6 +14,7 @@ import (
"net/url"
"regexp"
"strings"
+ "sync"
"time"
"git.arvados.org/arvados.git/lib/config"
@@ -253,30 +254,51 @@ func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arva
return conn.local.Login(ctx, options)
}
+var v2TokenRegexp = regexp.MustCompile(`^v2/[a-z0-9]{5}-gj3su-[a-z0-9]{15}/`)
+
func (conn *Conn) Logout(ctx context.Context, options arvados.LogoutOptions) (arvados.LogoutResponse, error) {
- // If the logout request comes with an API token from a known
- // remote cluster, redirect to that cluster's logout handler
- // so it has an opportunity to clear sessions, expire tokens,
- // etc. Otherwise use the local endpoint.
- reqauth, ok := auth.FromContext(ctx)
- if !ok || len(reqauth.Tokens) == 0 || len(reqauth.Tokens[0]) < 8 || !strings.HasPrefix(reqauth.Tokens[0], "v2/") {
- return conn.local.Logout(ctx, options)
- }
- id := reqauth.Tokens[0][3:8]
- if id == conn.cluster.ClusterID {
- return conn.local.Logout(ctx, options)
- }
- remote, ok := conn.remotes[id]
- if !ok {
- return conn.local.Logout(ctx, options)
+ // If the token was issued by another cluster, we want to issue a logout
+ // request to the issuing instance to invalidate the token federation-wide.
+ // If this federation has a login cluster, that's always considered the
+ // issuing cluster.
+ // Otherwise, if this is a v2 token, use the UUID to find the issuing
+ // cluster.
+ // Note that remoteBE may still be conn.local even *after* one of these
+ // conditions is true.
+ var remoteBE backend = conn.local
+ if conn.cluster.Login.LoginCluster != "" {
+ remoteBE = conn.chooseBackend(conn.cluster.Login.LoginCluster)
+ } else {
+ reqauth, ok := auth.FromContext(ctx)
+ if ok && len(reqauth.Tokens) > 0 && v2TokenRegexp.MatchString(reqauth.Tokens[0]) {
+ remoteBE = conn.chooseBackend(reqauth.Tokens[0][3:8])
+ }
}
- baseURL := remote.BaseURL()
- target, err := baseURL.Parse(arvados.EndpointLogout.Path)
- if err != nil {
- return arvados.LogoutResponse{}, fmt.Errorf("internal error getting redirect target: %s", err)
+
+ // We always want to invalidate the token locally. Start that process.
+ var localResponse arvados.LogoutResponse
+ var localErr error
+ wg := sync.WaitGroup{}
+ wg.Add(1)
+ go func() {
+ localResponse, localErr = conn.local.Logout(ctx, options)
+ wg.Done()
+ }()
+
+ // If the token was issued by another cluster, log out there too.
+ if remoteBE != conn.local {
+ response, err := remoteBE.Logout(ctx, options)
+ // If the issuing cluster returns a redirect or error, that's more
+ // important to return to the user than anything that happens locally.
+ if response.RedirectLocation != "" || err != nil {
+ return response, err
+ }
}
- target.RawQuery = url.Values{"return_to": {options.ReturnTo}}.Encode()
- return arvados.LogoutResponse{RedirectLocation: target.String()}, nil
+
+ // Either the local cluster is the issuing cluster, or the issuing cluster's
+ // response was uninteresting.
+ wg.Wait()
+ return localResponse, localErr
}
func (conn *Conn) AuthorizedKeyCreate(ctx context.Context, options arvados.CreateOptions) (arvados.AuthorizedKey, error) {
diff --git a/lib/controller/federation/login_test.go b/lib/controller/federation/login_test.go
index a6743b320b..ab39619c79 100644
--- a/lib/controller/federation/login_test.go
+++ b/lib/controller/federation/login_test.go
@@ -8,10 +8,8 @@ import (
"context"
"net/url"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
- "git.arvados.org/arvados.git/sdk/go/auth"
check "gopkg.in/check.v1"
)
@@ -40,40 +38,3 @@ func (s *LoginSuite) TestDeferToLoginCluster(c *check.C) {
c.Check(remotePresent, check.Equals, remote != "")
}
}
-
-func (s *LoginSuite) TestLogout(c *check.C) {
- otherOrigin := arvados.URL{Scheme: "https", Host: "app.example.com", Path: "/"}
- otherURL := "https://app.example.com/foo"
- s.cluster.Services.Workbench1.ExternalURL = arvados.URL{Scheme: "https", Host: "workbench1.example.com"}
- s.cluster.Services.Workbench2.ExternalURL = arvados.URL{Scheme: "https", Host: "workbench2.example.com"}
- s.cluster.Login.TrustedClients = map[arvados.URL]struct{}{otherOrigin: {}}
- s.addHTTPRemote(c, "zhome", &arvadostest.APIStub{})
- s.cluster.Login.LoginCluster = "zhome"
- // s.fed is already set by SetUpTest, but we need to
- // reinitialize with the above config changes.
- s.fed = New(s.ctx, s.cluster, nil, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
-
- for _, trial := range []struct {
- token string
- returnTo string
- target string
- }{
- {token: "", returnTo: "", target: s.cluster.Services.Workbench2.ExternalURL.String()},
- {token: "", returnTo: otherURL, target: otherURL},
- {token: "zzzzzzzzzzzzzzzzzzzzz", returnTo: otherURL, target: otherURL},
- {token: "v2/zzzzz-aaaaa-aaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", returnTo: otherURL, target: otherURL},
- {token: "v2/zhome-aaaaa-aaaaaaaaaaaaaaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", returnTo: otherURL, target: "http://" + s.cluster.RemoteClusters["zhome"].Host + "/logout?" + url.Values{"return_to": {otherURL}}.Encode()},
- } {
- c.Logf("trial %#v", trial)
- ctx := s.ctx
- if trial.token != "" {
- ctx = auth.NewContext(ctx, &auth.Credentials{Tokens: []string{trial.token}})
- }
- resp, err := s.fed.Logout(ctx, arvados.LogoutOptions{ReturnTo: trial.returnTo})
- c.Assert(err, check.IsNil)
- c.Logf(" RedirectLocation %q", resp.RedirectLocation)
- target, err := url.Parse(resp.RedirectLocation)
- c.Check(err, check.IsNil)
- c.Check(target.String(), check.Equals, trial.target)
- }
-}
diff --git a/lib/controller/federation/logout_test.go b/lib/controller/federation/logout_test.go
new file mode 100644
index 0000000000..af6f6d9ed2
--- /dev/null
+++ b/lib/controller/federation/logout_test.go
@@ -0,0 +1,246 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package federation
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/url"
+
+ "git.arvados.org/arvados.git/lib/ctrlctx"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&LogoutSuite{})
+var emptyURL = &url.URL{}
+
+type LogoutStub struct {
+ arvadostest.APIStub
+ redirectLocation *url.URL
+}
+
+func (as *LogoutStub) CheckCalls(c *check.C, returnURL *url.URL) bool {
+ actual := as.APIStub.Calls(as.APIStub.Logout)
+ allOK := c.Check(actual, check.Not(check.HasLen), 0,
+ check.Commentf("Logout stub never called"))
+ expected := returnURL.String()
+ for _, call := range actual {
+ opts, ok := call.Options.(arvados.LogoutOptions)
+ allOK = c.Check(ok, check.Equals, true,
+ check.Commentf("call options were not LogoutOptions")) &&
+ c.Check(opts.ReturnTo, check.Equals, expected) &&
+ allOK
+ }
+ return allOK
+}
+
+func (as *LogoutStub) Logout(ctx context.Context, options arvados.LogoutOptions) (arvados.LogoutResponse, error) {
+ as.APIStub.Logout(ctx, options)
+ loc := as.redirectLocation.String()
+ if loc == "" {
+ loc = options.ReturnTo
+ }
+ return arvados.LogoutResponse{
+ RedirectLocation: loc,
+ }, as.Error
+}
+
+type LogoutSuite struct {
+ FederationSuite
+}
+
+func (s *LogoutSuite) badReturnURL(path string) *url.URL {
+ return &url.URL{
+ Scheme: "https",
+ Host: "example.net",
+ Path: path,
+ }
+}
+
+func (s *LogoutSuite) goodReturnURL(path string) *url.URL {
+ u, _ := url.Parse(s.cluster.Services.Workbench2.ExternalURL.String())
+ u.Path = path
+ return u
+}
+
+func (s *LogoutSuite) setupFederation(loginCluster string) {
+ if loginCluster == "" {
+ s.cluster.Login.Test.Enable = true
+ } else {
+ s.cluster.Login.LoginCluster = loginCluster
+ }
+ dbconn := ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}
+ s.fed = New(s.ctx, s.cluster, nil, dbconn.GetDB)
+}
+
+func (s *LogoutSuite) setupStub(c *check.C, id string, stubURL *url.URL, stubErr error) *LogoutStub {
+ loc, err := url.Parse(stubURL.String())
+ c.Check(err, check.IsNil)
+ stub := LogoutStub{redirectLocation: loc}
+ stub.Error = stubErr
+ if id == s.cluster.ClusterID {
+ s.fed.local = &stub
+ } else {
+ s.addDirectRemote(c, id, &stub)
+ }
+ return &stub
+}
+
+func (s *LogoutSuite) v2Token(clusterID string) string {
+ return fmt.Sprintf("v2/%s-gj3su-12345abcde67890/abcdefghijklmnopqrstuvwxy", clusterID)
+}
+
+func (s *LogoutSuite) TestLocalLogoutOK(c *check.C) {
+ s.setupFederation("")
+ resp, err := s.fed.Logout(s.ctx, arvados.LogoutOptions{})
+ c.Check(err, check.IsNil)
+ c.Check(resp.RedirectLocation, check.Equals, s.cluster.Services.Workbench2.ExternalURL.String())
+}
+
+func (s *LogoutSuite) TestLocalLogoutRedirect(c *check.C) {
+ s.setupFederation("")
+ expURL := s.cluster.Services.Workbench1.ExternalURL
+ opts := arvados.LogoutOptions{ReturnTo: expURL.String()}
+ resp, err := s.fed.Logout(s.ctx, opts)
+ c.Check(err, check.IsNil)
+ c.Check(resp.RedirectLocation, check.Equals, expURL.String())
+}
+
+func (s *LogoutSuite) TestLocalLogoutBadRequestError(c *check.C) {
+ s.setupFederation("")
+ returnTo := s.badReturnURL("TestLocalLogoutBadRequestError")
+ opts := arvados.LogoutOptions{ReturnTo: returnTo.String()}
+ _, err := s.fed.Logout(s.ctx, opts)
+ c.Check(err, check.NotNil)
+}
+
+func (s *LogoutSuite) TestRemoteLogoutRedirect(c *check.C) {
+ s.setupFederation("zhome")
+ redirect := url.URL{Scheme: "https", Host: "example.com"}
+ loginStub := s.setupStub(c, "zhome", &redirect, nil)
+ returnTo := s.goodReturnURL("TestRemoteLogoutRedirect")
+ resp, err := s.fed.Logout(s.ctx, arvados.LogoutOptions{ReturnTo: returnTo.String()})
+ c.Check(err, check.IsNil)
+ c.Check(resp.RedirectLocation, check.Equals, redirect.String())
+ loginStub.CheckCalls(c, returnTo)
+}
+
+func (s *LogoutSuite) TestRemoteLogoutError(c *check.C) {
+ s.setupFederation("zhome")
+ expErr := errors.New("TestRemoteLogoutError expErr")
+ loginStub := s.setupStub(c, "zhome", emptyURL, expErr)
+ returnTo := s.goodReturnURL("TestRemoteLogoutError")
+ _, err := s.fed.Logout(s.ctx, arvados.LogoutOptions{ReturnTo: returnTo.String()})
+ c.Check(err, check.Equals, expErr)
+ loginStub.CheckCalls(c, returnTo)
+}
+
+func (s *LogoutSuite) TestRemoteLogoutLocalRedirect(c *check.C) {
+ s.setupFederation("zhome")
+ loginStub := s.setupStub(c, "zhome", emptyURL, nil)
+ redirect := url.URL{Scheme: "https", Host: "example.com"}
+ localStub := s.setupStub(c, "aaaaa", &redirect, nil)
+ resp, err := s.fed.Logout(s.ctx, arvados.LogoutOptions{})
+ c.Check(err, check.IsNil)
+ c.Check(resp.RedirectLocation, check.Equals, redirect.String())
+ // emptyURL to match the empty LogoutOptions
+ loginStub.CheckCalls(c, emptyURL)
+ localStub.CheckCalls(c, emptyURL)
+}
+
+func (s *LogoutSuite) TestRemoteLogoutLocalError(c *check.C) {
+ s.setupFederation("zhome")
+ expErr := errors.New("TestRemoteLogoutLocalError expErr")
+ loginStub := s.setupStub(c, "zhome", emptyURL, nil)
+ localStub := s.setupStub(c, "aaaaa", emptyURL, expErr)
+ _, err := s.fed.Logout(s.ctx, arvados.LogoutOptions{})
+ c.Check(err, check.Equals, expErr)
+ loginStub.CheckCalls(c, emptyURL)
+ localStub.CheckCalls(c, emptyURL)
+}
+
+func (s *LogoutSuite) TestV2TokenRedirect(c *check.C) {
+ s.setupFederation("")
+ redirect := url.URL{Scheme: "https", Host: "example.com"}
+ returnTo := s.goodReturnURL("TestV2TokenRedirect")
+ localErr := errors.New("TestV2TokenRedirect error")
+ tokenStub := s.setupStub(c, "zzzzz", &redirect, nil)
+ s.setupStub(c, "aaaaa", emptyURL, localErr)
+ tokens := []string{s.v2Token("zzzzz")}
+ ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: tokens})
+ resp, err := s.fed.Logout(ctx, arvados.LogoutOptions{ReturnTo: returnTo.String()})
+ c.Check(err, check.IsNil)
+ c.Check(resp.RedirectLocation, check.Equals, redirect.String())
+ tokenStub.CheckCalls(c, returnTo)
+}
+
+func (s *LogoutSuite) TestV2TokenError(c *check.C) {
+ s.setupFederation("")
+ returnTo := s.goodReturnURL("TestV2TokenError")
+ tokenErr := errors.New("TestV2TokenError error")
+ tokenStub := s.setupStub(c, "zzzzz", emptyURL, tokenErr)
+ s.setupStub(c, "aaaaa", emptyURL, nil)
+ tokens := []string{s.v2Token("zzzzz")}
+ ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: tokens})
+ _, err := s.fed.Logout(ctx, arvados.LogoutOptions{ReturnTo: returnTo.String()})
+ c.Check(err, check.Equals, tokenErr)
+ tokenStub.CheckCalls(c, returnTo)
+}
+
+func (s *LogoutSuite) TestV2TokenLocalRedirect(c *check.C) {
+ s.setupFederation("")
+ redirect := url.URL{Scheme: "https", Host: "example.com"}
+ tokenStub := s.setupStub(c, "zzzzz", emptyURL, nil)
+ localStub := s.setupStub(c, "aaaaa", &redirect, nil)
+ tokens := []string{s.v2Token("zzzzz")}
+ ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: tokens})
+ resp, err := s.fed.Logout(ctx, arvados.LogoutOptions{})
+ c.Check(err, check.IsNil)
+ c.Check(resp.RedirectLocation, check.Equals, redirect.String())
+ tokenStub.CheckCalls(c, emptyURL)
+ localStub.CheckCalls(c, emptyURL)
+}
+
+func (s *LogoutSuite) TestV2TokenLocalError(c *check.C) {
+ s.setupFederation("")
+ tokenErr := errors.New("TestV2TokenLocalError error")
+ tokenStub := s.setupStub(c, "zzzzz", emptyURL, nil)
+ localStub := s.setupStub(c, "aaaaa", emptyURL, tokenErr)
+ tokens := []string{s.v2Token("zzzzz")}
+ ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: tokens})
+ _, err := s.fed.Logout(ctx, arvados.LogoutOptions{})
+ c.Check(err, check.Equals, tokenErr)
+ tokenStub.CheckCalls(c, emptyURL)
+ localStub.CheckCalls(c, emptyURL)
+}
+
+func (s *LogoutSuite) TestV2LocalTokenRedirect(c *check.C) {
+ s.setupFederation("")
+ redirect := url.URL{Scheme: "https", Host: "example.com"}
+ returnTo := s.goodReturnURL("TestV2LocalTokenRedirect")
+ localStub := s.setupStub(c, "aaaaa", &redirect, nil)
+ tokens := []string{s.v2Token("aaaaa")}
+ ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: tokens})
+ resp, err := s.fed.Logout(ctx, arvados.LogoutOptions{ReturnTo: returnTo.String()})
+ c.Check(err, check.IsNil)
+ c.Check(resp.RedirectLocation, check.Equals, redirect.String())
+ localStub.CheckCalls(c, returnTo)
+}
+
+func (s *LogoutSuite) TestV2LocalTokenError(c *check.C) {
+ s.setupFederation("")
+ returnTo := s.goodReturnURL("TestV2LocalTokenError")
+ tokenErr := errors.New("TestV2LocalTokenError error")
+ localStub := s.setupStub(c, "aaaaa", emptyURL, tokenErr)
+ tokens := []string{s.v2Token("aaaaa")}
+ ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: tokens})
+ _, err := s.fed.Logout(ctx, arvados.LogoutOptions{ReturnTo: returnTo.String()})
+ c.Check(err, check.Equals, tokenErr)
+ localStub.CheckCalls(c, returnTo)
+}
commit 623ee5523a2fc13fa2578fab510edb613e10f310
Author: Tom Clegg <tom at curii.com>
Date: Wed Nov 8 15:55:53 2023 -0500
Merge branch '21184-fix-build'
refs #21184
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 43a8d2f5d6..586b724b69 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -589,7 +589,7 @@ setup_virtualenv() {
elif [[ -n "$short" ]]; then
return
fi
- "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
+ "$venvdest/bin/pip3" install --no-cache-dir 'setuptools>=68' 'pip>=20'
}
initialize() {
diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go
index f1a873ae8d..3a91c78641 100644
--- a/lib/crunchrun/executor_test.go
+++ b/lib/crunchrun/executor_test.go
@@ -134,6 +134,10 @@ func (s *executorSuite) TestExecCleanEnv(c *C) {
// singularity also sets this by itself (v3.5.2, but not v3.7.4)
case "PROMPT_COMMAND", "PS1", "SINGULARITY_BIND", "SINGULARITY_COMMAND", "SINGULARITY_ENVIRONMENT":
// singularity also sets these by itself (v3.7.4)
+ case "SINGULARITY_NO_EVAL":
+ // our singularity driver sets this to control
+ // singularity behavior, and it gets passed
+ // through to the container
default:
got[kv[0]] = kv[1]
}
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index f52768d3d3..437aa39eb8 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -42,10 +42,7 @@ from cwltool.utils import (
CWLOutputType,
)
-if os.name == "posix" and sys.version_info[0] < 3:
- import subprocess32 as subprocess
-else:
- import subprocess
+import subprocess
from schema_salad.sourceline import SourceLine, cmap
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 9e20efd2a3..1da0d53ce8 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -58,7 +58,6 @@ setup(name='arvados-cwl-runner',
test_suite='tests',
tests_require=[
'mock>=1.0,<4',
- 'subprocess32>=3.5.1',
],
zip_safe=True,
)
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 4c5b13aa17..7b7367080f 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -19,10 +19,7 @@ import fcntl
from operator import itemgetter
from stat import *
-if os.name == "posix" and sys.version_info[0] < 3:
- import subprocess32 as subprocess
-else:
- import subprocess
+import subprocess
import arvados
import arvados.util
commit 9084d255611326869a1a603b3269d307329a4c59
Author: Tom Clegg <tom at curii.com>
Date: Fri Oct 27 11:44:38 2023 -0400
Merge branch '21124-max-rails-reqs'
fixes #21124
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index c10bf0e0f2..a633216be7 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -225,7 +225,17 @@ Clusters:
# Maximum number of concurrent requests to process concurrently
# in a single service process, or 0 for no limit.
- MaxConcurrentRequests: 8
+ #
+ # Note this applies to all Arvados services (controller, webdav,
+ # websockets, etc.). Concurrency in the controller service is
+ # also effectively limited by MaxConcurrentRailsRequests (see
+ # below) because most controller requests proxy through to the
+ # RailsAPI service.
+ MaxConcurrentRequests: 64
+
+ # Maximum number of concurrent requests to process concurrently
+ # in a single RailsAPI service process, or 0 for no limit.
+ MaxConcurrentRailsRequests: 8
# Maximum number of incoming requests to hold in a priority
# queue waiting for one of the MaxConcurrentRequests slots to be
diff --git a/lib/config/export.go b/lib/config/export.go
index 3b577b023f..cbd9ff6d7f 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -68,6 +68,7 @@ var whitelist = map[string]bool{
"API.KeepServiceRequestTimeout": false,
"API.LockBeforeUpdate": false,
"API.LogCreateRequestFraction": false,
+ "API.MaxConcurrentRailsRequests": false,
"API.MaxConcurrentRequests": false,
"API.MaxIndexDatabaseRead": false,
"API.MaxItemsPerResponse": true,
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 854b94861f..725f86f3bd 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -148,6 +148,19 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
return 1
}
+ maxReqs := cluster.API.MaxConcurrentRequests
+ if maxRails := cluster.API.MaxConcurrentRailsRequests; maxRails > 0 &&
+ (maxRails < maxReqs || maxReqs == 0) &&
+ strings.HasSuffix(prog, "controller") {
+ // Ideally, we would accept up to
+ // MaxConcurrentRequests, and apply the
+ // MaxConcurrentRailsRequests limit only for requests
+ // that require calling upstream to RailsAPI. But for
+ // now we make the simplifying assumption that every
+ // controller request causes an upstream RailsAPI
+ // request.
+ maxReqs = maxRails
+ }
instrumented := httpserver.Instrument(reg, log,
httpserver.HandlerWithDeadline(cluster.API.RequestTimeout.Duration(),
httpserver.AddRequestIDs(
@@ -156,7 +169,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
interceptHealthReqs(cluster.ManagementToken, handler.CheckHealth,
&httpserver.RequestLimiter{
Handler: handler,
- MaxConcurrent: cluster.API.MaxConcurrentRequests,
+ MaxConcurrent: maxReqs,
MaxQueue: cluster.API.MaxQueuedRequests,
MaxQueueTimeForMinPriority: cluster.API.MaxQueueTimeForLockRequests.Duration(),
Priority: c.requestPriority,
@@ -199,7 +212,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
<-handler.Done()
srv.Close()
}()
- go c.requestQueueDumpCheck(cluster, prog, reg, &srv.Server, logger)
+ go c.requestQueueDumpCheck(cluster, maxReqs, prog, reg, &srv.Server, logger)
err = srv.Wait()
if err != nil {
return 1
@@ -211,9 +224,9 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
// server's incoming HTTP request queue size. When it exceeds 90% of
// API.MaxConcurrentRequests, write the /_inspect/requests data to a
// JSON file in the specified directory.
-func (c *command) requestQueueDumpCheck(cluster *arvados.Cluster, prog string, reg *prometheus.Registry, srv *http.Server, logger logrus.FieldLogger) {
+func (c *command) requestQueueDumpCheck(cluster *arvados.Cluster, maxReqs int, prog string, reg *prometheus.Registry, srv *http.Server, logger logrus.FieldLogger) {
outdir := cluster.SystemLogs.RequestQueueDumpDirectory
- if outdir == "" || cluster.ManagementToken == "" || cluster.API.MaxConcurrentRequests < 1 {
+ if outdir == "" || cluster.ManagementToken == "" || maxReqs < 1 {
return
}
logger = logger.WithField("worker", "RequestQueueDump")
@@ -228,7 +241,7 @@ func (c *command) requestQueueDumpCheck(cluster *arvados.Cluster, prog string, r
for _, mf := range mfs {
if mf.Name != nil && *mf.Name == "arvados_concurrent_requests" && len(mf.Metric) == 1 {
n := int(mf.Metric[0].GetGauge().GetValue())
- if n > 0 && n >= cluster.API.MaxConcurrentRequests*9/10 {
+ if n > 0 && n >= maxReqs*9/10 {
dump = true
break
}
diff --git a/lib/service/cmd_test.go b/lib/service/cmd_test.go
index 97a6bd8a4c..08b3a239dc 100644
--- a/lib/service/cmd_test.go
+++ b/lib/service/cmd_test.go
@@ -39,15 +39,19 @@ const (
contextKey key = iota
)
-func (*Suite) TestGetListenAddress(c *check.C) {
+func unusedPort(c *check.C) string {
// Find an available port on the testing host, so the test
// cases don't get confused by "already in use" errors.
listener, err := net.Listen("tcp", ":")
c.Assert(err, check.IsNil)
- _, unusedPort, err := net.SplitHostPort(listener.Addr().String())
- c.Assert(err, check.IsNil)
listener.Close()
+ _, port, err := net.SplitHostPort(listener.Addr().String())
+ c.Assert(err, check.IsNil)
+ return port
+}
+func (*Suite) TestGetListenAddress(c *check.C) {
+ port := unusedPort(c)
defer os.Unsetenv("ARVADOS_SERVICE_INTERNAL_URL")
for idx, trial := range []struct {
// internalURL => listenURL, both with trailing "/"
@@ -60,17 +64,17 @@ func (*Suite) TestGetListenAddress(c *check.C) {
expectInternal string
}{
{
- internalURLs: map[string]string{"http://localhost:" + unusedPort + "/": ""},
- expectListen: "http://localhost:" + unusedPort + "/",
- expectInternal: "http://localhost:" + unusedPort + "/",
+ internalURLs: map[string]string{"http://localhost:" + port + "/": ""},
+ expectListen: "http://localhost:" + port + "/",
+ expectInternal: "http://localhost:" + port + "/",
},
{ // implicit port 80 in InternalURLs
internalURLs: map[string]string{"http://localhost/": ""},
expectErrorMatch: `.*:80: bind: permission denied`,
},
{ // implicit port 443 in InternalURLs
- internalURLs: map[string]string{"https://host.example/": "http://localhost:" + unusedPort + "/"},
- expectListen: "http://localhost:" + unusedPort + "/",
+ internalURLs: map[string]string{"https://host.example/": "http://localhost:" + port + "/"},
+ expectListen: "http://localhost:" + port + "/",
expectInternal: "https://host.example/",
},
{ // implicit port 443 in ListenURL
@@ -85,16 +89,16 @@ func (*Suite) TestGetListenAddress(c *check.C) {
{
internalURLs: map[string]string{
"https://hostname1.example/": "http://localhost:12435/",
- "https://hostname2.example/": "http://localhost:" + unusedPort + "/",
+ "https://hostname2.example/": "http://localhost:" + port + "/",
},
envVar: "https://hostname2.example", // note this works despite missing trailing "/"
- expectListen: "http://localhost:" + unusedPort + "/",
+ expectListen: "http://localhost:" + port + "/",
expectInternal: "https://hostname2.example/",
},
{ // cannot listen on any of the ListenURLs
internalURLs: map[string]string{
- "https://hostname1.example/": "http://1.2.3.4:" + unusedPort + "/",
- "https://hostname2.example/": "http://1.2.3.4:" + unusedPort + "/",
+ "https://hostname1.example/": "http://1.2.3.4:" + port + "/",
+ "https://hostname2.example/": "http://1.2.3.4:" + port + "/",
},
expectErrorMatch: "configuration does not enable the \"arvados-controller\" service on this host",
},
@@ -194,10 +198,19 @@ func (*Suite) TestCommand(c *check.C) {
c.Check(stderr.String(), check.Matches, `(?ms).*"msg":"CheckHealth called".*`)
}
-func (*Suite) TestDumpRequests(c *check.C) {
+func (s *Suite) TestDumpRequestsKeepweb(c *check.C) {
+ s.testDumpRequests(c, arvados.ServiceNameKeepweb, "MaxConcurrentRequests")
+}
+
+func (s *Suite) TestDumpRequestsController(c *check.C) {
+ s.testDumpRequests(c, arvados.ServiceNameController, "MaxConcurrentRailsRequests")
+}
+
+func (*Suite) testDumpRequests(c *check.C, serviceName arvados.ServiceName, maxReqsConfigKey string) {
defer func(orig time.Duration) { requestQueueDumpCheckInterval = orig }(requestQueueDumpCheckInterval)
requestQueueDumpCheckInterval = time.Second / 10
+ port := unusedPort(c)
tmpdir := c.MkDir()
cf, err := ioutil.TempFile(tmpdir, "cmd_test.")
c.Assert(err, check.IsNil)
@@ -211,14 +224,18 @@ Clusters:
SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
ManagementToken: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
API:
- MaxConcurrentRequests: %d
+ `+maxReqsConfigKey+`: %d
MaxQueuedRequests: 0
SystemLogs: {RequestQueueDumpDirectory: %q}
Services:
Controller:
- ExternalURL: "http://localhost:12345"
- InternalURLs: {"http://localhost:12345": {}}
+ ExternalURL: "http://localhost:`+port+`"
+ InternalURLs: {"http://localhost:`+port+`": {}}
+ WebDAV:
+ ExternalURL: "http://localhost:`+port+`"
+ InternalURLs: {"http://localhost:`+port+`": {}}
`, max, tmpdir)
+ cf.Close()
started := make(chan bool, max+1)
hold := make(chan bool)
@@ -230,7 +247,7 @@ Clusters:
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
- cmd := Command(arvados.ServiceNameController, func(ctx context.Context, _ *arvados.Cluster, token string, reg *prometheus.Registry) Handler {
+ cmd := Command(serviceName, func(ctx context.Context, _ *arvados.Cluster, token string, reg *prometheus.Registry) Handler {
return &testHandler{ctx: ctx, handler: handler, healthCheck: healthCheck}
})
cmd.(*command).ctx = context.WithValue(ctx, contextKey, "bar")
@@ -239,7 +256,7 @@ Clusters:
var stdin, stdout, stderr bytes.Buffer
go func() {
- cmd.RunCommand("arvados-controller", []string{"-config", cf.Name()}, &stdin, &stdout, &stderr)
+ cmd.RunCommand(string(serviceName), []string{"-config", cf.Name()}, &stdin, &stdout, &stderr)
close(exited)
}()
select {
@@ -252,10 +269,10 @@ Clusters:
deadline := time.Now().Add(time.Second * 2)
for i := 0; i < max+1; i++ {
go func() {
- resp, err := client.Get("http://localhost:12345/testpath")
+ resp, err := client.Get("http://localhost:" + port + "/testpath")
for err != nil && strings.Contains(err.Error(), "dial tcp") && deadline.After(time.Now()) {
time.Sleep(time.Second / 100)
- resp, err = client.Get("http://localhost:12345/testpath")
+ resp, err = client.Get("http://localhost:" + port + "/testpath")
}
if c.Check(err, check.IsNil) {
c.Logf("resp StatusCode %d", resp.StatusCode)
@@ -267,12 +284,12 @@ Clusters:
case <-started:
case <-time.After(time.Second):
c.Logf("%s", stderr.String())
- panic("timed out")
+ c.Fatal("timed out")
}
}
for delay := time.Second / 100; ; delay = delay * 2 {
time.Sleep(delay)
- j, err := os.ReadFile(tmpdir + "/arvados-controller-requests.json")
+ j, err := os.ReadFile(tmpdir + "/" + string(serviceName) + "-requests.json")
if os.IsNotExist(err) && deadline.After(time.Now()) {
continue
}
@@ -298,7 +315,7 @@ Clusters:
}
for _, path := range []string{"/_inspect/requests", "/metrics"} {
- req, err := http.NewRequest("GET", "http://localhost:12345"+path, nil)
+ req, err := http.NewRequest("GET", "http://localhost:"+port+""+path, nil)
c.Assert(err, check.IsNil)
req.Header.Set("Authorization", "Bearer bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
resp, err := client.Do(req)
@@ -320,10 +337,10 @@ Clusters:
}
close(hold)
cancel()
-
}
func (*Suite) TestTLS(c *check.C) {
+ port := unusedPort(c)
cwd, err := os.Getwd()
c.Assert(err, check.IsNil)
@@ -333,8 +350,8 @@ Clusters:
SystemRootToken: abcde
Services:
Controller:
- ExternalURL: "https://localhost:12345"
- InternalURLs: {"https://localhost:12345": {}}
+ ExternalURL: "https://localhost:` + port + `"
+ InternalURLs: {"https://localhost:` + port + `": {}}
TLS:
Key: file://` + cwd + `/../../services/api/tmp/self-signed.key
Certificate: file://` + cwd + `/../../services/api/tmp/self-signed.pem
@@ -359,7 +376,7 @@ Clusters:
defer close(got)
client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
for range time.NewTicker(time.Millisecond).C {
- resp, err := client.Get("https://localhost:12345")
+ resp, err := client.Get("https://localhost:" + port)
if err != nil {
c.Log(err)
continue
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 0afcf2be3c..64424fc3e9 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -99,6 +99,7 @@ type Cluster struct {
DisabledAPIs StringSet
MaxIndexDatabaseRead int
MaxItemsPerResponse int
+ MaxConcurrentRailsRequests int
MaxConcurrentRequests int
MaxQueuedRequests int
MaxQueueTimeForLockRequests Duration
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
index 2944b4073c..dc98c43ace 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
@@ -117,7 +117,8 @@ arvados:
### API
API:
- MaxConcurrentRequests: {{ max_workers * 2 }}
+ MaxConcurrentRailsRequests: {{ max_workers * 2 }}
+ MaxConcurrentRequests: {{ max_reqs }}
MaxQueuedRequests: {{ max_reqs }}
### CONTAINERS
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
index 4c0aea25fe..de4c830906 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
@@ -29,7 +29,7 @@ nginx:
# Make the passenger queue small (twice the concurrency, so
# there's at most one pending request for each busy worker)
# because controller reorders requests based on priority, and
- # won't send more than API.MaxConcurrentRequests to passenger
+ # won't send more than API.MaxConcurrentRailsRequests to passenger
# (which is max_workers * 2), so things that are moved to the head
# of the line get processed quickly.
passenger_max_request_queue_size: {{ max_workers * 2 + 1 }}
commit 06150c2955d59c22857ebd5f07e3f5df758a0bab
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Oct 27 14:54:19 2023 -0400
Merge branch '20284-username-conflict-fix' refs #20284
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index ded86aa66d..88377dbaf6 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -35,10 +35,15 @@ class Arvados::V1::UsersController < ApplicationController
loginCluster = Rails.configuration.Login.LoginCluster
if u.uuid[0..4] == loginCluster && !needupdate[:username].nil?
local_user = User.find_by_username(needupdate[:username])
- # A cached user record from the LoginCluster is stale, reset its username
- # and retry the update operation.
- if local_user.andand.uuid[0..4] == loginCluster && local_user.uuid != u.uuid
- new_username = "#{needupdate[:username]}conflict#{rand(99999999)}"
+ # The username of this record conflicts with an existing,
+ # different user record. This can happen because the
+ # username changed upstream on the login cluster, or
+ # because we're federated with another cluster with a user
+ # by the same username. The login cluster is the source
+ # of truth, so change the username on the conflicting
+ # record and retry the update operation.
+ if local_user.uuid != u.uuid
+ new_username = "#{needupdate[:username]}#{rand(99999999)}"
Rails.logger.warn("cached username '#{needupdate[:username]}' collision with user '#{local_user.uuid}' - renaming to '#{new_username}' before retrying")
local_user.update_attributes!({username: new_username})
retry
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index b7d683df29..50d1606a08 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -1043,12 +1043,16 @@ The Arvados team.
existinguuid = 'remot-tpzed-foobarbazwazqux'
newuuid = 'remot-tpzed-newnarnazwazqux'
unchanginguuid = 'remot-tpzed-nochangingattrs'
+ conflictinguuid1 = 'remot-tpzed-conflictingnam1'
+ conflictinguuid2 = 'remot-tpzed-conflictingnam2'
act_as_system_user do
User.create!(uuid: existinguuid, email: 'root at existing.example.com')
User.create!(uuid: unchanginguuid, email: 'root at unchanging.example.com', prefs: {'foo' => {'bar' => 'baz'}})
end
assert_equal(1, Log.where(object_uuid: unchanginguuid).count)
+ Rails.configuration.Login.LoginCluster = 'remot'
+
authorize_with(:admin)
patch(:batch_update,
params: {
@@ -1069,6 +1073,14 @@ The Arvados team.
'email' => 'root at unchanging.example.com',
'prefs' => {'foo' => {'bar' => 'baz'}},
},
+ conflictinguuid1 => {
+ 'email' => 'root at conflictingname1.example.com',
+ 'username' => 'active'
+ },
+ conflictinguuid2 => {
+ 'email' => 'root at conflictingname2.example.com',
+ 'username' => 'federatedactive'
+ },
}})
assert_response(:success)
commit aaa257baff8c39b2349ca146f8b5601cccba3e49
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Oct 30 09:48:40 2023 -0400
Merge branch '21025-keep-web-redirect-bypass'
Closes #21025.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 3af326a1ad..123c4fe34d 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -293,12 +293,18 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
reqTokens = auth.CredentialsFromRequest(r).Tokens
}
- formToken := r.FormValue("api_token")
+ r.ParseForm()
origin := r.Header.Get("Origin")
cors := origin != "" && !strings.HasSuffix(origin, "://"+r.Host)
safeAjax := cors && (r.Method == http.MethodGet || r.Method == http.MethodHead)
- safeAttachment := attachment && r.URL.Query().Get("api_token") == ""
- if formToken == "" {
+ // Important distinction: safeAttachment checks whether api_token exists
+ // as a query parameter. haveFormTokens checks whether api_token exists
+ // as request form data *or* a query parameter. Different checks are
+ // necessary because both the request disposition and the location of
+ // the API token affect whether or not the request needs to be
+ // redirected. The different branch comments below explain further.
+ safeAttachment := attachment && !r.URL.Query().Has("api_token")
+ if formTokens, haveFormTokens := r.Form["api_token"]; !haveFormTokens {
// No token to use or redact.
} else if safeAjax || safeAttachment {
// If this is a cross-origin request, the URL won't
@@ -313,7 +319,9 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
// form?" problem, so provided the token isn't
// embedded in the URL, there's no reason to do
// redirect-with-cookie in this case either.
- reqTokens = append(reqTokens, formToken)
+ for _, tok := range formTokens {
+ reqTokens = append(reqTokens, tok)
+ }
} else if browserMethod[r.Method] {
// If this is a page view, and the client provided a
// token via query string or POST body, we must put
@@ -776,7 +784,7 @@ func applyContentDispositionHdr(w http.ResponseWriter, r *http.Request, filename
}
func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, location string, credentialsOK bool) {
- if formToken := r.FormValue("api_token"); formToken != "" {
+ if formTokens, haveFormTokens := r.Form["api_token"]; haveFormTokens {
if !credentialsOK {
// It is not safe to copy the provided token
// into a cookie unless the current vhost
@@ -797,13 +805,19 @@ func (h *handler) seeOtherWithCookie(w http.ResponseWriter, r *http.Request, loc
// bar, and in the case of a POST request to avoid
// raising warnings when the user refreshes the
// resulting page.
- http.SetCookie(w, &http.Cookie{
- Name: "arvados_api_token",
- Value: auth.EncodeTokenCookie([]byte(formToken)),
- Path: "/",
- HttpOnly: true,
- SameSite: http.SameSiteLaxMode,
- })
+ for _, tok := range formTokens {
+ if tok == "" {
+ continue
+ }
+ http.SetCookie(w, &http.Cookie{
+ Name: "arvados_api_token",
+ Value: auth.EncodeTokenCookie([]byte(tok)),
+ Path: "/",
+ HttpOnly: true,
+ SameSite: http.SameSiteLaxMode,
+ })
+ break
+ }
}
// Propagate query parameters (except api_token) from
diff --git a/services/keep-web/handler_test.go b/services/keep-web/handler_test.go
index 4a76276392..5e81f3a01e 100644
--- a/services/keep-web/handler_test.go
+++ b/services/keep-web/handler_test.go
@@ -797,6 +797,34 @@ func (s *IntegrationSuite) TestVhostRedirectQueryTokenAttachmentOnlyHost(c *chec
c.Check(resp.Header().Get("Content-Disposition"), check.Equals, "attachment")
}
+func (s *IntegrationSuite) TestVhostRedirectMultipleTokens(c *check.C) {
+ baseUrl := arvadostest.FooCollection + ".example.com/foo"
+ query := url.Values{}
+
+ // The intent of these tests is to check that requests are redirected
+ // correctly in the presence of multiple API tokens. The exact response
+ // codes and content are not closely considered: they're just how
+ // keep-web responded when we made the smallest possible fix. Changing
+ // those responses may be okay, but you should still test all these
+ // different cases and the associated redirect logic.
+ query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
+ query["api_token"] = []string{arvadostest.ActiveToken, arvadostest.AnonymousToken, ""}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
+ query["api_token"] = []string{arvadostest.ActiveToken, "", arvadostest.AnonymousToken}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
+ query["api_token"] = []string{"", arvadostest.ActiveToken}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusOK, "foo")
+
+ expectContent := regexp.QuoteMeta(unauthorizedMessage + "\n")
+ query["api_token"] = []string{arvadostest.AnonymousToken, "invalidtoo"}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
+ query["api_token"] = []string{arvadostest.AnonymousToken, ""}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
+ query["api_token"] = []string{"", arvadostest.AnonymousToken}
+ s.testVhostRedirectTokenToCookie(c, "GET", baseUrl, "?"+query.Encode(), nil, "", http.StatusUnauthorized, expectContent)
+}
+
func (s *IntegrationSuite) TestVhostRedirectPOSTFormTokenToCookie(c *check.C) {
s.testVhostRedirectTokenToCookie(c, "POST",
arvadostest.FooCollection+".example.com/foo",
@@ -999,20 +1027,36 @@ func (s *IntegrationSuite) testVhostRedirectTokenToCookie(c *check.C, method, ho
s.handler.ServeHTTP(resp, req)
if resp.Code != http.StatusSeeOther {
+ attachment, _ := regexp.MatchString(`^attachment(;|$)`, resp.Header().Get("Content-Disposition"))
+ // Since we're not redirecting, check that any api_token in the URL is
+ // handled safely.
+ // If there is no token in the URL, then we're good.
+ // Otherwise, if the response code is an error, the body is expected to
+ // be static content, and nothing that might maliciously introspect the
+ // URL. It's considered safe and allowed.
+ // Otherwise, if the response content has attachment disposition,
+ // that's considered safe for all the reasons explained in the
+ // safeAttachment comment in handler.go.
+ c.Check(!u.Query().Has("api_token") || resp.Code >= 400 || attachment, check.Equals, true)
return resp
}
+
+ loc, err := url.Parse(resp.Header().Get("Location"))
+ c.Assert(err, check.IsNil)
+ c.Check(loc.Scheme, check.Equals, u.Scheme)
+ c.Check(loc.Host, check.Equals, u.Host)
+ c.Check(loc.RawPath, check.Equals, u.RawPath)
+ // If the response was a redirect, it should never include an API token.
+ c.Check(loc.Query().Has("api_token"), check.Equals, false)
c.Check(resp.Body.String(), check.Matches, `.*href="http://`+regexp.QuoteMeta(html.EscapeString(hostPath))+`(\?[^"]*)?".*`)
- c.Check(strings.Split(resp.Header().Get("Location"), "?")[0], check.Equals, "http://"+hostPath)
cookies := (&http.Response{Header: resp.Header()}).Cookies()
- u, err := u.Parse(resp.Header().Get("Location"))
- c.Assert(err, check.IsNil)
c.Logf("following redirect to %s", u)
req = &http.Request{
Method: "GET",
- Host: u.Host,
- URL: u,
- RequestURI: u.RequestURI(),
+ Host: loc.Host,
+ URL: loc,
+ RequestURI: loc.RequestURI(),
Header: reqHeader,
}
for _, c := range cookies {
commit 8214f0e9139dc9ce2b1b3f1f050e3fb2c0991141
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Wed Nov 1 10:41:13 2023 -0300
Merge branch '21125-installer-s3-prefix-length'. Closes #21125
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
index 84df363c2e..2944b4073c 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
@@ -152,6 +152,11 @@ arvados:
Bucket: __KEEP_AWS_S3_BUCKET__
IAMRole: __KEEP_AWS_IAM_ROLE__
Region: __KEEP_AWS_REGION__
+ # IMPORTANT: The default value for PrefixLength is 0, and should not
+ # be changed once the volume is in use. For new installations it's
+ # recommended to set it to 3 for better performance.
+ # See: https://doc.arvados.org/install/configure-s3-object-storage.html
+ PrefixLength: 3
Users:
NewUsersAreActive: true
commit edbf66a8ebc84eb985d350e1ef1c29a1486d44da
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Nov 9 13:03:03 2023 -0500
Handle non-nil, empty Instances return refs #20978
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 0d181be0e9..07a146d99f 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -327,7 +327,7 @@ func (instanceSet *ec2InstanceSet) Create(
atomic.StoreInt32(&instanceSet.currentSubnetIDIndex, int32(tryIndex))
break
}
- if rsv == nil {
+ if rsv == nil || len(rsv.Instances) == 0 {
return nil, wrapError(errToReturn, &instanceSet.throttleDelayCreate)
}
return &ec2Instance{
commit ac39afed6cd3de1704d75aecdbc46544b02f02b2
Author: Tom Clegg <tom at curii.com>
Date: Tue Nov 7 11:25:10 2023 -0500
Merge branch '20978-instance-types'
closes #20978
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 46b008ca70..3e7428eb17 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -28,6 +28,26 @@ TODO: extract this information based on git commit messages and generate changel
<div class="releasenotes">
</notextile>
+h2(#2_7_1). v2.7.1 (2023-11-29)
+
+"previous: Upgrading to 2.7.0":#v2_7_0
+
+h3. Check implications of Containers.MaximumPriceFactor 1.5
+
+When scheduling a container, Arvados now considers using instance types other than the lowest-cost type consistent with the container's resource constraints. If a larger instance is already running and idle, or the cloud provider reports that the optimal instance type is not currently available, Arvados will select a larger instance type, provided the cost does not exceed 1.5x the optimal instance type cost.
+
+This will typically reduce overall latency for containers and reduce instance booting/shutdown overhead, but may increase costs depending on workload and instance availability. To avoid this behavior, configure @Containers.MaximumPriceFactor: 1.0 at .
+
+h3. Synchronize keepstore and keep-balance upgrades
+
+The internal communication between keepstore and keep-balance about read-only volumes has changed. After keep-balance is upgraded, old versions of keepstore will be treated as read-only. We recommend upgrading and restarting all keepstore services first, then upgrading and restarting keep-balance.
+
+h3. Separate configs for MaxConcurrentRequests and MaxConcurrentRailsRequests
+
+The default configuration value @API.MaxConcurrentRequests@ (the number of concurrent requests that will be processed by a single instance of an arvados service process) is raised from 8 to 64.
+
+A new configuration key @API.MaxConcurrentRailsRequests@ (default 8) limits the number of concurrent requests processed by a RailsAPI service process.
+
h2(#v2_7_0). v2.7.0 (2023-09-21)
"previous: Upgrading to 2.6.3":#v2_6_3
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 816df48d90..0d181be0e9 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -288,7 +288,7 @@ func (instanceSet *ec2InstanceSet) Create(
}
var rsv *ec2.Reservation
- var err error
+ var errToReturn error
subnets := instanceSet.ec2config.SubnetID
currentSubnetIDIndex := int(atomic.LoadInt32(&instanceSet.currentSubnetIDIndex))
for tryOffset := 0; ; tryOffset++ {
@@ -299,8 +299,15 @@ func (instanceSet *ec2InstanceSet) Create(
trySubnet = subnets[tryIndex]
rii.NetworkInterfaces[0].SubnetId = aws.String(trySubnet)
}
+ var err error
rsv, err = instanceSet.client.RunInstances(&rii)
instanceSet.mInstanceStarts.WithLabelValues(trySubnet, boolLabelValue[err == nil]).Add(1)
+ if !isErrorCapacity(errToReturn) || isErrorCapacity(err) {
+ // We want to return the last capacity error,
+ // if any; otherwise the last non-capacity
+ // error.
+ errToReturn = err
+ }
if isErrorSubnetSpecific(err) &&
tryOffset < len(subnets)-1 {
instanceSet.logger.WithError(err).WithField("SubnetID", subnets[tryIndex]).
@@ -320,9 +327,8 @@ func (instanceSet *ec2InstanceSet) Create(
atomic.StoreInt32(&instanceSet.currentSubnetIDIndex, int32(tryIndex))
break
}
- err = wrapError(err, &instanceSet.throttleDelayCreate)
- if err != nil {
- return nil, err
+ if rsv == nil {
+ return nil, wrapError(errToReturn, &instanceSet.throttleDelayCreate)
}
return &ec2Instance{
provider: instanceSet,
@@ -711,7 +717,22 @@ func isErrorSubnetSpecific(err error) bool {
code := aerr.Code()
return strings.Contains(code, "Subnet") ||
code == "InsufficientInstanceCapacity" ||
- code == "InsufficientVolumeCapacity"
+ code == "InsufficientVolumeCapacity" ||
+ code == "Unsupported"
+}
+
+// isErrorCapacity returns true if the error indicates lack of
+// capacity (either temporary or permanent) to run a specific instance
+// type -- i.e., retrying with a different instance type might
+// succeed.
+func isErrorCapacity(err error) bool {
+ aerr, ok := err.(awserr.Error)
+ if !ok {
+ return false
+ }
+ code := aerr.Code()
+ return code == "InsufficientInstanceCapacity" ||
+ (code == "Unsupported" && strings.Contains(aerr.Message(), "requested instance type"))
}
type ec2QuotaError struct {
@@ -737,7 +758,7 @@ func wrapError(err error, throttleValue *atomic.Value) error {
return rateLimitError{error: err, earliestRetry: time.Now().Add(d)}
} else if isErrorQuota(err) {
return &ec2QuotaError{err}
- } else if aerr, ok := err.(awserr.Error); ok && aerr != nil && aerr.Code() == "InsufficientInstanceCapacity" {
+ } else if isErrorCapacity(err) {
return &capacityError{err, true}
} else if err != nil {
throttleValue.Store(time.Duration(0))
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index a57fcebf76..4b83005896 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -399,6 +399,37 @@ func (*EC2InstanceSetSuite) TestCreateAllSubnetsFailing(c *check.C) {
`.*`)
}
+func (*EC2InstanceSetSuite) TestCreateOneSubnetFailingCapacity(c *check.C) {
+ if *live != "" {
+ c.Skip("not applicable in live mode")
+ return
+ }
+ ap, img, cluster, reg := GetInstanceSet(c, `{"SubnetID":["subnet-full","subnet-broken"]}`)
+ ap.client.(*ec2stub).subnetErrorOnRunInstances = map[string]error{
+ "subnet-full": &ec2stubError{
+ code: "InsufficientFreeAddressesInSubnet",
+ message: "subnet is full",
+ },
+ "subnet-broken": &ec2stubError{
+ code: "InsufficientInstanceCapacity",
+ message: "insufficient capacity",
+ },
+ }
+ for i := 0; i < 3; i++ {
+ _, err := ap.Create(cluster.InstanceTypes["tiny"], img, nil, "", nil)
+ c.Check(err, check.NotNil)
+ c.Check(err, check.ErrorMatches, `.*InsufficientInstanceCapacity.*`)
+ }
+ c.Check(ap.client.(*ec2stub).runInstancesCalls, check.HasLen, 6)
+ metrics := arvadostest.GatherMetricsAsString(reg)
+ c.Check(metrics, check.Matches, `(?ms).*`+
+ `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="0"} 3\n`+
+ `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-broken",success="1"} 0\n`+
+ `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="0"} 3\n`+
+ `arvados_dispatchcloud_ec2_instance_starts_total{subnet_id="subnet-full",success="1"} 0\n`+
+ `.*`)
+}
+
func (*EC2InstanceSetSuite) TestTagInstances(c *check.C) {
ap, _, _, _ := GetInstanceSet(c, "{}")
l, err := ap.Instances(nil)
@@ -513,10 +544,18 @@ func (*EC2InstanceSetSuite) TestWrapError(c *check.C) {
_, ok = wrapped.(cloud.QuotaError)
c.Check(ok, check.Equals, true)
- capacityError := awserr.New("InsufficientInstanceCapacity", "", nil)
- wrapped = wrapError(capacityError, nil)
- caperr, ok := wrapped.(cloud.CapacityError)
- c.Check(ok, check.Equals, true)
- c.Check(caperr.IsCapacityError(), check.Equals, true)
- c.Check(caperr.IsInstanceTypeSpecific(), check.Equals, true)
+ for _, trial := range []struct {
+ code string
+ msg string
+ }{
+ {"InsufficientInstanceCapacity", ""},
+ {"Unsupported", "Your requested instance type (t3.micro) is not supported in your requested Availability Zone (us-east-1e). Please retry your request by not specifying an Availability Zone or choosing us-east-1a, us-east-1b, us-east-1c, us-east-1d, us-east-1f."},
+ } {
+ capacityError := awserr.New(trial.code, trial.msg, nil)
+ wrapped = wrapError(capacityError, nil)
+ caperr, ok := wrapped.(cloud.CapacityError)
+ c.Check(ok, check.Equals, true)
+ c.Check(caperr.IsCapacityError(), check.Equals, true)
+ c.Check(caperr.IsInstanceTypeSpecific(), check.Equals, true)
+ }
}
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 32727b1bce..c10bf0e0f2 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -1102,6 +1102,17 @@ Clusters:
# A price factor of 1.0 is a reasonable starting point.
PreemptiblePriceFactor: 0
+ # When the lowest-priced instance type for a given container is
+ # not available, try other instance types, up to the indicated
+ # maximum price factor.
+ #
+ # For example, with AvailabilityPriceFactor 1.5, if the
+ # lowest-cost instance type A suitable for a given container
+ # costs $2/h, Arvados may run the container on any instance type
+ # B costing $3/h or less when instance type A is not available
+ # or an idle instance of type B is already running.
+ MaximumPriceFactor: 1.5
+
# PEM encoded SSH key (RSA, DSA, or ECDSA) used by the
# cloud dispatcher for executing containers on worker VMs.
# Begins with "-----BEGIN RSA PRIVATE KEY-----\n"
diff --git a/lib/config/export.go b/lib/config/export.go
index 88c64f69a1..3b577b023f 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -135,6 +135,7 @@ var whitelist = map[string]bool{
"Containers.LogReuseDecisions": false,
"Containers.LSF": false,
"Containers.MaxDispatchAttempts": false,
+ "Containers.MaximumPriceFactor": true,
"Containers.MaxRetryAttempts": true,
"Containers.MinRetryPeriod": true,
"Containers.PreemptiblePriceFactor": false,
diff --git a/lib/dispatchcloud/container/queue.go b/lib/dispatchcloud/container/queue.go
index 77752013e8..8d8b7ff9af 100644
--- a/lib/dispatchcloud/container/queue.go
+++ b/lib/dispatchcloud/container/queue.go
@@ -22,7 +22,7 @@ import (
// load at the cost of increased under light load.
const queuedContainersTarget = 100
-type typeChooser func(*arvados.Container) (arvados.InstanceType, error)
+type typeChooser func(*arvados.Container) ([]arvados.InstanceType, error)
// An APIClient performs Arvados API requests. It is typically an
// *arvados.Client.
@@ -36,9 +36,9 @@ type QueueEnt struct {
// The container to run. Only the UUID, State, Priority,
// RuntimeConstraints, ContainerImage, SchedulingParameters,
// and CreatedAt fields are populated.
- Container arvados.Container `json:"container"`
- InstanceType arvados.InstanceType `json:"instance_type"`
- FirstSeenAt time.Time `json:"first_seen_at"`
+ Container arvados.Container `json:"container"`
+ InstanceTypes []arvados.InstanceType `json:"instance_types"`
+ FirstSeenAt time.Time `json:"first_seen_at"`
}
// String implements fmt.Stringer by returning the queued container's
@@ -252,7 +252,7 @@ func (cq *Queue) addEnt(uuid string, ctr arvados.Container) {
logger.WithError(err).Warn("error getting mounts")
return
}
- it, err := cq.chooseType(&ctr)
+ types, err := cq.chooseType(&ctr)
// Avoid wasting memory on a large Mounts attr (we don't need
// it after choosing type).
@@ -304,13 +304,20 @@ func (cq *Queue) addEnt(uuid string, ctr arvados.Container) {
}()
return
}
+ typeNames := ""
+ for _, it := range types {
+ if typeNames != "" {
+ typeNames += ", "
+ }
+ typeNames += it.Name
+ }
cq.logger.WithFields(logrus.Fields{
"ContainerUUID": ctr.UUID,
"State": ctr.State,
"Priority": ctr.Priority,
- "InstanceType": it.Name,
+ "InstanceTypes": typeNames,
}).Info("adding container to queue")
- cq.current[uuid] = QueueEnt{Container: ctr, InstanceType: it, FirstSeenAt: time.Now()}
+ cq.current[uuid] = QueueEnt{Container: ctr, InstanceTypes: types, FirstSeenAt: time.Now()}
}
// Lock acquires the dispatch lock for the given container.
@@ -577,7 +584,7 @@ func (cq *Queue) runMetrics(reg *prometheus.Registry) {
}
ents, _ := cq.Entries()
for _, ent := range ents {
- count[entKey{ent.Container.State, ent.InstanceType.Name}]++
+ count[entKey{ent.Container.State, ent.InstanceTypes[0].Name}]++
}
for k, v := range count {
mEntries.WithLabelValues(string(k.state), k.inst).Set(float64(v))
diff --git a/lib/dispatchcloud/container/queue_test.go b/lib/dispatchcloud/container/queue_test.go
index ca10983534..928c6dd8c8 100644
--- a/lib/dispatchcloud/container/queue_test.go
+++ b/lib/dispatchcloud/container/queue_test.go
@@ -40,9 +40,9 @@ func (suite *IntegrationSuite) TearDownTest(c *check.C) {
}
func (suite *IntegrationSuite) TestGetLockUnlockCancel(c *check.C) {
- typeChooser := func(ctr *arvados.Container) (arvados.InstanceType, error) {
+ typeChooser := func(ctr *arvados.Container) ([]arvados.InstanceType, error) {
c.Check(ctr.Mounts["/tmp"].Capacity, check.Equals, int64(24000000000))
- return arvados.InstanceType{Name: "testType"}, nil
+ return []arvados.InstanceType{{Name: "testType"}}, nil
}
client := arvados.NewClientFromEnv()
@@ -62,7 +62,8 @@ func (suite *IntegrationSuite) TestGetLockUnlockCancel(c *check.C) {
var wg sync.WaitGroup
for uuid, ent := range ents {
c.Check(ent.Container.UUID, check.Equals, uuid)
- c.Check(ent.InstanceType.Name, check.Equals, "testType")
+ c.Check(ent.InstanceTypes, check.HasLen, 1)
+ c.Check(ent.InstanceTypes[0].Name, check.Equals, "testType")
c.Check(ent.Container.State, check.Equals, arvados.ContainerStateQueued)
c.Check(ent.Container.Priority > 0, check.Equals, true)
// Mounts should be deleted to avoid wasting memory
@@ -108,7 +109,7 @@ func (suite *IntegrationSuite) TestGetLockUnlockCancel(c *check.C) {
}
func (suite *IntegrationSuite) TestCancelIfNoInstanceType(c *check.C) {
- errorTypeChooser := func(ctr *arvados.Container) (arvados.InstanceType, error) {
+ errorTypeChooser := func(ctr *arvados.Container) ([]arvados.InstanceType, error) {
// Make sure the relevant container fields are
// actually populated.
c.Check(ctr.ContainerImage, check.Equals, "test")
@@ -116,7 +117,7 @@ func (suite *IntegrationSuite) TestCancelIfNoInstanceType(c *check.C) {
c.Check(ctr.RuntimeConstraints.RAM, check.Equals, int64(12000000000))
c.Check(ctr.Mounts["/tmp"].Capacity, check.Equals, int64(24000000000))
c.Check(ctr.Mounts["/var/spool/cwl"].Capacity, check.Equals, int64(24000000000))
- return arvados.InstanceType{}, errors.New("no suitable instance type")
+ return nil, errors.New("no suitable instance type")
}
client := arvados.NewClientFromEnv()
diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 49be9e68a2..47e60abdee 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -111,7 +111,7 @@ func (disp *dispatcher) newExecutor(inst cloud.Instance) worker.Executor {
return exr
}
-func (disp *dispatcher) typeChooser(ctr *arvados.Container) (arvados.InstanceType, error) {
+func (disp *dispatcher) typeChooser(ctr *arvados.Container) ([]arvados.InstanceType, error) {
return ChooseInstanceType(disp.Cluster, ctr)
}
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index 6c057edc70..33d7f4e9ac 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -15,6 +15,7 @@ import (
"net/url"
"os"
"sync"
+ "sync/atomic"
"time"
"git.arvados.org/arvados.git/lib/config"
@@ -72,6 +73,7 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
RuntimeEngine: "stub",
MaxDispatchAttempts: 10,
+ MaximumPriceFactor: 1.5,
CloudVMs: arvados.CloudVMsConfig{
Driver: "test",
SyncInterval: arvados.Duration(10 * time.Millisecond),
@@ -160,7 +162,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
s.disp.setupOnce.Do(s.disp.initialize)
queue := &test.Queue{
MaxDispatchAttempts: 5,
- ChooseType: func(ctr *arvados.Container) (arvados.InstanceType, error) {
+ ChooseType: func(ctr *arvados.Container) ([]arvados.InstanceType, error) {
return ChooseInstanceType(s.cluster, ctr)
},
Logger: ctxlog.TestLogger(c),
@@ -205,9 +207,15 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
finishContainer(ctr)
return int(rand.Uint32() & 0x3)
}
+ var countCapacityErrors int64
n := 0
s.stubDriver.Queue = queue
- s.stubDriver.SetupVM = func(stubvm *test.StubVM) {
+ s.stubDriver.SetupVM = func(stubvm *test.StubVM) error {
+ if pt := stubvm.Instance().ProviderType(); pt == test.InstanceType(6).ProviderType {
+ c.Logf("test: returning capacity error for instance type %s", pt)
+ atomic.AddInt64(&countCapacityErrors, 1)
+ return test.CapacityError{InstanceTypeSpecific: true}
+ }
n++
stubvm.Boot = time.Now().Add(time.Duration(rand.Int63n(int64(5 * time.Millisecond))))
stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
@@ -235,6 +243,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
stubvm.CrunchRunCrashRate = 0.1
stubvm.ArvMountDeadlockRate = 0.1
}
+ return nil
}
s.stubDriver.Bugf = c.Errorf
@@ -270,6 +279,8 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
}
}
+ c.Check(countCapacityErrors, check.Not(check.Equals), int64(0))
+
req := httptest.NewRequest("GET", "/metrics", nil)
req.Header.Set("Authorization", "Bearer "+s.cluster.ManagementToken)
resp := httptest.NewRecorder()
diff --git a/lib/dispatchcloud/node_size.go b/lib/dispatchcloud/node_size.go
index 0b394f4cfe..802bc65c28 100644
--- a/lib/dispatchcloud/node_size.go
+++ b/lib/dispatchcloud/node_size.go
@@ -6,6 +6,7 @@ package dispatchcloud
import (
"errors"
+ "math"
"regexp"
"sort"
"strconv"
@@ -99,12 +100,16 @@ func versionLess(vs1 string, vs2 string) (bool, error) {
return v1 < v2, nil
}
-// ChooseInstanceType returns the cheapest available
-// arvados.InstanceType big enough to run ctr.
-func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvados.InstanceType, err error) {
+// ChooseInstanceType returns the arvados.InstanceTypes eligible to
+// run ctr, i.e., those that have enough RAM, VCPUs, etc., and are not
+// too expensive according to cluster configuration.
+//
+// The returned types are sorted with lower prices first.
+//
+// The error is non-nil if and only if the returned slice is empty.
+func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) ([]arvados.InstanceType, error) {
if len(cc.InstanceTypes) == 0 {
- err = ErrInstanceTypesNotConfigured
- return
+ return nil, ErrInstanceTypesNotConfigured
}
needScratch := EstimateScratchSpace(ctr)
@@ -121,31 +126,33 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
}
needRAM = (needRAM * 100) / int64(100-discountConfiguredRAMPercent)
- ok := false
+ maxPriceFactor := math.Max(cc.Containers.MaximumPriceFactor, 1)
+ var types []arvados.InstanceType
+ var maxPrice float64
for _, it := range cc.InstanceTypes {
driverInsuff, driverErr := versionLess(it.CUDA.DriverVersion, ctr.RuntimeConstraints.CUDA.DriverVersion)
capabilityInsuff, capabilityErr := versionLess(it.CUDA.HardwareCapability, ctr.RuntimeConstraints.CUDA.HardwareCapability)
switch {
// reasons to reject a node
- case ok && it.Price > best.Price: // already selected a node, and this one is more expensive
+ case maxPrice > 0 && it.Price > maxPrice: // too expensive
case int64(it.Scratch) < needScratch: // insufficient scratch
case int64(it.RAM) < needRAM: // insufficient RAM
case it.VCPUs < needVCPUs: // insufficient VCPUs
case it.Preemptible != ctr.SchedulingParameters.Preemptible: // wrong preemptable setting
- case it.Price == best.Price && (it.RAM < best.RAM || it.VCPUs < best.VCPUs): // same price, worse specs
case it.CUDA.DeviceCount < ctr.RuntimeConstraints.CUDA.DeviceCount: // insufficient CUDA devices
case ctr.RuntimeConstraints.CUDA.DeviceCount > 0 && (driverInsuff || driverErr != nil): // insufficient driver version
case ctr.RuntimeConstraints.CUDA.DeviceCount > 0 && (capabilityInsuff || capabilityErr != nil): // insufficient hardware capability
// Don't select this node
default:
// Didn't reject the node, so select it
- // Lower price || (same price && better specs)
- best = it
- ok = true
+ types = append(types, it)
+ if newmax := it.Price * maxPriceFactor; newmax < maxPrice || maxPrice == 0 {
+ maxPrice = newmax
+ }
}
}
- if !ok {
+ if len(types) == 0 {
availableTypes := make([]arvados.InstanceType, 0, len(cc.InstanceTypes))
for _, t := range cc.InstanceTypes {
availableTypes = append(availableTypes, t)
@@ -153,11 +160,39 @@ func ChooseInstanceType(cc *arvados.Cluster, ctr *arvados.Container) (best arvad
sort.Slice(availableTypes, func(a, b int) bool {
return availableTypes[a].Price < availableTypes[b].Price
})
- err = ConstraintsNotSatisfiableError{
+ return nil, ConstraintsNotSatisfiableError{
errors.New("constraints not satisfiable by any configured instance type"),
availableTypes,
}
- return
}
- return
+ sort.Slice(types, func(i, j int) bool {
+ if types[i].Price != types[j].Price {
+ // prefer lower price
+ return types[i].Price < types[j].Price
+ }
+ if types[i].RAM != types[j].RAM {
+ // if same price, prefer more RAM
+ return types[i].RAM > types[j].RAM
+ }
+ if types[i].VCPUs != types[j].VCPUs {
+ // if same price and RAM, prefer more VCPUs
+ return types[i].VCPUs > types[j].VCPUs
+ }
+ if types[i].Scratch != types[j].Scratch {
+ // if same price and RAM and VCPUs, prefer more scratch
+ return types[i].Scratch > types[j].Scratch
+ }
+ // no preference, just sort the same way each time
+ return types[i].Name < types[j].Name
+ })
+ // Truncate types at maxPrice. We rejected it.Price>maxPrice
+ // in the loop above, but at that point maxPrice wasn't
+ // necessarily the final (lowest) maxPrice.
+ for i, it := range types {
+ if i > 0 && it.Price > maxPrice {
+ types = types[:i]
+ break
+ }
+ }
+ return types, nil
}
diff --git a/lib/dispatchcloud/node_size_test.go b/lib/dispatchcloud/node_size_test.go
index 86bfbec7b6..5d2713e982 100644
--- a/lib/dispatchcloud/node_size_test.go
+++ b/lib/dispatchcloud/node_size_test.go
@@ -93,12 +93,51 @@ func (*NodeSizeSuite) TestChoose(c *check.C) {
KeepCacheRAM: 123456789,
},
})
- c.Check(err, check.IsNil)
- c.Check(best.Name, check.Equals, "best")
- c.Check(best.RAM >= 1234567890, check.Equals, true)
- c.Check(best.VCPUs >= 2, check.Equals, true)
- c.Check(best.Scratch >= 2*GiB, check.Equals, true)
+ c.Assert(err, check.IsNil)
+ c.Assert(best, check.Not(check.HasLen), 0)
+ c.Check(best[0].Name, check.Equals, "best")
+ c.Check(best[0].RAM >= 1234567890, check.Equals, true)
+ c.Check(best[0].VCPUs >= 2, check.Equals, true)
+ c.Check(best[0].Scratch >= 2*GiB, check.Equals, true)
+ for i := range best {
+ // If multiple instance types are returned
+ // then they should all have the same price,
+ // because we didn't set MaximumPriceFactor>1.
+ c.Check(best[i].Price, check.Equals, best[0].Price)
+ }
+ }
+}
+
+func (*NodeSizeSuite) TestMaximumPriceFactor(c *check.C) {
+ menu := map[string]arvados.InstanceType{
+ "best+7": {Price: 3.4, RAM: 8000000000, VCPUs: 8, Scratch: 64 * GiB, Name: "best+7"},
+ "best+5": {Price: 3.0, RAM: 8000000000, VCPUs: 8, Scratch: 16 * GiB, Name: "best+5"},
+ "best+3": {Price: 2.6, RAM: 4000000000, VCPUs: 8, Scratch: 16 * GiB, Name: "best+3"},
+ "best+2": {Price: 2.4, RAM: 4000000000, VCPUs: 8, Scratch: 4 * GiB, Name: "best+2"},
+ "best+1": {Price: 2.2, RAM: 2000000000, VCPUs: 4, Scratch: 4 * GiB, Name: "best+1"},
+ "best": {Price: 2.0, RAM: 2000000000, VCPUs: 4, Scratch: 2 * GiB, Name: "best"},
+ "small+1": {Price: 1.1, RAM: 1000000000, VCPUs: 2, Scratch: 16 * GiB, Name: "small+1"},
+ "small": {Price: 1.0, RAM: 2000000000, VCPUs: 2, Scratch: 1 * GiB, Name: "small"},
}
+ best, err := ChooseInstanceType(&arvados.Cluster{InstanceTypes: menu, Containers: arvados.ContainersConfig{
+ MaximumPriceFactor: 1.5,
+ }}, &arvados.Container{
+ Mounts: map[string]arvados.Mount{
+ "/tmp": {Kind: "tmp", Capacity: 2 * int64(GiB)},
+ },
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 2,
+ RAM: 987654321,
+ KeepCacheRAM: 123456789,
+ },
+ })
+ c.Assert(err, check.IsNil)
+ c.Assert(best, check.HasLen, 5)
+ c.Check(best[0].Name, check.Equals, "best") // best price is $2
+ c.Check(best[1].Name, check.Equals, "best+1")
+ c.Check(best[2].Name, check.Equals, "best+2")
+ c.Check(best[3].Name, check.Equals, "best+3")
+ c.Check(best[4].Name, check.Equals, "best+5") // max price is $2 * 1.5 = $3
}
func (*NodeSizeSuite) TestChooseWithBlobBuffersOverhead(c *check.C) {
@@ -121,7 +160,8 @@ func (*NodeSizeSuite) TestChooseWithBlobBuffersOverhead(c *check.C) {
},
})
c.Check(err, check.IsNil)
- c.Check(best.Name, check.Equals, "best")
+ c.Assert(best, check.HasLen, 1)
+ c.Check(best[0].Name, check.Equals, "best")
}
func (*NodeSizeSuite) TestChoosePreemptible(c *check.C) {
@@ -145,11 +185,12 @@ func (*NodeSizeSuite) TestChoosePreemptible(c *check.C) {
},
})
c.Check(err, check.IsNil)
- c.Check(best.Name, check.Equals, "best")
- c.Check(best.RAM >= 1234567890, check.Equals, true)
- c.Check(best.VCPUs >= 2, check.Equals, true)
- c.Check(best.Scratch >= 2*GiB, check.Equals, true)
- c.Check(best.Preemptible, check.Equals, true)
+ c.Assert(best, check.HasLen, 1)
+ c.Check(best[0].Name, check.Equals, "best")
+ c.Check(best[0].RAM >= 1234567890, check.Equals, true)
+ c.Check(best[0].VCPUs >= 2, check.Equals, true)
+ c.Check(best[0].Scratch >= 2*GiB, check.Equals, true)
+ c.Check(best[0].Preemptible, check.Equals, true)
}
func (*NodeSizeSuite) TestScratchForDockerImage(c *check.C) {
@@ -252,9 +293,10 @@ func (*NodeSizeSuite) TestChooseGPU(c *check.C) {
CUDA: tc.CUDA,
},
})
- if best.Name != "" {
+ if len(best) > 0 {
c.Check(err, check.IsNil)
- c.Check(best.Name, check.Equals, tc.SelectedInstance)
+ c.Assert(best, check.HasLen, 1)
+ c.Check(best[0].Name, check.Equals, tc.SelectedInstance)
} else {
c.Check(err, check.Not(check.IsNil))
}
diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go
index 3505c3e064..2f1f175890 100644
--- a/lib/dispatchcloud/scheduler/run_queue.go
+++ b/lib/dispatchcloud/scheduler/run_queue.go
@@ -163,10 +163,9 @@ func (sch *Scheduler) runQueue() {
tryrun:
for i, ent := range sorted {
- ctr, it := ent.Container, ent.InstanceType
+ ctr, types := ent.Container, ent.InstanceTypes
logger := sch.logger.WithFields(logrus.Fields{
"ContainerUUID": ctr.UUID,
- "InstanceType": it.Name,
})
if ctr.SchedulingParameters.Supervisor {
supervisors += 1
@@ -178,6 +177,35 @@ tryrun:
if _, running := running[ctr.UUID]; running || ctr.Priority < 1 {
continue
}
+ // If we have unalloc instances of any of the eligible
+ // instance types, unallocOK is true and unallocType
+ // is the lowest-cost type.
+ var unallocOK bool
+ var unallocType arvados.InstanceType
+ for _, it := range types {
+ if unalloc[it] > 0 {
+ unallocOK = true
+ unallocType = it
+ break
+ }
+ }
+ // If the pool is not reporting AtCapacity for any of
+ // the eligible instance types, availableOK is true
+ // and availableType is the lowest-cost type.
+ var availableOK bool
+ var availableType arvados.InstanceType
+ for _, it := range types {
+ if atcapacity[it.ProviderType] {
+ continue
+ } else if sch.pool.AtCapacity(it) {
+ atcapacity[it.ProviderType] = true
+ continue
+ } else {
+ availableOK = true
+ availableType = it
+ break
+ }
+ }
switch ctr.State {
case arvados.ContainerStateQueued:
if sch.maxConcurrency > 0 && trying >= sch.maxConcurrency {
@@ -185,14 +213,13 @@ tryrun:
continue
}
trying++
- if unalloc[it] < 1 && sch.pool.AtQuota() {
+ if !unallocOK && sch.pool.AtQuota() {
logger.Trace("not locking: AtQuota and no unalloc workers")
overquota = sorted[i:]
break tryrun
}
- if unalloc[it] < 1 && (atcapacity[it.ProviderType] || sch.pool.AtCapacity(it)) {
+ if !unallocOK && !availableOK {
logger.Trace("not locking: AtCapacity and no unalloc workers")
- atcapacity[it.ProviderType] = true
continue
}
if sch.pool.KillContainer(ctr.UUID, "about to lock") {
@@ -200,16 +227,36 @@ tryrun:
continue
}
go sch.lockContainer(logger, ctr.UUID)
- unalloc[it]--
+ unalloc[unallocType]--
case arvados.ContainerStateLocked:
if sch.maxConcurrency > 0 && trying >= sch.maxConcurrency {
logger.Tracef("not starting: already at maxConcurrency %d", sch.maxConcurrency)
continue
}
trying++
- if unalloc[it] > 0 {
- unalloc[it]--
- } else if sch.pool.AtQuota() {
+ if unallocOK {
+ // We have a suitable instance type,
+ // so mark it as allocated, and try to
+ // start the container.
+ unalloc[unallocType]--
+ logger = logger.WithField("InstanceType", unallocType)
+ if dontstart[unallocType] {
+ // We already tried & failed to start
+ // a higher-priority container on the
+ // same instance type. Don't let this
+ // one sneak in ahead of it.
+ } else if sch.pool.KillContainer(ctr.UUID, "about to start") {
+ logger.Info("not restarting yet: crunch-run process from previous attempt has not exited")
+ } else if sch.pool.StartContainer(unallocType, ctr) {
+ logger.Trace("StartContainer => true")
+ } else {
+ logger.Trace("StartContainer => false")
+ containerAllocatedWorkerBootingCount += 1
+ dontstart[unallocType] = true
+ }
+ continue
+ }
+ if sch.pool.AtQuota() {
// Don't let lower-priority containers
// starve this one by using keeping
// idle workers alive on different
@@ -217,7 +264,8 @@ tryrun:
logger.Trace("overquota")
overquota = sorted[i:]
break tryrun
- } else if atcapacity[it.ProviderType] || sch.pool.AtCapacity(it) {
+ }
+ if !availableOK {
// Continue trying lower-priority
// containers in case they can run on
// different instance types that are
@@ -231,41 +279,26 @@ tryrun:
// container A on the next call to
// runQueue(), rather than run
// container B now.
- //
- // TODO: try running this container on
- // a bigger (but not much more
- // expensive) instance type.
- logger.WithField("InstanceType", it.Name).Trace("at capacity")
- atcapacity[it.ProviderType] = true
+ logger.Trace("all eligible types at capacity")
continue
- } else if sch.pool.Create(it) {
- // Success. (Note pool.Create works
- // asynchronously and does its own
- // logging about the eventual outcome,
- // so we don't need to.)
- logger.Info("creating new instance")
- } else {
+ }
+ logger = logger.WithField("InstanceType", availableType)
+ if !sch.pool.Create(availableType) {
// Failed despite not being at quota,
// e.g., cloud ops throttled.
logger.Trace("pool declined to create new instance")
continue
}
-
- if dontstart[it] {
- // We already tried & failed to start
- // a higher-priority container on the
- // same instance type. Don't let this
- // one sneak in ahead of it.
- } else if sch.pool.KillContainer(ctr.UUID, "about to start") {
- logger.Info("not restarting yet: crunch-run process from previous attempt has not exited")
- } else if sch.pool.StartContainer(it, ctr) {
- logger.Trace("StartContainer => true")
- // Success.
- } else {
- logger.Trace("StartContainer => false")
- containerAllocatedWorkerBootingCount += 1
- dontstart[it] = true
- }
+ // Success. (Note pool.Create works
+ // asynchronously and does its own logging
+ // about the eventual outcome, so we don't
+ // need to.)
+ logger.Info("creating new instance")
+ // Don't bother trying to start the container
+ // yet -- obviously the instance will take
+ // some time to boot and become ready.
+ containerAllocatedWorkerBootingCount += 1
+ dontstart[availableType] = true
}
}
diff --git a/lib/dispatchcloud/scheduler/run_queue_test.go b/lib/dispatchcloud/scheduler/run_queue_test.go
index da6955029a..e4a05daba5 100644
--- a/lib/dispatchcloud/scheduler/run_queue_test.go
+++ b/lib/dispatchcloud/scheduler/run_queue_test.go
@@ -139,8 +139,8 @@ func (p *stubPool) StartContainer(it arvados.InstanceType, ctr arvados.Container
return true
}
-func chooseType(ctr *arvados.Container) (arvados.InstanceType, error) {
- return test.InstanceType(ctr.RuntimeConstraints.VCPUs), nil
+func chooseType(ctr *arvados.Container) ([]arvados.InstanceType, error) {
+ return []arvados.InstanceType{test.InstanceType(ctr.RuntimeConstraints.VCPUs)}, nil
}
var _ = check.Suite(&SchedulerSuite{})
@@ -364,7 +364,7 @@ func (*SchedulerSuite) TestInstanceCapacity(c *check.C) {
// type4, so we skip trying to create an instance for
// container3, skip locking container2, but do try to create a
// type1 instance for container1.
- c.Check(pool.starts, check.DeepEquals, []string{test.ContainerUUID(4), test.ContainerUUID(1)})
+ c.Check(pool.starts, check.DeepEquals, []string{test.ContainerUUID(4)})
c.Check(pool.shutdowns, check.Equals, 0)
c.Check(pool.creates, check.DeepEquals, []arvados.InstanceType{test.InstanceType(1)})
c.Check(queue.StateChanges(), check.HasLen, 0)
diff --git a/lib/dispatchcloud/test/queue.go b/lib/dispatchcloud/test/queue.go
index 2be8246bd6..ea2b98236f 100644
--- a/lib/dispatchcloud/test/queue.go
+++ b/lib/dispatchcloud/test/queue.go
@@ -22,7 +22,7 @@ type Queue struct {
// ChooseType will be called for each entry in Containers. It
// must not be nil.
- ChooseType func(*arvados.Container) (arvados.InstanceType, error)
+ ChooseType func(*arvados.Container) ([]arvados.InstanceType, error)
// Mimic railsapi implementation of MaxDispatchAttempts config
MaxDispatchAttempts int
@@ -167,12 +167,12 @@ func (q *Queue) Update() error {
ent.Container = ctr
upd[ctr.UUID] = ent
} else {
- it, _ := q.ChooseType(&ctr)
+ types, _ := q.ChooseType(&ctr)
ctr.Mounts = nil
upd[ctr.UUID] = container.QueueEnt{
- Container: ctr,
- InstanceType: it,
- FirstSeenAt: time.Now(),
+ Container: ctr,
+ InstanceTypes: types,
+ FirstSeenAt: time.Now(),
}
}
}
diff --git a/lib/dispatchcloud/test/stub_driver.go b/lib/dispatchcloud/test/stub_driver.go
index 6e0b129487..0a74d97606 100644
--- a/lib/dispatchcloud/test/stub_driver.go
+++ b/lib/dispatchcloud/test/stub_driver.go
@@ -34,7 +34,10 @@ type StubDriver struct {
// SetupVM, if set, is called upon creation of each new
// StubVM. This is the caller's opportunity to customize the
// VM's error rate and other behaviors.
- SetupVM func(*StubVM)
+ //
+ // If SetupVM returns an error, that error will be returned to
+ // the caller of Create(), and the new VM will be discarded.
+ SetupVM func(*StubVM) error
// Bugf, if set, is called if a bug is detected in the caller
// or stub. Typically set to (*check.C)Errorf. If unset,
@@ -152,7 +155,10 @@ func (sis *StubInstanceSet) Create(it arvados.InstanceType, image cloud.ImageID,
Exec: svm.Exec,
}
if setup := sis.driver.SetupVM; setup != nil {
- setup(svm)
+ err := setup(svm)
+ if err != nil {
+ return nil, err
+ }
}
sis.servers[svm.id] = svm
return svm.Instance(), nil
@@ -195,6 +201,12 @@ type RateLimitError struct{ Retry time.Time }
func (e RateLimitError) Error() string { return fmt.Sprintf("rate limited until %s", e.Retry) }
func (e RateLimitError) EarliestRetry() time.Time { return e.Retry }
+type CapacityError struct{ InstanceTypeSpecific bool }
+
+func (e CapacityError) Error() string { return "insufficient capacity" }
+func (e CapacityError) IsCapacityError() bool { return true }
+func (e CapacityError) IsInstanceTypeSpecific() bool { return e.InstanceTypeSpecific }
+
// StubVM is a fake server that runs an SSH service. It represents a
// VM running in a fake cloud.
//
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index a3e54952da..0afcf2be3c 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -515,6 +515,7 @@ type ContainersConfig struct {
SupportedDockerImageFormats StringSet
AlwaysUsePreemptibleInstances bool
PreemptiblePriceFactor float64
+ MaximumPriceFactor float64
RuntimeEngine string
LocalKeepBlobBuffersPerVCPU int
LocalKeepLogsToContainerLog string
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 1c0f6ad28f..5a9ef91c3d 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -197,14 +197,16 @@ func (disp *Dispatcher) sbatchArgs(container arvados.Container) ([]string, error
if disp.cluster == nil {
// no instance types configured
args = append(args, disp.slurmConstraintArgs(container)...)
- } else if it, err := dispatchcloud.ChooseInstanceType(disp.cluster, &container); err == dispatchcloud.ErrInstanceTypesNotConfigured {
+ } else if types, err := dispatchcloud.ChooseInstanceType(disp.cluster, &container); err == dispatchcloud.ErrInstanceTypesNotConfigured {
// ditto
args = append(args, disp.slurmConstraintArgs(container)...)
} else if err != nil {
return nil, err
} else {
- // use instancetype constraint instead of slurm mem/cpu/tmp specs
- args = append(args, "--constraint=instancetype="+it.Name)
+ // use instancetype constraint instead of slurm
+ // mem/cpu/tmp specs (note types[0] is the lowest-cost
+ // suitable instance type)
+ args = append(args, "--constraint=instancetype="+types[0].Name)
}
if len(container.SchedulingParameters.Partitions) > 0 {
commit 39134d21b871113ded0531667f98f3936969dc93
Author: Tom Clegg <tom at curii.com>
Date: Tue Oct 17 18:53:08 2023 -0400
Merge branch '21055-sysctl-in-docker'
closes #21055
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index cbf050d7dc..08daec6c2f 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -232,7 +232,7 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
}
if dev || test {
- if havedockerversion, err := exec.Command("docker", "--version").CombinedOutput(); err == nil {
+ if havedockerversion, err2 := exec.Command("docker", "--version").CombinedOutput(); err2 == nil {
logger.Printf("%s installed, assuming that version is ok", bytes.TrimSuffix(havedockerversion, []byte("\n")))
} else if osv.Debian {
var codename string
@@ -241,6 +241,8 @@ func (inst *installCommand) RunCommand(prog string, args []string, stdin io.Read
codename = "buster"
case 11:
codename = "bullseye"
+ case 12:
+ codename = "bookworm"
default:
err = fmt.Errorf("don't know how to install docker-ce for debian %d", osv.Major)
return 1
@@ -260,6 +262,21 @@ DEBIAN_FRONTEND=noninteractive apt-get --yes --no-install-recommends install doc
err = fmt.Errorf("don't know how to install docker for osversion %v", osv)
return 1
}
+
+ err = inst.runBash(`
+key=fs.inotify.max_user_watches
+min=524288
+if [[ "$(sysctl --values "${key}")" -lt "${min}" ]]; then
+ sysctl "${key}=${min}"
+ # writing sysctl worked, so we should make it permanent
+ echo "${key}=${min}" | tee -a /etc/sysctl.conf
+ sysctl -p
+fi
+`, stdout, stderr)
+ if err != nil {
+ err = fmt.Errorf("couldn't set fs.inotify.max_user_watches value. (Is this a docker container? Fix this on the docker host by adding fs.inotify.max_user_watches=524288 to /etc/sysctl.conf and running `sysctl -p`)")
+ return 1
+ }
}
os.Mkdir("/var/lib/arvados", 0755)
commit 4c41a14e7452da1b97877af38795528d410f48a2
Author: Tom Clegg <tom at curii.com>
Date: Fri Oct 20 11:26:35 2023 -0400
Merge branch '21086-cert-source'
fixes #21086
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index ef04551883..0e0d3c43e4 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -46,6 +46,8 @@ import (
type command struct{}
+var arvadosCertPath = "/etc/arvados/ca-certificates.crt"
+
var Command = command{}
// ConfigData contains environment variables and (when needed) cluster
@@ -494,7 +496,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
}
}
- if bind == "/etc/arvados/ca-certificates.crt" {
+ if bind == arvadosCertPath {
needCertMount = false
}
@@ -644,10 +646,19 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
}
if needCertMount && runner.Container.RuntimeConstraints.API {
- for _, certfile := range arvadosclient.CertFiles {
- _, err := os.Stat(certfile)
- if err == nil {
- bindmounts["/etc/arvados/ca-certificates.crt"] = bindmount{HostPath: certfile, ReadOnly: true}
+ for _, certfile := range []string{
+ // Populated by caller, or sdk/go/arvados init(), or test suite:
+ os.Getenv("SSL_CERT_FILE"),
+ // Copied from Go 1.21 stdlib (src/crypto/x509/root_linux.go):
+ "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
+ "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
+ "/etc/ssl/ca-bundle.pem", // OpenSUSE
+ "/etc/pki/tls/cacert.pem", // OpenELEC
+ "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
+ "/etc/ssl/cert.pem", // Alpine Linux
+ } {
+ if _, err := os.Stat(certfile); err == nil {
+ bindmounts[arvadosCertPath] = bindmount{HostPath: certfile, ReadOnly: true}
break
}
}
@@ -1996,7 +2007,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
time.Sleep(*sleep)
if *caCertsPath != "" {
- arvadosclient.CertFiles = []string{*caCertsPath}
+ os.Setenv("SSL_CERT_FILE", *caCertsPath)
}
keepstore, err := startLocalKeepstore(conf, io.MultiWriter(&keepstoreLogbuf, stderr))
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 30f3ea4bb4..c533821351 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -1413,11 +1413,11 @@ func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, er
return nil, nil
}
-func stubCert(temp string) string {
+func stubCert(c *C, temp string) string {
path := temp + "/ca-certificates.crt"
- crt, _ := os.Create(path)
- crt.Close()
- arvadosclient.CertFiles = []string{path}
+ err := os.WriteFile(path, []byte{}, 0666)
+ c.Assert(err, IsNil)
+ os.Setenv("SSL_CERT_FILE", path)
return path
}
@@ -1432,7 +1432,7 @@ func (s *TestSuite) TestSetupMounts(c *C) {
realTemp := c.MkDir()
certTemp := c.MkDir()
- stubCertPath := stubCert(certTemp)
+ stubCertPath := stubCert(c, certTemp)
cr.parentTemp = realTemp
i := 0
diff --git a/sdk/go/arvados/tls_certs.go b/sdk/go/arvados/tls_certs.go
new file mode 100644
index 0000000000..db52781339
--- /dev/null
+++ b/sdk/go/arvados/tls_certs.go
@@ -0,0 +1,23 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "os"
+
+// Load root CAs from /etc/arvados/ca-certificates.crt if it exists
+// and SSL_CERT_FILE does not already specify a different file.
+func init() {
+ envvar := "SSL_CERT_FILE"
+ certfile := "/etc/arvados/ca-certificates.crt"
+ if os.Getenv(envvar) != "" {
+ // Caller has already specified SSL_CERT_FILE.
+ return
+ }
+ if _, err := os.ReadFile(certfile); err != nil {
+ // Custom cert file is not present/readable.
+ return
+ }
+ os.Setenv(envvar, certfile)
+}
diff --git a/sdk/go/arvados/tls_certs_test.go b/sdk/go/arvados/tls_certs_test.go
new file mode 100644
index 0000000000..7900867715
--- /dev/null
+++ b/sdk/go/arvados/tls_certs_test.go
@@ -0,0 +1,32 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "os"
+ "os/exec"
+
+ check "gopkg.in/check.v1"
+)
+
+type tlsCertsSuite struct{}
+
+var _ = check.Suite(&tlsCertsSuite{})
+
+func (s *tlsCertsSuite) TestCustomCert(c *check.C) {
+ certfile := "/etc/arvados/ca-certificates.crt"
+ if _, err := os.Stat(certfile); err != nil {
+ c.Skip("custom cert file " + certfile + " does not exist")
+ }
+ out, err := exec.Command("bash", "-c", "SSL_CERT_FILE= go run tls_certs_test_showenv.go").CombinedOutput()
+ c.Logf("%s", out)
+ c.Assert(err, check.IsNil)
+ c.Check(string(out), check.Equals, certfile+"\n")
+
+ out, err = exec.Command("bash", "-c", "SSL_CERT_FILE=/dev/null go run tls_certs_test_showenv.go").CombinedOutput()
+ c.Logf("%s", out)
+ c.Assert(err, check.IsNil)
+ c.Check(string(out), check.Equals, "/dev/null\n")
+}
diff --git a/sdk/go/arvados/tls_certs_test_showenv.go b/sdk/go/arvados/tls_certs_test_showenv.go
new file mode 100644
index 0000000000..f2622cf11d
--- /dev/null
+++ b/sdk/go/arvados/tls_certs_test_showenv.go
@@ -0,0 +1,22 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+//go:build ignore
+
+// This is a test program invoked by tls_certs_test.go
+
+package main
+
+import (
+ "fmt"
+ "os"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+var _ = arvados.Client{}
+
+func main() {
+ fmt.Println(os.Getenv("SSL_CERT_FILE"))
+}
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 516187c0e6..461320eca9 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -9,16 +9,12 @@ package arvadosclient
import (
"bytes"
"crypto/tls"
- "crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
- "io/ioutil"
- "log"
"net/http"
"net/url"
- "os"
"strings"
"sync"
"time"
@@ -121,40 +117,10 @@ type ArvadosClient struct {
RequestID string
}
-var CertFiles = []string{
- "/etc/arvados/ca-certificates.crt",
- "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
- "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL
-}
-
// MakeTLSConfig sets up TLS configuration for communicating with
// Arvados and Keep services.
func MakeTLSConfig(insecure bool) *tls.Config {
- tlsconfig := tls.Config{InsecureSkipVerify: insecure}
-
- if !insecure {
- // Use the first entry in CertFiles that we can read
- // certificates from. If none of those work out, use
- // the Go defaults.
- certs := x509.NewCertPool()
- for _, file := range CertFiles {
- data, err := ioutil.ReadFile(file)
- if err != nil {
- if !os.IsNotExist(err) {
- log.Printf("proceeding without loading cert file %q: %s", file, err)
- }
- continue
- }
- if !certs.AppendCertsFromPEM(data) {
- log.Printf("unable to load any certificates from %v", file)
- continue
- }
- tlsconfig.RootCAs = certs
- break
- }
- }
-
- return &tlsconfig
+ return &tls.Config{InsecureSkipVerify: insecure}
}
// New returns an ArvadosClient using the given arvados.Client
commit d8dccc38423c4ac44e30d1b15297874c813ecf55
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Oct 19 14:26:44 2023 -0400
Merge branch '21030-update-perm-cte' refs #21030
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/api/app/models/group.rb b/services/api/app/models/group.rb
index 5c0aeba589..0c9248b819 100644
--- a/services/api/app/models/group.rb
+++ b/services/api/app/models/group.rb
@@ -163,63 +163,70 @@ class Group < ArvadosModel
# Remove groups that don't belong from trash
# Add/update groups that do belong in the trash
- temptable = "group_subtree_#{rand(2**64).to_s(10)}"
- ActiveRecord::Base.connection.exec_query(
- "create temporary table #{temptable} on commit drop " +
- "as select * from project_subtree_with_trash_at($1, LEAST($2, $3)::timestamp)",
+ frozen_descendants = ActiveRecord::Base.connection.exec_query(%{
+with temptable as (select * from project_subtree_with_trash_at($1, LEAST($2, $3)::timestamp))
+ select uuid from frozen_groups, temptable where uuid = target_uuid
+},
"Group.update_trash.select",
[[nil, self.uuid],
[nil, TrashedGroup.find_by_group_uuid(self.owner_uuid).andand.trash_at],
[nil, self.trash_at]])
- frozen_descendants = ActiveRecord::Base.connection.exec_query(
- "select uuid from frozen_groups, #{temptable} where uuid = target_uuid",
- "Group.update_trash.check_frozen")
if frozen_descendants.any?
raise ArgumentError.new("cannot trash project containing frozen project #{frozen_descendants[0]["uuid"]}")
end
- ActiveRecord::Base.connection.exec_delete(
- "delete from trashed_groups where group_uuid in (select target_uuid from #{temptable} where trash_at is NULL)",
- "Group.update_trash.delete")
- ActiveRecord::Base.connection.exec_query(
- "insert into trashed_groups (group_uuid, trash_at) "+
- "select target_uuid as group_uuid, trash_at from #{temptable} where trash_at is not NULL " +
- "on conflict (group_uuid) do update set trash_at=EXCLUDED.trash_at",
- "Group.update_trash.insert")
- ActiveRecord::Base.connection.exec_query(
- "select container_uuid from container_requests where " +
- "owner_uuid in (select target_uuid from #{temptable}) and " +
- "requesting_container_uuid is NULL and state = 'Committed' and container_uuid is not NULL",
- "Group.update_trash.update_priorities").each do |container_uuid|
+
+ ActiveRecord::Base.connection.exec_query(%{
+with temptable as (select * from project_subtree_with_trash_at($1, LEAST($2, $3)::timestamp)),
+
+delete_rows as (delete from trashed_groups where group_uuid in (select target_uuid from temptable where trash_at is NULL)),
+
+insert_rows as (insert into trashed_groups (group_uuid, trash_at)
+ select target_uuid as group_uuid, trash_at from temptable where trash_at is not NULL
+ on conflict (group_uuid) do update set trash_at=EXCLUDED.trash_at)
+
+select container_uuid from container_requests where
+ owner_uuid in (select target_uuid from temptable) and
+ requesting_container_uuid is NULL and state = 'Committed' and container_uuid is not NULL
+},
+ "Group.update_trash.select",
+ [[nil, self.uuid],
+ [nil, TrashedGroup.find_by_group_uuid(self.owner_uuid).andand.trash_at],
+ [nil, self.trash_at]]).each do |container_uuid|
update_priorities container_uuid["container_uuid"]
end
end
def update_frozen
return unless saved_change_to_frozen_by_uuid? || saved_change_to_owner_uuid?
- temptable = "group_subtree_#{rand(2**64).to_s(10)}"
- ActiveRecord::Base.connection.exec_query(
- "create temporary table #{temptable} on commit drop as select * from project_subtree_with_is_frozen($1,$2)",
- "Group.update_frozen.select",
- [[nil, self.uuid],
- [nil, !self.frozen_by_uuid.nil?]])
+
if frozen_by_uuid
- rows = ActiveRecord::Base.connection.exec_query(
- "select cr.uuid, cr.state from container_requests cr, #{temptable} frozen " +
- "where cr.owner_uuid = frozen.uuid and frozen.is_frozen " +
- "and cr.state not in ($1, $2) limit 1",
- "Group.update_frozen.check_container_requests",
- [[nil, ContainerRequest::Uncommitted],
- [nil, ContainerRequest::Final]])
+ rows = ActiveRecord::Base.connection.exec_query(%{
+with temptable as (select * from project_subtree_with_is_frozen($1,$2))
+
+select cr.uuid, cr.state from container_requests cr, temptable frozen
+ where cr.owner_uuid = frozen.uuid and frozen.is_frozen
+ and cr.state not in ($3, $4) limit 1
+},
+ "Group.update_frozen.check_container_requests",
+ [[nil, self.uuid],
+ [nil, !self.frozen_by_uuid.nil?],
+ [nil, ContainerRequest::Uncommitted],
+ [nil, ContainerRequest::Final]])
if rows.any?
raise ArgumentError.new("cannot freeze project containing container request #{rows.first['uuid']} with state = #{rows.first['state']}")
end
end
- ActiveRecord::Base.connection.exec_delete(
- "delete from frozen_groups where uuid in (select uuid from #{temptable} where not is_frozen)",
- "Group.update_frozen.delete")
- ActiveRecord::Base.connection.exec_query(
- "insert into frozen_groups (uuid) select uuid from #{temptable} where is_frozen on conflict do nothing",
- "Group.update_frozen.insert")
+
+ActiveRecord::Base.connection.exec_query(%{
+with temptable as (select * from project_subtree_with_is_frozen($1,$2)),
+
+delete_rows as (delete from frozen_groups where uuid in (select uuid from temptable where not is_frozen))
+
+insert into frozen_groups (uuid) select uuid from temptable where is_frozen on conflict do nothing
+}, "Group.update_frozen.update",
+ [[nil, self.uuid],
+ [nil, !self.frozen_by_uuid.nil?]])
+
end
def before_ownership_change
diff --git a/services/api/db/migrate/20231013000000_compute_permission_index.rb b/services/api/db/migrate/20231013000000_compute_permission_index.rb
new file mode 100644
index 0000000000..ecd85ef1bc
--- /dev/null
+++ b/services/api/db/migrate/20231013000000_compute_permission_index.rb
@@ -0,0 +1,27 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class ComputePermissionIndex < ActiveRecord::Migration[5.2]
+ def up
+ # The inner part of compute_permission_subgraph has a query clause like this:
+ #
+ # where u.perm_origin_uuid = m.target_uuid AND m.traverse_owned
+ # AND (m.user_uuid = m.target_uuid or m.target_uuid not like '_____-tpzed-_______________')
+ #
+ # This will end up doing a sequential scan on
+ # materialized_permissions, which can easily have millions of
+ # rows, unless we fully index the table for this query. In one test,
+ # this brought the compute_permission_subgraph query from over 6
+ # seconds down to 250ms.
+ #
+ ActiveRecord::Base.connection.execute "drop index if exists index_materialized_permissions_target_is_not_user"
+ ActiveRecord::Base.connection.execute %{
+create index index_materialized_permissions_target_is_not_user on materialized_permissions (target_uuid, traverse_owned, (user_uuid = target_uuid or target_uuid not like '_____-tpzed-_______________'));
+}
+ end
+
+ def down
+ ActiveRecord::Base.connection.execute "drop index if exists index_materialized_permissions_target_is_not_user"
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index a26db2e5db..c0d4263d97 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -62,10 +62,10 @@ with
permission (permission origin is self).
*/
perm_from_start(perm_origin_uuid, target_uuid, val, traverse_owned) as (
-
+
WITH RECURSIVE
traverse_graph(origin_uuid, target_uuid, val, traverse_owned, starting_set) as (
-
+
values (perm_origin_uuid, starting_uuid, starting_perm,
should_traverse_owned(starting_uuid, starting_perm),
(perm_origin_uuid = starting_uuid or starting_uuid not like '_____-tpzed-_______________'))
@@ -107,10 +107,10 @@ case (edges.edge_id = perm_edge_id)
can_manage permission granted by ownership.
*/
additional_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
-
+
WITH RECURSIVE
traverse_graph(origin_uuid, target_uuid, val, traverse_owned, starting_set) as (
-
+
select edges.tail_uuid as origin_uuid, edges.head_uuid as target_uuid, edges.val,
should_traverse_owned(edges.head_uuid, edges.val),
edges.head_uuid like '_____-j7d0g-_______________'
@@ -342,30 +342,6 @@ SET default_tablespace = '';
SET default_with_oids = false;
---
--- Name: groups; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.groups (
- id bigint NOT NULL,
- uuid character varying(255),
- owner_uuid character varying(255),
- created_at timestamp without time zone NOT NULL,
- modified_by_client_uuid character varying(255),
- modified_by_user_uuid character varying(255),
- modified_at timestamp without time zone,
- name character varying(255) NOT NULL,
- description character varying(524288),
- updated_at timestamp without time zone NOT NULL,
- group_class character varying(255),
- trash_at timestamp without time zone,
- is_trashed boolean DEFAULT false NOT NULL,
- delete_at timestamp without time zone,
- properties jsonb DEFAULT '{}'::jsonb,
- frozen_by_uuid character varying
-);
-
-
--
-- Name: api_client_authorizations; Type: TABLE; Schema: public; Owner: -
--
@@ -690,6 +666,30 @@ CREATE TABLE public.frozen_groups (
);
+--
+-- Name: groups; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.groups (
+ id bigint NOT NULL,
+ uuid character varying(255),
+ owner_uuid character varying(255),
+ created_at timestamp without time zone NOT NULL,
+ modified_by_client_uuid character varying(255),
+ modified_by_user_uuid character varying(255),
+ modified_at timestamp without time zone,
+ name character varying(255) NOT NULL,
+ description character varying(524288),
+ updated_at timestamp without time zone NOT NULL,
+ group_class character varying(255),
+ trash_at timestamp without time zone,
+ is_trashed boolean DEFAULT false NOT NULL,
+ delete_at timestamp without time zone,
+ properties jsonb DEFAULT '{}'::jsonb,
+ frozen_by_uuid character varying
+);
+
+
--
-- Name: groups_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
@@ -2590,6 +2590,13 @@ CREATE INDEX index_logs_on_summary ON public.logs USING btree (summary);
CREATE UNIQUE INDEX index_logs_on_uuid ON public.logs USING btree (uuid);
+--
+-- Name: index_materialized_permissions_target_is_not_user; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_materialized_permissions_target_is_not_user ON public.materialized_permissions USING btree (target_uuid, traverse_owned, ((((user_uuid)::text = (target_uuid)::text) OR ((target_uuid)::text !~~ '_____-tpzed-_______________'::text))));
+
+
--
-- Name: index_nodes_on_created_at; Type: INDEX; Schema: public; Owner: -
--
@@ -3308,6 +3315,5 @@ INSERT INTO "schema_migrations" (version) VALUES
('20230503224107'),
('20230815160000'),
('20230821000000'),
-('20230922000000');
-
-
+('20230922000000'),
+('20231013000000');
diff --git a/services/api/lib/update_permissions.rb b/services/api/lib/update_permissions.rb
index b7e5476404..5c8072b48a 100644
--- a/services/api/lib/update_permissions.rb
+++ b/services/api/lib/update_permissions.rb
@@ -100,44 +100,41 @@ def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
# tested this on Postgres 9.6, so in the future we should reevaluate
# the performance & query plan on Postgres 12.
#
+ # Update: as of 2023-10-13, incorrect merge join behavior is still
+ # observed on at least one major user installation that is using
+ # Postgres 14, so it seems this workaround is still needed.
+ #
# https://git.furworks.de/opensourcemirror/postgresql/commit/a314c34079cf06d05265623dd7c056f8fa9d577f
#
# Disable merge join for just this query (also local for this transaction), then reenable it.
ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to false;"
- temptable_perms = "temp_perms_#{rand(2**64).to_s(10)}"
- ActiveRecord::Base.connection.exec_query %{
-create temporary table #{temptable_perms} on commit drop
-as select * from compute_permission_subgraph($1, $2, $3, $4)
-},
- 'update_permissions.select',
- [[nil, perm_origin_uuid],
- [nil, starting_uuid],
- [nil, perm_level],
- [nil, edge_id]]
-
- ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to true;"
-
- # Now that we have recomputed a set of permissions, delete any
- # rows from the materialized_permissions table where (target_uuid,
- # user_uuid) is not present or has perm_level=0 in the recomputed
- # set.
- ActiveRecord::Base.connection.exec_delete %{
-delete from #{PERMISSION_VIEW} where
- target_uuid in (select target_uuid from #{temptable_perms}) and
- not exists (select 1 from #{temptable_perms}
- where target_uuid=#{PERMISSION_VIEW}.target_uuid and
- user_uuid=#{PERMISSION_VIEW}.user_uuid and
- val>0)
-},
- "update_permissions.delete"
-
- # Now insert-or-update permissions in the recomputed set. The
- # WHERE clause is important to avoid redundantly updating rows
- # that haven't actually changed.
ActiveRecord::Base.connection.exec_query %{
+with temptable_perms as (
+ select * from compute_permission_subgraph($1, $2, $3, $4)),
+
+/*
+ Now that we have recomputed a set of permissions, delete any
+ rows from the materialized_permissions table where (target_uuid,
+ user_uuid) is not present or has perm_level=0 in the recomputed
+ set.
+*/
+delete_rows as (
+ delete from #{PERMISSION_VIEW} where
+ target_uuid in (select target_uuid from temptable_perms) and
+ not exists (select 1 from temptable_perms
+ where target_uuid=#{PERMISSION_VIEW}.target_uuid and
+ user_uuid=#{PERMISSION_VIEW}.user_uuid and
+ val>0)
+)
+
+/*
+ Now insert-or-update permissions in the recomputed set. The
+ WHERE clause is important to avoid redundantly updating rows
+ that haven't actually changed.
+*/
insert into #{PERMISSION_VIEW} (user_uuid, target_uuid, perm_level, traverse_owned)
- select user_uuid, target_uuid, val as perm_level, traverse_owned from #{temptable_perms} where val>0
+ select user_uuid, target_uuid, val as perm_level, traverse_owned from temptable_perms where val>0
on conflict (user_uuid, target_uuid) do update
set perm_level=EXCLUDED.perm_level, traverse_owned=EXCLUDED.traverse_owned
where #{PERMISSION_VIEW}.user_uuid=EXCLUDED.user_uuid and
@@ -145,7 +142,11 @@ where #{PERMISSION_VIEW}.user_uuid=EXCLUDED.user_uuid and
(#{PERMISSION_VIEW}.perm_level != EXCLUDED.perm_level or
#{PERMISSION_VIEW}.traverse_owned != EXCLUDED.traverse_owned);
},
- "update_permissions.insert"
+ 'update_permissions.select',
+ [[nil, perm_origin_uuid],
+ [nil, starting_uuid],
+ [nil, perm_level],
+ [nil, edge_id]]
if perm_level>0
check_permissions_against_full_refresh
commit cf182ddd634cc754c5444ad4291f6027fbf07acf
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Oct 19 14:44:52 2023 -0400
Merge branch '20825-cwl-separate-runner' refs #20825
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index f4246ed70a..389add4104 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -429,3 +429,22 @@ $graph:
doc: |
If the container failed on its first run, re-submit the
container with the RAM request multiplied by this factor.
+
+- name: SeparateRunner
+ type: record
+ extends: cwl:ProcessRequirement
+ inVocab: false
+ doc: |
+ Indicates that a subworkflow should run in a separate
+ arvados-cwl-runner process.
+ fields:
+ - name: class
+ type: string
+ doc: "Always 'arv:SeparateRunner'"
+ jsonldPredicate:
+ _id: "@type"
+ _type: "@vocab"
+ - name: runnerProcessName
+ type: ['null', string, cwl:Expression]
+ doc: |
+ Custom name to use for the runner process
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index ea7c9f7a33..6e3e42975e 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -593,7 +593,7 @@ class RunnerContainer(Runner):
"ram": 1024*1024 * (math.ceil(self.submit_runner_ram) + math.ceil(self.collection_cache_size)),
"API": True
},
- "use_existing": False, # Never reuse the runner container - see #15497.
+ "use_existing": self.reuse_runner,
"properties": {}
}
@@ -617,6 +617,8 @@ class RunnerContainer(Runner):
"content": packed
}
container_req["properties"]["template_uuid"] = self.embedded_tool.tool["id"][6:33]
+ elif self.embedded_tool.tool.get("id", "").startswith("file:"):
+ raise WorkflowException("Tool id '%s' is a local file but expected keep: or arvwf:" % self.embedded_tool.tool.get("id"))
else:
main = self.loadingContext.loader.idx["_:main"]
if main.get("id") == "_:main":
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 3ad2c6419a..c592b83dc7 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -38,6 +38,7 @@ import ruamel.yaml as yaml
from .runner import (upload_dependencies, packed_workflow, upload_workflow_collection,
trim_anonymous_location, remove_redundant_fields, discover_secondary_files,
make_builder, arvados_jobs_image, FileUpdates)
+from .arvcontainer import RunnerContainer
from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
from ._version import __version__
@@ -147,7 +148,7 @@ def make_wrapper_workflow(arvRunner, main, packed, project_uuid, name, git_info,
def rel_ref(s, baseuri, urlexpander, merged_map, jobmapper):
- if s.startswith("keep:"):
+ if s.startswith("keep:") or s.startswith("arvwf:"):
return s
uri = urlexpander(s, baseuri)
@@ -616,17 +617,8 @@ class ArvadosWorkflow(Workflow):
super(ArvadosWorkflow, self).__init__(toolpath_object, self.loadingContext)
self.cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
- def job(self, joborder, output_callback, runtimeContext):
-
- builder = make_builder(joborder, self.hints, self.requirements, runtimeContext, self.metadata)
- runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
-
- req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
- if not req:
- return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
-
- # RunInSingleContainer is true
+ def runInSingleContainer(self, joborder, output_callback, runtimeContext, builder):
with SourceLine(self.tool, None, WorkflowException, logger.isEnabledFor(logging.DEBUG)):
if "id" not in self.tool:
raise WorkflowException("%s object must have 'id'" % (self.tool["class"]))
@@ -789,6 +781,51 @@ class ArvadosWorkflow(Workflow):
})
return ArvadosCommandTool(self.arvrunner, wf_runner, self.loadingContext).job(joborder_resolved, output_callback, runtimeContext)
+
+ def separateRunner(self, joborder, output_callback, runtimeContext, req, builder):
+
+ name = runtimeContext.name
+
+ rpn = req.get("runnerProcessName")
+ if rpn:
+ name = builder.do_eval(rpn)
+
+ return RunnerContainer(self.arvrunner,
+ self,
+ self.loadingContext,
+ runtimeContext.enable_reuse,
+ None,
+ None,
+ submit_runner_ram=runtimeContext.submit_runner_ram,
+ name=name,
+ on_error=runtimeContext.on_error,
+ submit_runner_image=runtimeContext.submit_runner_image,
+ intermediate_output_ttl=runtimeContext.intermediate_output_ttl,
+ merged_map=None,
+ priority=runtimeContext.priority,
+ secret_store=self.arvrunner.secret_store,
+ collection_cache_size=runtimeContext.collection_cache_size,
+ collection_cache_is_default=self.arvrunner.should_estimate_cache_size,
+ git_info=runtimeContext.git_info,
+ reuse_runner=True).job(joborder, output_callback, runtimeContext)
+
+
+ def job(self, joborder, output_callback, runtimeContext):
+
+ builder = make_builder(joborder, self.hints, self.requirements, runtimeContext, self.metadata)
+ runtimeContext = set_cluster_target(self.tool, self.arvrunner, builder, runtimeContext)
+
+ req, _ = self.get_requirement("http://arvados.org/cwl#RunInSingleContainer")
+ if req:
+ return self.runInSingleContainer(joborder, output_callback, runtimeContext, builder)
+
+ req, _ = self.get_requirement("http://arvados.org/cwl#SeparateRunner")
+ if req:
+ return self.separateRunner(joborder, output_callback, runtimeContext, req, builder)
+
+ return super(ArvadosWorkflow, self).job(joborder, output_callback, runtimeContext)
+
+
def make_workflow_step(self,
toolpath_object, # type: Dict[Text, Any]
pos, # type: int
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index dd64879b9f..0439cb5b15 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -45,6 +45,7 @@ class ArvRuntimeContext(RuntimeContext):
self.prefer_cached_downloads = False
self.cached_docker_lookups = {}
self.print_keep_deps = False
+ self.git_info = {}
super(ArvRuntimeContext, self).__init__(kwargs)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 677e10d265..2db6a9bfe2 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -603,6 +603,8 @@ The 'jobs' API is no longer supported.
if git_info[g]:
logger.info(" %s: %s", g.split("#", 1)[1], git_info[g])
+ runtimeContext.git_info = git_info
+
workbench1 = self.api.config()["Services"]["Workbench1"]["ExternalURL"]
workbench2 = self.api.config()["Services"]["Workbench2"]["ExternalURL"]
controller = self.api.config()["Services"]["Controller"]["ExternalURL"]
@@ -874,7 +876,8 @@ The 'jobs' API is no longer supported.
if (self.task_queue.in_flight + len(self.processes)) > 0:
self.workflow_eval_lock.wait(3)
else:
- logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.")
+ if self.final_status is None:
+ logger.error("Workflow is deadlocked, no runnable processes and not waiting on any pending processes.")
break
if self.stop_polling.is_set():
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index f6aab4b93f..f52768d3d3 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -828,7 +828,8 @@ class Runner(Process):
priority=None, secret_store=None,
collection_cache_size=256,
collection_cache_is_default=True,
- git_info=None):
+ git_info=None,
+ reuse_runner=False):
self.loadingContext = loadingContext.copy()
@@ -861,6 +862,7 @@ class Runner(Process):
self.enable_dev = self.loadingContext.enable_dev
self.git_info = git_info
self.fast_parser = self.loadingContext.fast_parser
+ self.reuse_runner = reuse_runner
self.submit_runner_cores = 1
self.submit_runner_ram = 1024 # defaut 1 GiB
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index a93c64a224..e0bdd8a5a3 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -494,3 +494,9 @@
output: {}
tool: oom/19975-oom3.cwl
doc: "Test feature 19975 - retry on custom error"
+
+- job: null
+ output:
+ out: out
+ tool: wf/runseparate-wf.cwl
+ doc: "test arv:SeparateRunner"
diff --git a/sdk/cwl/tests/wf/runseparate-wf.cwl b/sdk/cwl/tests/wf/runseparate-wf.cwl
new file mode 100644
index 0000000000..e4ab627256
--- /dev/null
+++ b/sdk/cwl/tests/wf/runseparate-wf.cwl
@@ -0,0 +1,68 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+class: Workflow
+cwlVersion: v1.0
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+inputs:
+ sleeptime:
+ type: int
+ default: 5
+ fileblub:
+ type: File
+ default:
+ class: File
+ location: keep:d7514270f356df848477718d58308cc4+94/a
+ secondaryFiles:
+ - class: File
+ location: keep:d7514270f356df848477718d58308cc4+94/b
+outputs:
+ out:
+ type: string
+ outputSource: substep/out
+requirements:
+ SubworkflowFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ InlineJavascriptRequirement: {}
+ StepInputExpressionRequirement: {}
+steps:
+ substep:
+ in:
+ sleeptime: sleeptime
+ fileblub: fileblub
+ out: [out]
+ hints:
+ - class: arv:SeparateRunner
+ runnerProcessName: $("sleeptime "+inputs.sleeptime)
+ - class: DockerRequirement
+ dockerPull: arvados/jobs:2.2.2
+ run:
+ class: Workflow
+ id: mysub
+ inputs:
+ fileblub: File
+ sleeptime: int
+ outputs:
+ out:
+ type: string
+ outputSource: sleep1/out
+ steps:
+ sleep1:
+ in:
+ fileblub: fileblub
+ out: [out]
+ run:
+ class: CommandLineTool
+ id: subtool
+ inputs:
+ fileblub:
+ type: File
+ inputBinding: {position: 1}
+ outputs:
+ out:
+ type: string
+ outputBinding:
+ outputEval: 'out'
+ baseCommand: cat
commit 407c1ced46cb1ac3514e4357843f666e4e1ebeb6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Oct 12 10:00:13 2023 -0400
Merge branch '20933-arv-copy-cwl' refs #20933
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index 86b68f2854..46ecfb2394 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -71,10 +71,14 @@ Additionally, if you need to specify the storage classes where to save the copie
h3. How to copy a workflow
+Copying workflows requires @arvados-cwl-runner@ to be available in your @$PATH at .
+
We will use the uuid @jutro-7fd4e-mkmmq53m1ze6apx@ as an example workflow.
+Arv-copy will infer the source cluster is @jutro@ from the object uuid, and destination cluster is @pirca@ from @--project-uuid at .
+
<notextile>
-<pre><code>~$ <span class="userinput">arv-copy --src jutro --dst pirca --project-uuid pirca-j7d0g-ecak8knpefz8ere jutro-7fd4e-mkmmq53m1ze6apx</span>
+<pre><code>~$ <span class="userinput">arv-copy --project-uuid pirca-j7d0g-ecak8knpefz8ere jutro-7fd4e-mkmmq53m1ze6apx</span>
ae480c5099b81e17267b7445e35b4bc7+180: 23M / 23M 100.0%
2463fa9efeb75e099685528b3b9071e0+438: 156M / 156M 100.0%
jutro-4zz18-vvvqlops0a0kpdl: 94M / 94M 100.0%
@@ -91,6 +95,8 @@ h3. How to copy a project
We will use the uuid @jutro-j7d0g-xj19djofle3aryq@ as an example project.
+Arv-copy will infer the source cluster is @jutro@ from the source project uuid, and destination cluster is @pirca@ from @--project-uuid at .
+
<notextile>
<pre><code>~$ <span class="userinput">~$ arv-copy --project-uuid pirca-j7d0g-lr8sq3tx3ovn68k jutro-j7d0g-xj19djofle3aryq
2021-09-08 21:29:32 arvados.arv-copy[6377] INFO:
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 7968fb1e2b..fd3b7a5d16 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -123,6 +123,8 @@ def arg_parser(): # type: () -> argparse.ArgumentParser
exgroup.add_argument("--create-workflow", action="store_true", help="Register an Arvados workflow that can be run from Workbench")
exgroup.add_argument("--update-workflow", metavar="UUID", help="Update an existing Arvados workflow with the given UUID.")
+ exgroup.add_argument("--print-keep-deps", action="store_true", help="To assist copying, print a list of Keep collections that this workflow depends on.")
+
exgroup = parser.add_mutually_exclusive_group()
exgroup.add_argument("--wait", action="store_true", help="After submitting workflow runner, wait for completion.",
default=True, dest="wait")
@@ -324,7 +326,9 @@ def main(args=sys.argv[1:],
return 1
arvargs.work_api = want_api
- if (arvargs.create_workflow or arvargs.update_workflow) and not arvargs.job_order:
+ workflow_op = arvargs.create_workflow or arvargs.update_workflow or arvargs.print_keep_deps
+
+ if workflow_op and not arvargs.job_order:
job_order_object = ({}, "")
add_arv_hints()
@@ -416,9 +420,11 @@ def main(args=sys.argv[1:],
# unit tests.
stdout = None
+ executor.loadingContext.default_docker_image = arvargs.submit_runner_image or "arvados/jobs:"+__version__
+
if arvargs.workflow.startswith("arvwf:") or workflow_uuid_pattern.match(arvargs.workflow) or arvargs.workflow.startswith("keep:"):
executor.loadingContext.do_validate = False
- if arvargs.submit:
+ if arvargs.submit and not workflow_op:
executor.fast_submit = True
return cwltool.main.main(args=arvargs,
@@ -431,4 +437,4 @@ def main(args=sys.argv[1:],
custom_schema_callback=add_arv_hints,
loadingContext=executor.loadingContext,
runtimeContext=executor.toplevel_runtimeContext,
- input_required=not (arvargs.create_workflow or arvargs.update_workflow))
+ input_required=not workflow_op)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index a94fdac522..ea7c9f7a33 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -560,13 +560,19 @@ class RunnerContainer(Runner):
}
self.job_order[param] = {"$include": mnt}
+ container_image = arvados_jobs_image(self.arvrunner, self.jobs_image, runtimeContext)
+
+ workflow_runner_req, _ = self.embedded_tool.get_requirement("http://arvados.org/cwl#WorkflowRunnerResources")
+ if workflow_runner_req and workflow_runner_req.get("acrContainerImage"):
+ container_image = workflow_runner_req.get("acrContainerImage")
+
container_req = {
"name": self.name,
"output_path": "/var/spool/cwl",
"cwd": "/var/spool/cwl",
"priority": self.priority,
"state": "Committed",
- "container_image": arvados_jobs_image(self.arvrunner, self.jobs_image, runtimeContext),
+ "container_image": container_image,
"mounts": {
"/var/lib/cwl/cwl.input.json": {
"kind": "json",
diff --git a/sdk/cwl/arvados_cwl/arvtool.py b/sdk/cwl/arvados_cwl/arvtool.py
index b66e8ad3aa..86fecc0a1d 100644
--- a/sdk/cwl/arvados_cwl/arvtool.py
+++ b/sdk/cwl/arvados_cwl/arvtool.py
@@ -10,6 +10,7 @@ from ._version import __version__
from functools import partial
from schema_salad.sourceline import SourceLine
from cwltool.errors import WorkflowException
+from arvados.util import portable_data_hash_pattern
def validate_cluster_target(arvrunner, runtimeContext):
if (runtimeContext.submit_runner_cluster and
@@ -61,8 +62,12 @@ class ArvadosCommandTool(CommandLineTool):
(docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
if not docker_req:
- self.hints.append({"class": "DockerRequirement",
- "dockerPull": "arvados/jobs:"+__version__})
+ if portable_data_hash_pattern.match(loadingContext.default_docker_image):
+ self.hints.append({"class": "DockerRequirement",
+ "http://arvados.org/cwl#dockerCollectionPDH": loadingContext.default_docker_image})
+ else:
+ self.hints.append({"class": "DockerRequirement",
+ "dockerPull": loadingContext.default_docker_image})
self.arvrunner = arvrunner
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index cddcd15c54..3ad2c6419a 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -29,7 +29,7 @@ from cwltool.load_tool import fetch_document, resolve_and_validate_document
from cwltool.process import shortname, uniquename
from cwltool.workflow import Workflow, WorkflowException, WorkflowStep
from cwltool.utils import adjustFileObjs, adjustDirObjs, visit_class, normalizeFilesDirs
-from cwltool.context import LoadingContext
+from cwltool.context import LoadingContext, getdefault
from schema_salad.ref_resolver import file_uri, uri_file_path
@@ -42,6 +42,7 @@ from .pathmapper import ArvPathMapper, trim_listing
from .arvtool import ArvadosCommandTool, set_cluster_target
from ._version import __version__
from .util import common_prefix
+from .arvdocker import arv_docker_get_image
from .perf import Perf
@@ -178,14 +179,14 @@ def rel_ref(s, baseuri, urlexpander, merged_map, jobmapper):
def is_basetype(tp):
return _basetype_re.match(tp) is not None
-def update_refs(d, baseuri, urlexpander, merged_map, jobmapper, runtimeContext, prefix, replacePrefix):
+def update_refs(api, d, baseuri, urlexpander, merged_map, jobmapper, runtimeContext, prefix, replacePrefix):
if isinstance(d, MutableSequence):
for i, s in enumerate(d):
if prefix and isinstance(s, str):
if s.startswith(prefix):
d[i] = replacePrefix+s[len(prefix):]
else:
- update_refs(s, baseuri, urlexpander, merged_map, jobmapper, runtimeContext, prefix, replacePrefix)
+ update_refs(api, s, baseuri, urlexpander, merged_map, jobmapper, runtimeContext, prefix, replacePrefix)
elif isinstance(d, MutableMapping):
for field in ("id", "name"):
if isinstance(d.get(field), str) and d[field].startswith("_:"):
@@ -198,8 +199,8 @@ def update_refs(d, baseuri, urlexpander, merged_map, jobmapper, runtimeContext,
baseuri = urlexpander(d["name"], baseuri, scoped_id=True)
if d.get("class") == "DockerRequirement":
- dockerImageId = d.get("dockerImageId") or d.get("dockerPull")
- d["http://arvados.org/cwl#dockerCollectionPDH"] = runtimeContext.cached_docker_lookups.get(dockerImageId)
+ d["http://arvados.org/cwl#dockerCollectionPDH"] = arv_docker_get_image(api, d, False,
+ runtimeContext)
for field in d:
if field in ("location", "run", "name") and isinstance(d[field], str):
@@ -222,15 +223,21 @@ def update_refs(d, baseuri, urlexpander, merged_map, jobmapper, runtimeContext,
if isinstance(d["inputs"][inp], str) and not is_basetype(d["inputs"][inp]):
d["inputs"][inp] = rel_ref(d["inputs"][inp], baseuri, urlexpander, merged_map, jobmapper)
if isinstance(d["inputs"][inp], MutableMapping):
- update_refs(d["inputs"][inp], baseuri, urlexpander, merged_map, jobmapper, runtimeContext, prefix, replacePrefix)
+ update_refs(api, d["inputs"][inp], baseuri, urlexpander, merged_map, jobmapper, runtimeContext, prefix, replacePrefix)
continue
+ if field in ("requirements", "hints") and isinstance(d[field], MutableMapping):
+ dr = d[field].get("DockerRequirement")
+ if dr:
+ dr["http://arvados.org/cwl#dockerCollectionPDH"] = arv_docker_get_image(api, dr, False,
+ runtimeContext)
+
if field == "$schemas":
for n, s in enumerate(d["$schemas"]):
d["$schemas"][n] = rel_ref(d["$schemas"][n], baseuri, urlexpander, merged_map, jobmapper)
continue
- update_refs(d[field], baseuri, urlexpander, merged_map, jobmapper, runtimeContext, prefix, replacePrefix)
+ update_refs(api, d[field], baseuri, urlexpander, merged_map, jobmapper, runtimeContext, prefix, replacePrefix)
def fix_schemadef(req, baseuri, urlexpander, merged_map, jobmapper, pdh):
@@ -292,7 +299,8 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid,
# Find the longest common prefix among all the file names. We'll
# use this to recreate the directory structure in a keep
# collection with correct relative references.
- prefix = common_prefix(firstfile, all_files)
+ prefix = common_prefix(firstfile, all_files) if firstfile else ""
+
col = arvados.collection.Collection(api_client=arvRunner.api)
@@ -326,7 +334,7 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid,
# 2. find $import, $include, $schema, run, location
# 3. update field value
- update_refs(result, w, tool.doc_loader.expand_url, merged_map, jobmapper, runtimeContext, "", "")
+ update_refs(arvRunner.api, result, w, tool.doc_loader.expand_url, merged_map, jobmapper, runtimeContext, "", "")
# Write the updated file to the collection.
with col.open(w[len(prefix):], "wt") as f:
@@ -411,9 +419,10 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid,
wf_runner_resources = {"class": "http://arvados.org/cwl#WorkflowRunnerResources"}
hints.append(wf_runner_resources)
- wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner,
- submit_runner_image or "arvados/jobs:"+__version__,
- runtimeContext)
+ if "acrContainerImage" not in wf_runner_resources:
+ wf_runner_resources["acrContainerImage"] = arvados_jobs_image(arvRunner,
+ submit_runner_image or "arvados/jobs:"+__version__,
+ runtimeContext)
if submit_runner_ram:
wf_runner_resources["ramMin"] = submit_runner_ram
@@ -483,7 +492,7 @@ def upload_workflow(arvRunner, tool, job_order, project_uuid,
if r["class"] == "SchemaDefRequirement":
wrapper["requirements"][i] = fix_schemadef(r, main["id"], tool.doc_loader.expand_url, merged_map, jobmapper, col.portable_data_hash())
- update_refs(wrapper, main["id"], tool.doc_loader.expand_url, merged_map, jobmapper, runtimeContext, main["id"]+"#", "#main/")
+ update_refs(arvRunner.api, wrapper, main["id"], tool.doc_loader.expand_url, merged_map, jobmapper, runtimeContext, main["id"]+"#", "#main/")
doc = {"cwlVersion": "v1.2", "$graph": [wrapper]}
@@ -593,8 +602,18 @@ class ArvadosWorkflow(Workflow):
self.dynamic_resource_req = []
self.static_resource_req = []
self.wf_reffiles = []
- self.loadingContext = loadingContext
- super(ArvadosWorkflow, self).__init__(toolpath_object, loadingContext)
+ self.loadingContext = loadingContext.copy()
+
+ self.requirements = copy.deepcopy(getdefault(loadingContext.requirements, []))
+ tool_requirements = toolpath_object.get("requirements", [])
+ self.hints = copy.deepcopy(getdefault(loadingContext.hints, []))
+ tool_hints = toolpath_object.get("hints", [])
+
+ workflow_runner_req, _ = self.get_requirement("http://arvados.org/cwl#WorkflowRunnerResources")
+ if workflow_runner_req and workflow_runner_req.get("acrContainerImage"):
+ self.loadingContext.default_docker_image = workflow_runner_req.get("acrContainerImage")
+
+ super(ArvadosWorkflow, self).__init__(toolpath_object, self.loadingContext)
self.cluster_target_req, _ = self.get_requirement("http://arvados.org/cwl#ClusterTarget")
def job(self, joborder, output_callback, runtimeContext):
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 125527f783..dd64879b9f 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -7,6 +7,7 @@ from collections import namedtuple
class ArvLoadingContext(LoadingContext):
def __init__(self, kwargs=None):
+ self.default_docker_image = None
super(ArvLoadingContext, self).__init__(kwargs)
class ArvRuntimeContext(RuntimeContext):
@@ -43,6 +44,7 @@ class ArvRuntimeContext(RuntimeContext):
self.varying_url_params = ""
self.prefer_cached_downloads = False
self.cached_docker_lookups = {}
+ self.print_keep_deps = False
super(ArvRuntimeContext, self).__init__(kwargs)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index ce8aa42095..677e10d265 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -34,7 +34,7 @@ from arvados.errors import ApiError
import arvados_cwl.util
from .arvcontainer import RunnerContainer, cleanup_name_for_collection
-from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder, update_from_merged_map
+from .runner import Runner, upload_docker, upload_job_order, upload_workflow_deps, make_builder, update_from_merged_map, print_keep_deps
from .arvtool import ArvadosCommandTool, validate_cluster_target, ArvadosExpressionTool
from .arvworkflow import ArvadosWorkflow, upload_workflow, make_workflow_record
from .fsaccess import CollectionFsAccess, CollectionFetcher, collectionResolver, CollectionCache, pdh_size
@@ -649,6 +649,10 @@ The 'jobs' API is no longer supported.
runtimeContext.copy_deps = True
runtimeContext.match_local_docker = True
+ if runtimeContext.print_keep_deps:
+ runtimeContext.copy_deps = False
+ runtimeContext.match_local_docker = False
+
if runtimeContext.update_workflow and self.project_uuid is None:
# If we are updating a workflow, make sure anything that
# gets uploaded goes into the same parent project, unless
@@ -671,12 +675,10 @@ The 'jobs' API is no longer supported.
# are going to wait for the result, and always_submit_runner
# is false, then we don't submit a runner process.
- submitting = (runtimeContext.update_workflow or
- runtimeContext.create_workflow or
- (runtimeContext.submit and not
+ submitting = (runtimeContext.submit and not
(updated_tool.tool["class"] == "CommandLineTool" and
runtimeContext.wait and
- not runtimeContext.always_submit_runner)))
+ not runtimeContext.always_submit_runner))
loadingContext = self.loadingContext.copy()
loadingContext.do_validate = False
@@ -702,7 +704,7 @@ The 'jobs' API is no longer supported.
loadingContext.skip_resolve_all = True
workflow_wrapper = None
- if submitting and not self.fast_submit:
+ if (submitting and not self.fast_submit) or runtimeContext.update_workflow or runtimeContext.create_workflow or runtimeContext.print_keep_deps:
# upload workflow and get back the workflow wrapper
workflow_wrapper = upload_workflow(self, tool, job_order,
@@ -725,6 +727,11 @@ The 'jobs' API is no longer supported.
self.stdout.write(uuid + "\n")
return (None, "success")
+ if runtimeContext.print_keep_deps:
+ # Just find and print out all the collection dependencies and exit
+ print_keep_deps(self, runtimeContext, merged_map, tool)
+ return (None, "success")
+
# Did not register a workflow, we're going to submit
# it instead.
loadingContext.loader.idx.clear()
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 4432813f6a..f6aab4b93f 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -946,3 +946,42 @@ class Runner(Process):
self.arvrunner.output_callback({}, "permanentFail")
else:
self.arvrunner.output_callback(outputs, processStatus)
+
+
+def print_keep_deps_visitor(api, runtimeContext, references, doc_loader, tool):
+ def collect_locators(obj):
+ loc = obj.get("location", "")
+
+ g = arvados.util.keepuri_pattern.match(loc)
+ if g:
+ references.add(g[1])
+
+ if obj.get("class") == "http://arvados.org/cwl#WorkflowRunnerResources" and "acrContainerImage" in obj:
+ references.add(obj["acrContainerImage"])
+
+ if obj.get("class") == "DockerRequirement":
+ references.add(arvados_cwl.arvdocker.arv_docker_get_image(api, obj, False, runtimeContext))
+
+ sc_result = scandeps(tool["id"], tool,
+ set(),
+ set(("location", "id")),
+ None, urljoin=doc_loader.fetcher.urljoin,
+ nestdirs=False)
+
+ visit_class(sc_result, ("File", "Directory"), collect_locators)
+ visit_class(tool, ("DockerRequirement", "http://arvados.org/cwl#WorkflowRunnerResources"), collect_locators)
+
+
+def print_keep_deps(arvRunner, runtimeContext, merged_map, tool):
+ references = set()
+
+ tool.visit(partial(print_keep_deps_visitor, arvRunner.api, runtimeContext, references, tool.doc_loader))
+
+ for mm in merged_map:
+ for k, v in merged_map[mm].resolved.items():
+ g = arvados.util.keepuri_pattern.match(v)
+ if g:
+ references.add(g[1])
+
+ json.dump(sorted(references), arvRunner.stdout)
+ print(file=arvRunner.stdout)
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 92f0952af2..9e20efd2a3 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -43,7 +43,10 @@ setup(name='arvados-cwl-runner',
'networkx < 2.6',
'msgpack==1.0.3',
'importlib-metadata<5',
- 'setuptools>=40.3.0'
+ 'setuptools>=40.3.0',
+
+ # zipp 3.16 dropped support for Python 3.7
+ 'zipp<3.16.0; python_version<"3.8"'
],
data_files=[
('share/doc/arvados-cwl-runner', ['LICENSE-2.0.txt', 'README.rst']),
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index a2f404d7eb..8e3a8ab85e 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -85,7 +85,8 @@ class TestContainer(unittest.TestCase):
"construct_tool_object": runner.arv_make_tool,
"fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
"loader": Loader({}),
- "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
+ "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"}),
+ "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__
})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
@@ -1463,7 +1464,8 @@ class TestWorkflow(unittest.TestCase):
"make_fs_access": make_fs_access,
"loader": document_loader,
"metadata": {"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"},
- "construct_tool_object": runner.arv_make_tool})
+ "construct_tool_object": runner.arv_make_tool,
+ "default_docker_image": "arvados/jobs:"+arvados_cwl.__version__})
runtimeContext = arvados_cwl.context.ArvRuntimeContext(
{"work_api": "containers",
"basedir": "",
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 9dad245254..c8bf127951 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1180,7 +1180,7 @@ class TestSubmit(unittest.TestCase):
"out": [
{"id": "#main/step/out"}
],
- "run": "keep:7628e49da34b93de9f4baf08a6212817+247/secret_wf.cwl"
+ "run": "keep:991302581d01db470345a131480e623b+247/secret_wf.cwl"
}
]
}
@@ -1737,3 +1737,55 @@ class TestCreateWorkflow(unittest.TestCase):
self.assertEqual(stubs.capture_stdout.getvalue(),
stubs.expect_workflow_uuid + '\n')
self.assertEqual(exited, 0)
+
+ @stubs()
+ def test_create_map(self, stubs):
+ # test uploading a document that uses objects instead of arrays
+ # for certain fields like inputs and requirements.
+
+ project_uuid = 'zzzzz-j7d0g-zzzzzzzzzzzzzzz'
+ stubs.api.groups().get().execute.return_value = {"group_class": "project"}
+
+ exited = arvados_cwl.main(
+ ["--create-workflow", "--debug",
+ "--api=containers",
+ "--project-uuid", project_uuid,
+ "--disable-git",
+ "tests/wf/submit_wf_map.cwl", "tests/submit_test_job.json"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api)
+
+ stubs.api.pipeline_templates().create.refute_called()
+ stubs.api.container_requests().create.refute_called()
+
+ expect_workflow = StripYAMLComments(
+ open("tests/wf/expect_upload_wrapper_map.cwl").read().rstrip())
+
+ body = {
+ "workflow": {
+ "owner_uuid": project_uuid,
+ "name": "submit_wf_map.cwl",
+ "description": "",
+ "definition": expect_workflow,
+ }
+ }
+ stubs.api.workflows().create.assert_called_with(
+ body=JsonDiffMatcher(body))
+
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ stubs.expect_workflow_uuid + '\n')
+ self.assertEqual(exited, 0)
+
+
+class TestPrintKeepDeps(unittest.TestCase):
+ @stubs()
+ def test_print_keep_deps(self, stubs):
+ # test --print-keep-deps which is used by arv-copy
+
+ exited = arvados_cwl.main(
+ ["--print-keep-deps", "--debug",
+ "tests/wf/submit_wf_map.cwl"],
+ stubs.capture_stdout, sys.stderr, api_client=stubs.api)
+
+ self.assertEqual(stubs.capture_stdout.getvalue(),
+ '["5d373e7629203ce39e7c22af98a0f881+52", "999999999999999999999999999999d4+99"]' + '\n')
+ self.assertEqual(exited, 0)
diff --git a/sdk/cwl/tests/tool/submit_tool_map.cwl b/sdk/cwl/tests/tool/submit_tool_map.cwl
new file mode 100644
index 0000000000..7a833d471b
--- /dev/null
+++ b/sdk/cwl/tests/tool/submit_tool_map.cwl
@@ -0,0 +1,24 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Test case for arvados-cwl-runner
+#
+# Used to test whether scanning a tool file for dependencies (e.g. default
+# value blub.txt) and uploading to Keep works as intended.
+
+class: CommandLineTool
+cwlVersion: v1.0
+requirements:
+ DockerRequirement:
+ dockerPull: debian:buster-slim
+inputs:
+ x:
+ type: File
+ default:
+ class: File
+ location: blub.txt
+ inputBinding:
+ position: 1
+outputs: []
+baseCommand: cat
diff --git a/sdk/cwl/tests/wf/expect_upload_wrapper_map.cwl b/sdk/cwl/tests/wf/expect_upload_wrapper_map.cwl
new file mode 100644
index 0000000000..8f98f4718c
--- /dev/null
+++ b/sdk/cwl/tests/wf/expect_upload_wrapper_map.cwl
@@ -0,0 +1,88 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{
+ "$graph": [
+ {
+ "class": "Workflow",
+ "hints": [
+ {
+ "acrContainerImage": "999999999999999999999999999999d3+99",
+ "class": "http://arvados.org/cwl#WorkflowRunnerResources"
+ }
+ ],
+ "id": "#main",
+ "inputs": [
+ {
+ "default": {
+ "basename": "blorp.txt",
+ "class": "File",
+ "location": "keep:169f39d466a5438ac4a90e779bf750c7+53/blorp.txt",
+ "nameext": ".txt",
+ "nameroot": "blorp",
+ "size": 16
+ },
+ "id": "#main/x",
+ "type": "File"
+ },
+ {
+ "default": {
+ "basename": "99999999999999999999999999999998+99",
+ "class": "Directory",
+ "location": "keep:99999999999999999999999999999998+99"
+ },
+ "id": "#main/y",
+ "type": "Directory"
+ },
+ {
+ "default": {
+ "basename": "anonymous",
+ "class": "Directory",
+ "listing": [
+ {
+ "basename": "renamed.txt",
+ "class": "File",
+ "location": "keep:99999999999999999999999999999998+99/file1.txt",
+ "nameext": ".txt",
+ "nameroot": "renamed",
+ "size": 0
+ }
+ ]
+ },
+ "id": "#main/z",
+ "type": "Directory"
+ }
+ ],
+ "outputs": [],
+ "requirements": [
+ {
+ "class": "SubworkflowFeatureRequirement"
+ }
+ ],
+ "steps": [
+ {
+ "id": "#main/submit_wf_map.cwl",
+ "in": [
+ {
+ "id": "#main/step/x",
+ "source": "#main/x"
+ },
+ {
+ "id": "#main/step/y",
+ "source": "#main/y"
+ },
+ {
+ "id": "#main/step/z",
+ "source": "#main/z"
+ }
+ ],
+ "label": "submit_wf_map.cwl",
+ "out": [],
+ "run": "keep:2b94b65162db72023301a582e085646f+290/wf/submit_wf_map.cwl"
+ }
+ ]
+ }
+ ],
+ "cwlVersion": "v1.2"
+}
diff --git a/sdk/cwl/tests/wf/submit_wf_map.cwl b/sdk/cwl/tests/wf/submit_wf_map.cwl
new file mode 100644
index 0000000000..e8bb9cf77c
--- /dev/null
+++ b/sdk/cwl/tests/wf/submit_wf_map.cwl
@@ -0,0 +1,25 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Test case for arvados-cwl-runner
+#
+# Used to test whether scanning a workflow file for dependencies
+# (e.g. submit_tool.cwl) and uploading to Keep works as intended.
+
+class: Workflow
+cwlVersion: v1.2
+inputs:
+ x:
+ type: File
+ y:
+ type: Directory
+ z:
+ type: Directory
+outputs: []
+steps:
+ step1:
+ in:
+ x: x
+ out: []
+ run: ../tool/submit_tool_map.cwl
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 7326840896..7f5245db86 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -171,6 +171,9 @@ def main():
for d in listvalues(local_repo_dir):
shutil.rmtree(d, ignore_errors=True)
+ if not result:
+ exit(1)
+
# If no exception was thrown and the response does not have an
# error_token field, presume success
if result is None or 'error_token' in result or 'uuid' not in result:
@@ -324,21 +327,26 @@ def copy_workflow(wf_uuid, src, dst, args):
# copy collections and docker images
if args.recursive and wf["definition"]:
- wf_def = yaml.safe_load(wf["definition"])
- if wf_def is not None:
- locations = []
- docker_images = {}
- graph = wf_def.get('$graph', None)
- if graph is not None:
- workflow_collections(graph, locations, docker_images)
- else:
- workflow_collections(wf_def, locations, docker_images)
+ env = {"ARVADOS_API_HOST": urllib.parse.urlparse(src._rootDesc["rootUrl"]).netloc,
+ "ARVADOS_API_TOKEN": src.api_token,
+ "PATH": os.environ["PATH"]}
+ try:
+ result = subprocess.run(["arvados-cwl-runner", "--quiet", "--print-keep-deps", "arvwf:"+wf_uuid],
+ capture_output=True, env=env)
+ except FileNotFoundError:
+ no_arv_copy = True
+ else:
+ no_arv_copy = result.returncode == 2
+
+ if no_arv_copy:
+ raise Exception('Copying workflows requires arvados-cwl-runner 2.7.1 or later to be installed in PATH.')
+ elif result.returncode != 0:
+ raise Exception('There was an error getting Keep dependencies from workflow using arvados-cwl-runner --print-keep-deps')
- if locations:
- copy_collections(locations, src, dst, args)
+ locations = json.loads(result.stdout)
- for image in docker_images:
- copy_docker_image(image, docker_images[image], src, dst, args)
+ if locations:
+ copy_collections(locations, src, dst, args)
# copy the workflow itself
del wf['uuid']
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 1ee5f6355a..88adc8879b 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -24,9 +24,9 @@ CR_UNCOMMITTED = 'Uncommitted'
CR_COMMITTED = 'Committed'
CR_FINAL = 'Final'
-keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+\d+(\+\S+)*')
-signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+\d+(\+\S+)*\+A\S+(\+\S+)*')
-portable_data_hash_pattern = re.compile(r'[0-9a-f]{32}\+\d+')
+keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*')
+signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*\+A\S+(\+\S+)*')
+portable_data_hash_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+')
uuid_pattern = re.compile(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}')
collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
@@ -34,7 +34,9 @@ user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
-manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+\d+)(\+\S+)*)+( +\d+:\d+:\S+)+$)+', flags=re.MULTILINE)
+manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
+keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
+keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
def _deprecated(version=None, preferred=None):
"""Mark a callable as deprecated in the SDK
commit 660333eb025cc4d23d12e13063499ec31eea0972
Author: Tom Clegg <tom at curii.com>
Date: Tue Oct 24 15:32:49 2023 -0400
Merge branch '20984-instance-capacity'
fixes #20984
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/cloud/ec2/ec2.go b/lib/cloud/ec2/ec2.go
index 5e4df05f46..816df48d90 100644
--- a/lib/cloud/ec2/ec2.go
+++ b/lib/cloud/ec2/ec2.go
@@ -665,21 +665,36 @@ func (err rateLimitError) EarliestRetry() time.Time {
return err.earliestRetry
}
-var isCodeCapacity = map[string]bool{
+type capacityError struct {
+ error
+ isInstanceTypeSpecific bool
+}
+
+func (er *capacityError) IsCapacityError() bool {
+ return true
+}
+
+func (er *capacityError) IsInstanceTypeSpecific() bool {
+ return er.isInstanceTypeSpecific
+}
+
+var isCodeQuota = map[string]bool{
"InstanceLimitExceeded": true,
"InsufficientAddressCapacity": true,
"InsufficientFreeAddressesInSubnet": true,
- "InsufficientInstanceCapacity": true,
"InsufficientVolumeCapacity": true,
"MaxSpotInstanceCountExceeded": true,
"VcpuLimitExceeded": true,
}
-// isErrorCapacity returns whether the error is to be throttled based on its code.
+// isErrorQuota returns whether the error indicates we have reached
+// some usage quota/limit -- i.e., immediately retrying with an equal
+// or larger instance type will probably not work.
+//
// Returns false if error is nil.
-func isErrorCapacity(err error) bool {
+func isErrorQuota(err error) bool {
if aerr, ok := err.(awserr.Error); ok && aerr != nil {
- if _, ok := isCodeCapacity[aerr.Code()]; ok {
+ if _, ok := isCodeQuota[aerr.Code()]; ok {
return true
}
}
@@ -720,8 +735,10 @@ func wrapError(err error, throttleValue *atomic.Value) error {
}
throttleValue.Store(d)
return rateLimitError{error: err, earliestRetry: time.Now().Add(d)}
- } else if isErrorCapacity(err) {
+ } else if isErrorQuota(err) {
return &ec2QuotaError{err}
+ } else if aerr, ok := err.(awserr.Error); ok && aerr != nil && aerr.Code() == "InsufficientInstanceCapacity" {
+ return &capacityError{err, true}
} else if err != nil {
throttleValue.Store(time.Duration(0))
return err
diff --git a/lib/cloud/ec2/ec2_test.go b/lib/cloud/ec2/ec2_test.go
index 6fde4bbbca..a57fcebf76 100644
--- a/lib/cloud/ec2/ec2_test.go
+++ b/lib/cloud/ec2/ec2_test.go
@@ -508,8 +508,15 @@ func (*EC2InstanceSetSuite) TestWrapError(c *check.C) {
_, ok := wrapped.(cloud.RateLimitError)
c.Check(ok, check.Equals, true)
- quotaError := awserr.New("InsufficientInstanceCapacity", "", nil)
+ quotaError := awserr.New("InstanceLimitExceeded", "", nil)
wrapped = wrapError(quotaError, nil)
_, ok = wrapped.(cloud.QuotaError)
c.Check(ok, check.Equals, true)
+
+ capacityError := awserr.New("InsufficientInstanceCapacity", "", nil)
+ wrapped = wrapError(capacityError, nil)
+ caperr, ok := wrapped.(cloud.CapacityError)
+ c.Check(ok, check.Equals, true)
+ c.Check(caperr.IsCapacityError(), check.Equals, true)
+ c.Check(caperr.IsInstanceTypeSpecific(), check.Equals, true)
}
diff --git a/lib/cloud/interfaces.go b/lib/cloud/interfaces.go
index 7c532fda4a..a2aa9e1432 100644
--- a/lib/cloud/interfaces.go
+++ b/lib/cloud/interfaces.go
@@ -37,6 +37,20 @@ type QuotaError interface {
error
}
+// A CapacityError should be returned by an InstanceSet's Create
+// method when the cloud service indicates it has insufficient
+// capacity to create new instances -- i.e., we shouldn't retry right
+// away.
+type CapacityError interface {
+ // If true, wait before trying to create more instances.
+ IsCapacityError() bool
+ // If true, the condition is specific to the requested
+ // instance types. Wait before trying to create more
+ // instances of that same type.
+ IsInstanceTypeSpecific() bool
+ error
+}
+
type SharedResourceTags map[string]string
type InstanceSetID string
type InstanceTags map[string]string
diff --git a/lib/dispatchcloud/scheduler/interfaces.go b/lib/dispatchcloud/scheduler/interfaces.go
index 78f8c804e2..6e56bd8c40 100644
--- a/lib/dispatchcloud/scheduler/interfaces.go
+++ b/lib/dispatchcloud/scheduler/interfaces.go
@@ -34,6 +34,7 @@ type WorkerPool interface {
Running() map[string]time.Time
Unallocated() map[arvados.InstanceType]int
CountWorkers() map[worker.State]int
+ AtCapacity(arvados.InstanceType) bool
AtQuota() bool
Create(arvados.InstanceType) bool
Shutdown(arvados.InstanceType) bool
diff --git a/lib/dispatchcloud/scheduler/run_queue.go b/lib/dispatchcloud/scheduler/run_queue.go
index 6a717bf444..3505c3e064 100644
--- a/lib/dispatchcloud/scheduler/run_queue.go
+++ b/lib/dispatchcloud/scheduler/run_queue.go
@@ -149,6 +149,7 @@ func (sch *Scheduler) runQueue() {
}).Debug("runQueue")
dontstart := map[arvados.InstanceType]bool{}
+ var atcapacity = map[string]bool{} // ProviderTypes reported as AtCapacity during this runQueue() invocation
var overquota []container.QueueEnt // entries that are unmappable because of worker pool quota
var overmaxsuper []container.QueueEnt // unmappable because max supervisors (these are not included in overquota)
var containerAllocatedWorkerBootingCount int
@@ -189,6 +190,11 @@ tryrun:
overquota = sorted[i:]
break tryrun
}
+ if unalloc[it] < 1 && (atcapacity[it.ProviderType] || sch.pool.AtCapacity(it)) {
+ logger.Trace("not locking: AtCapacity and no unalloc workers")
+ atcapacity[it.ProviderType] = true
+ continue
+ }
if sch.pool.KillContainer(ctr.UUID, "about to lock") {
logger.Info("not locking: crunch-run process from previous attempt has not exited")
continue
@@ -211,6 +217,27 @@ tryrun:
logger.Trace("overquota")
overquota = sorted[i:]
break tryrun
+ } else if atcapacity[it.ProviderType] || sch.pool.AtCapacity(it) {
+ // Continue trying lower-priority
+ // containers in case they can run on
+ // different instance types that are
+ // available.
+ //
+ // The local "atcapacity" cache helps
+ // when the pool's flag resets after
+ // we look at container A but before
+ // we look at lower-priority container
+ // B. In that case we want to run
+ // container A on the next call to
+ // runQueue(), rather than run
+ // container B now.
+ //
+ // TODO: try running this container on
+ // a bigger (but not much more
+ // expensive) instance type.
+ logger.WithField("InstanceType", it.Name).Trace("at capacity")
+ atcapacity[it.ProviderType] = true
+ continue
} else if sch.pool.Create(it) {
// Success. (Note pool.Create works
// asynchronously and does its own
@@ -219,10 +246,7 @@ tryrun:
logger.Info("creating new instance")
} else {
// Failed despite not being at quota,
- // e.g., cloud ops throttled. TODO:
- // avoid getting starved here if
- // instances of a specific type always
- // fail.
+ // e.g., cloud ops throttled.
logger.Trace("pool declined to create new instance")
continue
}
diff --git a/lib/dispatchcloud/scheduler/run_queue_test.go b/lib/dispatchcloud/scheduler/run_queue_test.go
index a8df944b4c..da6955029a 100644
--- a/lib/dispatchcloud/scheduler/run_queue_test.go
+++ b/lib/dispatchcloud/scheduler/run_queue_test.go
@@ -32,10 +32,12 @@ var (
type stubPool struct {
notify <-chan struct{}
unalloc map[arvados.InstanceType]int // idle+booting+unknown
+ busy map[arvados.InstanceType]int
idle map[arvados.InstanceType]int
unknown map[arvados.InstanceType]int
running map[string]time.Time
quota int
+ capacity map[string]int
canCreate int
creates []arvados.InstanceType
starts []string
@@ -55,6 +57,20 @@ func (p *stubPool) AtQuota() bool {
}
return n >= p.quota
}
+func (p *stubPool) AtCapacity(it arvados.InstanceType) bool {
+ supply, ok := p.capacity[it.ProviderType]
+ if !ok {
+ return false
+ }
+ for _, existing := range []map[arvados.InstanceType]int{p.unalloc, p.busy} {
+ for eit, n := range existing {
+ if eit.ProviderType == it.ProviderType {
+ supply -= n
+ }
+ }
+ }
+ return supply < 1
+}
func (p *stubPool) Subscribe() <-chan struct{} { return p.notify }
func (p *stubPool) Unsubscribe(<-chan struct{}) {}
func (p *stubPool) Running() map[string]time.Time {
@@ -116,6 +132,7 @@ func (p *stubPool) StartContainer(it arvados.InstanceType, ctr arvados.Container
if p.idle[it] == 0 {
return false
}
+ p.busy[it]++
p.idle[it]--
p.unalloc[it]--
p.running[ctr.UUID] = time.Time{}
@@ -186,6 +203,7 @@ func (*SchedulerSuite) TestUseIdleWorkers(c *check.C) {
test.InstanceType(1): 1,
test.InstanceType(2): 2,
},
+ busy: map[arvados.InstanceType]int{},
running: map[string]time.Time{},
canCreate: 0,
}
@@ -236,6 +254,7 @@ func (*SchedulerSuite) TestShutdownAtQuota(c *check.C) {
idle: map[arvados.InstanceType]int{
test.InstanceType(2): 2,
},
+ busy: map[arvados.InstanceType]int{},
running: map[string]time.Time{},
creates: []arvados.InstanceType{},
starts: []string{},
@@ -272,6 +291,85 @@ func (*SchedulerSuite) TestShutdownAtQuota(c *check.C) {
}
}
+// If pool.AtCapacity(it) is true for one instance type, try running a
+// lower-priority container that uses a different node type. Don't
+// lock/unlock/start any container that requires the affected instance
+// type.
+func (*SchedulerSuite) TestInstanceCapacity(c *check.C) {
+ ctx := ctxlog.Context(context.Background(), ctxlog.TestLogger(c))
+
+ queue := test.Queue{
+ ChooseType: chooseType,
+ Containers: []arvados.Container{
+ {
+ UUID: test.ContainerUUID(1),
+ Priority: 1,
+ State: arvados.ContainerStateLocked,
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 1,
+ RAM: 1 << 30,
+ },
+ },
+ {
+ UUID: test.ContainerUUID(2),
+ Priority: 2,
+ State: arvados.ContainerStateQueued,
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 4,
+ RAM: 4 << 30,
+ },
+ },
+ {
+ UUID: test.ContainerUUID(3),
+ Priority: 3,
+ State: arvados.ContainerStateLocked,
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 4,
+ RAM: 4 << 30,
+ },
+ },
+ {
+ UUID: test.ContainerUUID(4),
+ Priority: 4,
+ State: arvados.ContainerStateLocked,
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ VCPUs: 4,
+ RAM: 4 << 30,
+ },
+ },
+ },
+ }
+ queue.Update()
+ pool := stubPool{
+ quota: 99,
+ capacity: map[string]int{test.InstanceType(4).ProviderType: 1},
+ unalloc: map[arvados.InstanceType]int{
+ test.InstanceType(4): 1,
+ },
+ idle: map[arvados.InstanceType]int{
+ test.InstanceType(4): 1,
+ },
+ busy: map[arvados.InstanceType]int{},
+ running: map[string]time.Time{},
+ creates: []arvados.InstanceType{},
+ starts: []string{},
+ canCreate: 99,
+ }
+ sch := New(ctx, arvados.NewClientFromEnv(), &queue, &pool, nil, time.Millisecond, time.Millisecond, 0, 0, 0)
+ sch.sync()
+ sch.runQueue()
+ sch.sync()
+
+ // Start container4, but then pool reports AtCapacity for
+ // type4, so we skip trying to create an instance for
+ // container3, skip locking container2, but do try to create a
+ // type1 instance for container1.
+ c.Check(pool.starts, check.DeepEquals, []string{test.ContainerUUID(4), test.ContainerUUID(1)})
+ c.Check(pool.shutdowns, check.Equals, 0)
+ c.Check(pool.creates, check.DeepEquals, []arvados.InstanceType{test.InstanceType(1)})
+ c.Check(queue.StateChanges(), check.HasLen, 0)
+}
+
// Don't unlock containers or shutdown unalloc (booting/idle) nodes
// just because some 503 errors caused us to reduce maxConcurrency
// below the current load level.
@@ -347,6 +445,10 @@ func (*SchedulerSuite) TestIdleIn503QuietPeriod(c *check.C) {
test.InstanceType(2): 1,
test.InstanceType(3): 1,
},
+ busy: map[arvados.InstanceType]int{
+ test.InstanceType(2): 1,
+ test.InstanceType(3): 1,
+ },
running: map[string]time.Time{
test.ContainerUUID(1): {},
test.ContainerUUID(3): {},
@@ -400,6 +502,9 @@ func (*SchedulerSuite) TestUnlockExcessSupervisors(c *check.C) {
idle: map[arvados.InstanceType]int{
test.InstanceType(2): 1,
},
+ busy: map[arvados.InstanceType]int{
+ test.InstanceType(2): 4,
+ },
running: map[string]time.Time{
test.ContainerUUID(1): {},
test.ContainerUUID(2): {},
@@ -461,6 +566,9 @@ func (*SchedulerSuite) TestExcessSupervisors(c *check.C) {
idle: map[arvados.InstanceType]int{
test.InstanceType(2): 1,
},
+ busy: map[arvados.InstanceType]int{
+ test.InstanceType(2): 2,
+ },
running: map[string]time.Time{
test.ContainerUUID(5): {},
test.ContainerUUID(6): {},
@@ -515,6 +623,7 @@ func (*SchedulerSuite) TestEqualPriorityContainers(c *check.C) {
idle: map[arvados.InstanceType]int{
test.InstanceType(3): 2,
},
+ busy: map[arvados.InstanceType]int{},
running: map[string]time.Time{},
creates: []arvados.InstanceType{},
starts: []string{},
@@ -553,6 +662,7 @@ func (*SchedulerSuite) TestStartWhileCreating(c *check.C) {
test.InstanceType(1): 1,
test.InstanceType(2): 1,
},
+ busy: map[arvados.InstanceType]int{},
running: map[string]time.Time{},
canCreate: 4,
}
@@ -646,6 +756,9 @@ func (*SchedulerSuite) TestKillNonexistentContainer(c *check.C) {
idle: map[arvados.InstanceType]int{
test.InstanceType(2): 0,
},
+ busy: map[arvados.InstanceType]int{
+ test.InstanceType(2): 1,
+ },
running: map[string]time.Time{
test.ContainerUUID(2): {},
},
@@ -742,6 +855,7 @@ func (*SchedulerSuite) TestContainersMetrics(c *check.C) {
pool = stubPool{
idle: map[arvados.InstanceType]int{test.InstanceType(1): 1},
unalloc: map[arvados.InstanceType]int{test.InstanceType(1): 1},
+ busy: map[arvados.InstanceType]int{},
running: map[string]time.Time{},
}
sch = New(ctx, arvados.NewClientFromEnv(), &queue, &pool, nil, time.Millisecond, time.Millisecond, 0, 0, 0)
@@ -815,6 +929,7 @@ func (*SchedulerSuite) TestSkipSupervisors(c *check.C) {
test.InstanceType(1): 4,
test.InstanceType(2): 4,
},
+ busy: map[arvados.InstanceType]int{},
running: map[string]time.Time{},
canCreate: 0,
}
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index 15b0dbcde5..13c369d0c6 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -82,6 +82,9 @@ const (
// instances have been shutdown.
quotaErrorTTL = time.Minute
+ // Time after a capacity error to try again
+ capacityErrorTTL = time.Minute
+
// Time between "X failed because rate limiting" messages
logRateLimitErrorInterval = time.Second * 10
)
@@ -181,6 +184,7 @@ type Pool struct {
atQuotaUntilFewerInstances int
atQuotaUntil time.Time
atQuotaErr cloud.QuotaError
+ atCapacityUntil map[string]time.Time
stop chan bool
mtx sync.RWMutex
setupOnce sync.Once
@@ -320,14 +324,11 @@ func (wp *Pool) Create(it arvados.InstanceType) bool {
// Boot probe is certain to fail.
return false
}
- wp.mtx.Lock()
- defer wp.mtx.Unlock()
- if time.Now().Before(wp.atQuotaUntil) ||
- wp.atQuotaUntilFewerInstances > 0 ||
- wp.instanceSet.throttleCreate.Error() != nil ||
- (wp.maxInstances > 0 && wp.maxInstances <= len(wp.workers)+len(wp.creating)) {
+ if wp.AtCapacity(it) || wp.AtQuota() || wp.instanceSet.throttleCreate.Error() != nil {
return false
}
+ wp.mtx.Lock()
+ defer wp.mtx.Unlock()
// The maxConcurrentInstanceCreateOps knob throttles the number of node create
// requests in flight. It was added to work around a limitation in Azure's
// managed disks, which support no more than 20 concurrent node creation
@@ -381,6 +382,19 @@ func (wp *Pool) Create(it arvados.InstanceType) bool {
logger.WithField("atQuotaUntilFewerInstances", n).Info("quota error -- waiting for next instance shutdown")
}
}
+ if err, ok := err.(cloud.CapacityError); ok && err.IsCapacityError() {
+ capKey := it.ProviderType
+ if !err.IsInstanceTypeSpecific() {
+ // set capacity flag for all
+ // instance types
+ capKey = ""
+ }
+ if wp.atCapacityUntil == nil {
+ wp.atCapacityUntil = map[string]time.Time{}
+ }
+ wp.atCapacityUntil[capKey] = time.Now().Add(capacityErrorTTL)
+ time.AfterFunc(capacityErrorTTL, wp.notify)
+ }
logger.WithError(err).Error("create failed")
wp.instanceSet.throttleCreate.CheckRateLimitError(err, wp.logger, "create instance", wp.notify)
return
@@ -393,6 +407,22 @@ func (wp *Pool) Create(it arvados.InstanceType) bool {
return true
}
+// AtCapacity returns true if Create() is currently expected to fail
+// for the given instance type.
+func (wp *Pool) AtCapacity(it arvados.InstanceType) bool {
+ wp.mtx.Lock()
+ defer wp.mtx.Unlock()
+ if t, ok := wp.atCapacityUntil[it.ProviderType]; ok && time.Now().Before(t) {
+ // at capacity for this instance type
+ return true
+ }
+ if t, ok := wp.atCapacityUntil[""]; ok && time.Now().Before(t) {
+ // at capacity for all instance types
+ return true
+ }
+ return false
+}
+
// AtQuota returns true if Create is not expected to work at the
// moment (e.g., cloud provider has reported quota errors, or we are
// already at our own configured quota).
commit c815f5ba6de486017822f5f1081e8f69555164b2
Author: Brett Smith <brett.smith at curii.com>
Date: Thu Sep 28 19:56:43 2023 -0400
Merge branch '20885-pdoc-style'
Closes #20885.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 6a94d6ff13..43a8d2f5d6 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -652,22 +652,14 @@ install_env() {
setup_virtualenv "$VENV3DIR"
. "$VENV3DIR/bin/activate"
- # Needed for run_test_server.py which is used by certain (non-Python) tests.
- # pdoc needed to generate the Python SDK documentation.
- (
- set -e
- "${VENV3DIR}/bin/pip3" install wheel
- "${VENV3DIR}/bin/pip3" install PyYAML
- "${VENV3DIR}/bin/pip3" install httplib2
- "${VENV3DIR}/bin/pip3" install future
- "${VENV3DIR}/bin/pip3" install google-api-python-client
- "${VENV3DIR}/bin/pip3" install ciso8601
- "${VENV3DIR}/bin/pip3" install pycurl
- "${VENV3DIR}/bin/pip3" install ws4py
- "${VENV3DIR}/bin/pip3" install pdoc
- cd "$WORKSPACE/sdk/python"
- python3 setup.py install
- ) || fatal "installing PyYAML and sdk/python failed"
+ # PyYAML is a test requirement used by run_test_server.py and needed for
+ # other, non-Python tests.
+ # pdoc is needed to build PySDK documentation.
+ # We run `setup.py build` first to generate _version.py.
+ env -C "$WORKSPACE/sdk/python" python3 setup.py build \
+ && python3 -m pip install "$WORKSPACE/sdk/python" \
+ && python3 -m pip install PyYAML pdoc \
+ || fatal "installing Python SDK and related dependencies failed"
}
retry() {
diff --git a/doc/README.textile b/doc/README.textile
index 9799ac564b..47951299cc 100644
--- a/doc/README.textile
+++ b/doc/README.textile
@@ -25,7 +25,7 @@ arvados/doc$ python3 -m venv .venv
arvados/doc$ .venv/bin/pip install pdoc
</pre>
-Then the generation process will need to be able to find @arvados/doc/.venv/bin/pdoc at . Either add that full directory to your @$PATH@, or make a @pdoc@ symlink in another directory that's already in your @$PATH at .
+Then you must activate the virtualenv (e.g., run @. .venv/bin/activate@) before you run the @bundle exec rake@ commands below.
h2. Generate HTML pages
diff --git a/doc/Rakefile b/doc/Rakefile
index c9322bb6ae..13e87167b6 100644
--- a/doc/Rakefile
+++ b/doc/Rakefile
@@ -54,11 +54,13 @@ file "sdk/python/arvados.html" do |t|
if ENV['NO_SDK'] || File.exist?("no-sdk")
next
end
+ # pysdk_pdoc.py is a wrapper around the pdoc CLI. `which pdoc` is an easy
+ # and good-enough test to check whether it's installed at all.
`which pdoc`
if $? == 0
raise unless system("python3", "setup.py", "build",
chdir: "../sdk/python", out: :err)
- raise unless system("pdoc", "-o", "sdk/python", "../sdk/python/build/lib/arvados/",
+ raise unless system("python3", "pysdk_pdoc.py",
out: :err)
else
puts "Warning: pdoc not found, Python documentation will not be generated".colorize(:light_red)
diff --git a/doc/pysdk_pdoc.py b/doc/pysdk_pdoc.py
new file mode 100755
index 0000000000..0fe584d08b
--- /dev/null
+++ b/doc/pysdk_pdoc.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+"""pysdk_pdoc.py - Run pdoc with extra rendering options
+
+This script is a wrapper around the standard `pdoc` tool that enables the
+`admonitions` and `smarty-pants` extras for nicer rendering. It checks that
+the version of `markdown2` included with `pdoc` supports those extras.
+
+If run without arguments, it uses arguments to build the Arvados Python SDK
+documentation.
+"""
+
+import collections
+import functools
+import os
+import sys
+
+import pdoc
+import pdoc.__main__
+import pdoc.markdown2
+import pdoc.render
+import pdoc.render_helpers
+
+DEFAULT_ARGLIST = [
+ '--output-directory=sdk/python',
+ '../sdk/python/build/lib/arvados/',
+]
+MD_EXTENSIONS = {
+ 'admonitions': None,
+ 'smarty-pants': None,
+}
+
+def main(arglist=None):
+ # Configure pdoc to use extras we want.
+ pdoc.render_helpers.markdown_extensions = collections.ChainMap(
+ pdoc.render_helpers.markdown_extensions,
+ MD_EXTENSIONS,
+ )
+
+ # Ensure markdown2 is new enough to support our desired extras.
+ if pdoc.markdown2.__version_info__ < (2, 4, 3):
+ print("error: need markdown2>=2.4.3 to render admonitions", file=sys.stderr)
+ return os.EX_SOFTWARE
+
+ pdoc.__main__.cli(arglist)
+ return os.EX_OK
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:] or DEFAULT_ARGLIST))
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index c62c1a74a9..ca9f17f866 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -198,45 +198,41 @@ def api_client(
Arguments:
- version: str
- : A string naming the version of the Arvados API to use.
+ * version: str --- A string naming the version of the Arvados API to use.
- discoveryServiceUrl: str
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`.
+ * discoveryServiceUrl: str --- The URL used to discover APIs passed
+ directly to `googleapiclient.discovery.build`.
- token: str
- : The authentication token to send with each API call.
+ * token: str --- The authentication token to send with each API call.
Keyword-only arguments:
- cache: bool
- : If true, loads the API discovery document from, or saves it to, a cache
- on disk (located at `~/.cache/arvados/discovery`).
+ * cache: bool --- If true, loads the API discovery document from, or
+ saves it to, a cache on disk (located at
+ `~/.cache/arvados/discovery`).
- http: httplib2.Http | None
- : The HTTP client object the API client object will use to make requests.
- If not provided, this function will build its own to use. Either way, the
- object will be patched as part of the build process.
+ * http: httplib2.Http | None --- The HTTP client object the API client
+ object will use to make requests. If not provided, this function will
+ build its own to use. Either way, the object will be patched as part
+ of the build process.
- insecure: bool
- : If true, ignore SSL certificate validation errors. Default `False`.
+ * insecure: bool --- If true, ignore SSL certificate validation
+ errors. Default `False`.
- num_retries: int
- : The number of times to retry each API request if it encounters a
- temporary failure. Default 10.
+ * num_retries: int --- The number of times to retry each API request if
+ it encounters a temporary failure. Default 10.
- request_id: str | None
- : Default `X-Request-Id` header value for outgoing requests that
- don't already provide one. If `None` or omitted, generate a random
- ID. When retrying failed requests, the same ID is used on all
- attempts.
+ * request_id: str | None --- Default `X-Request-Id` header value for
+ outgoing requests that don't already provide one. If `None` or
+ omitted, generate a random ID. When retrying failed requests, the same
+ ID is used on all attempts.
- timeout: int
- : A timeout value for HTTP requests in seconds. Default 300 (5 minutes).
+ * timeout: int --- A timeout value for HTTP requests in seconds. Default
+ 300 (5 minutes).
Additional keyword arguments will be passed directly to
`googleapiclient.discovery.build`.
+
"""
if http is None:
http = httplib2.Http(
@@ -313,22 +309,19 @@ def normalize_api_kwargs(
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- discoveryServiceUrl: str | None
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`. It is an error to pass both
- `discoveryServiceUrl` and `host`.
+ * discoveryServiceUrl: str | None --- The URL used to discover APIs
+ passed directly to `googleapiclient.discovery.build`. It is an error
+ to pass both `discoveryServiceUrl` and `host`.
- host: str | None
- : The hostname and optional port number of the Arvados API server. Used to
- build `discoveryServiceUrl`. It is an error to pass both
- `discoveryServiceUrl` and `host`.
+ * host: str | None --- The hostname and optional port number of the
+ Arvados API server. Used to build `discoveryServiceUrl`. It is an
+ error to pass both `discoveryServiceUrl` and `host`.
- token: str
- : The authentication token to send with each API call.
+ * token: str --- The authentication token to send with each API call.
Additional keyword arguments will be included in the return value.
"""
@@ -369,14 +362,15 @@ def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and
- optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls
- `arvados.config.settings` to get these parameters from user configuration.
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, calls
+ `arvados.config.settings` to get these parameters from user
+ configuration.
Additional keyword arguments will be included in the return value.
"""
@@ -419,19 +413,18 @@ def api(version=None, cache=True, host=None, token=None, insecure=False,
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- host: str | None
- : The hostname and optional port number of the Arvados API server.
+ * host: str | None --- The hostname and optional port number of the
+ Arvados API server.
- token: str | None
- : The authentication token to send with each API call.
+ * token: str | None --- The authentication token to send with each API
+ call.
- discoveryServiceUrl: str | None
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`.
+ * discoveryServiceUrl: str | None --- The URL used to discover APIs
+ passed directly to `googleapiclient.discovery.build`.
If `host`, `token`, and `discoveryServiceUrl` are all omitted, `host` and
`token` will be loaded from the user's configuration. Otherwise, you must
@@ -470,14 +463,15 @@ def api_from_config(version=None, apiconfig=None, **kwargs):
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and
- optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls
- `arvados.config.settings` to get these parameters from user configuration.
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, calls
+ `arvados.config.settings` to get these parameters from user
+ configuration.
Other arguments are passed directly to `api_client`. See that function's
docstring for more information about their meaning.
diff --git a/sdk/python/arvados/retry.py b/sdk/python/arvados/retry.py
index 2168146a4b..ea8a6f65af 100644
--- a/sdk/python/arvados/retry.py
+++ b/sdk/python/arvados/retry.py
@@ -49,34 +49,29 @@ class RetryLoop(object):
Arguments:
- num_retries: int
- : The maximum number of times to retry the loop if it
- doesn't succeed. This means the loop body could run at most
+ * num_retries: int --- The maximum number of times to retry the loop if
+ it doesn't succeed. This means the loop body could run at most
`num_retries + 1` times.
- success_check: Callable
- : This is a function that will be called each
- time the loop saves a result. The function should return
- `True` if the result indicates the code succeeded, `False` if it
- represents a permanent failure, and `None` if it represents a
- temporary failure. If no function is provided, the loop will
- end after any result is saved.
-
- backoff_start: float
- : The number of seconds that must pass before the loop's second
- iteration. Default 0, which disables all waiting.
-
- backoff_growth: float
- : The wait time multiplier after each iteration.
- Default 2 (i.e., double the wait time each time).
-
- save_results: int
- : Specify a number to store that many saved results from the loop.
- These are available through the `results` attribute, oldest first.
- Default 1.
-
- max_wait: float
- : Maximum number of seconds to wait between retries. Default 60.
+ * success_check: Callable --- This is a function that will be called
+ each time the loop saves a result. The function should return `True`
+ if the result indicates the code succeeded, `False` if it represents a
+ permanent failure, and `None` if it represents a temporary failure.
+ If no function is provided, the loop will end after any result is
+ saved.
+
+ * backoff_start: float --- The number of seconds that must pass before
+ the loop's second iteration. Default 0, which disables all waiting.
+
+ * backoff_growth: float --- The wait time multiplier after each
+ iteration. Default 2 (i.e., double the wait time each time).
+
+ * save_results: int --- Specify a number to store that many saved
+ results from the loop. These are available through the `results`
+ attribute, oldest first. Default 1.
+
+ * max_wait: float --- Maximum number of seconds to wait between
+ retries. Default 60.
"""
def __init__(self, num_retries, success_check=lambda r: True,
backoff_start=0, backoff_growth=2, save_results=1,
@@ -138,8 +133,8 @@ class RetryLoop(object):
Arguments:
- result: Any
- : The result from this loop attempt to check and save.
+ * result: Any --- The result from this loop attempt to check and
+ save.
"""
if not self.running():
raise arvados.errors.AssertionError(
@@ -207,8 +202,7 @@ def check_http_response_success(status_code):
Arguments:
- status_code: int
- : A numeric HTTP response code
+ * status_code: int --- A numeric HTTP response code
"""
if status_code in _HTTP_SUCCESSES:
return True
@@ -229,8 +223,8 @@ def retry_method(orig_func):
Arguments:
- orig_func: Callable
- : A class or instance method that accepts a `num_retries` keyword argument
+ * orig_func: Callable --- A class or instance method that accepts a
+ `num_retries` keyword argument
"""
@functools.wraps(orig_func)
def num_retries_setter(self, *args, **kwargs):
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 16f2999ca8..1ee5f6355a 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -74,16 +74,17 @@ def _deprecated(version=None, preferred=None):
func_doc = re.sub(r'\n\s*$', '', func.__doc__ or '')
match = re.search(r'\n([ \t]+)\S', func_doc)
indent = '' if match is None else match.group(1)
+ warning_doc = f'\n\n{indent}.. WARNING:: Deprecated\n{indent} {warning_msg}'
# Make the deprecation notice the second "paragraph" of the
# docstring if possible. Otherwise append it.
docstring, count = re.subn(
rf'\n[ \t]*\n{indent}',
- f'\n\n{indent}DEPRECATED: {warning_msg}\n\n{indent}',
+ f'{warning_doc}\n\n{indent}',
func_doc,
count=1,
)
if not count:
- docstring = f'{func_doc}\n\n{indent}DEPRECATED: {warning_msg}'.lstrip()
+ docstring = f'{func_doc.lstrip()}{warning_doc}'
deprecated_wrapper.__doc__ = docstring
return deprecated_wrapper
return deprecated_decorator
diff --git a/sdk/python/discovery2pydoc.py b/sdk/python/discovery2pydoc.py
index 151325228e..6ca3aafeb6 100755
--- a/sdk/python/discovery2pydoc.py
+++ b/sdk/python/discovery2pydoc.py
@@ -49,8 +49,8 @@ _ALIASED_METHODS = frozenset([
])
_DEPRECATED_NOTICE = '''
-!!! deprecated
- This resource is deprecated in the Arvados API.
+.. WARNING:: Deprecated
+ This resource is deprecated in the Arvados API.
'''
_DEPRECATED_RESOURCES = frozenset([
'Humans',
commit 13c0aa8c661827a82b9b666b47592ceb214783da
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Oct 20 11:57:45 2023 -0400
20885: Update argument docstrings after pdoc migration
Update arvados.api_resources to use our new function argument format.
Refs #20885.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/discovery2pydoc.py b/sdk/python/discovery2pydoc.py
index 9f7f87d988..151325228e 100755
--- a/sdk/python/discovery2pydoc.py
+++ b/sdk/python/discovery2pydoc.py
@@ -182,13 +182,17 @@ class Parameter(inspect.Parameter):
if default_value is None:
default_doc = ''
else:
- default_doc = f" Default {default_value!r}."
- # If there is no description, use a zero-width space to help Markdown
- # parsers retain the definition list structure.
- description = self._spec['description'] or '\u200b'
+ default_doc = f"Default {default_value!r}."
+ description = self._spec['description']
+ doc_parts = [f'{self.api_name}: {self.annotation}']
+ if description or default_doc:
+ doc_parts.append('---')
+ if description:
+ doc_parts.append(description)
+ if default_doc:
+ doc_parts.append(default_doc)
return f'''
-{self.api_name}: {self.annotation}
-: {description}{default_doc}
+* {' '.join(doc_parts)}
'''
commit 9b64ec17351e3c034b4f96470b12a1c00ff6813f
Author: Brett Smith <brett.smith at curii.com>
Date: Fri Sep 29 11:28:50 2023 -0400
Merge branch '19818-api-pydoc'
Closes #19818.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 8108934aae..7968fb1e2b 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -32,7 +32,6 @@ import arvados.logging
from arvados.keep import KeepClient
from arvados.errors import ApiError
import arvados.commands._util as arv_cmd
-from arvados.api import OrderedJsonModel
from .perf import Perf
from ._version import __version__
@@ -338,7 +337,6 @@ def main(args=sys.argv[1:],
if api_client is None:
api_client = arvados.safeapi.ThreadSafeApiCache(
api_params={
- 'model': OrderedJsonModel(),
'num_retries': arvargs.retries,
'timeout': arvargs.http_timeout,
},
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index c51be82b20..c62c1a74a9 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -43,13 +43,12 @@ _logger = logging.getLogger('arvados.api')
_googleapiclient_log_lock = threading.Lock()
MAX_IDLE_CONNECTION_DURATION = 30
-
-# These constants supported our own retry logic that we've since removed in
-# favor of using googleapiclient's num_retries. They're kept here purely for
-# API compatibility, but set to 0 to indicate no retries happen.
-RETRY_DELAY_INITIAL = 0
-RETRY_DELAY_BACKOFF = 0
-RETRY_COUNT = 0
+"""
+Number of seconds that API client HTTP connections should be allowed to idle
+in keepalive state before they are forced closed. Client code can adjust this
+constant, and it will be used for all Arvados API clients constructed after
+that point.
+"""
# An unused HTTP 5xx status code to request a retry internally.
# See _intercept_http_request. This should not be user-visible.
@@ -58,26 +57,6 @@ _RETRY_4XX_STATUS = 545
if sys.version_info >= (3,):
httplib2.SSLHandshakeError = None
-class OrderedJsonModel(apiclient.model.JsonModel):
- """Model class for JSON that preserves the contents' order.
-
- API clients that care about preserving the order of fields in API
- server responses can use this model to do so, like this:
-
- from arvados.api import OrderedJsonModel
- client = arvados.api('v1', ..., model=OrderedJsonModel())
- """
-
- def deserialize(self, content):
- # This is a very slightly modified version of the parent class'
- # implementation. Copyright (c) 2010 Google.
- content = content.decode('utf-8')
- body = json.loads(content, object_pairs_hook=collections.OrderedDict)
- if self._data_wrapper and isinstance(body, dict) and 'data' in body:
- body = body['data']
- return body
-
-
_orig_retry_request = apiclient.http._retry_request
def _retry_request(http, num_retries, *args, **kwargs):
try:
@@ -174,6 +153,18 @@ def _new_http_error(cls, *args, **kwargs):
apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
def http_cache(data_type):
+ """Set up an HTTP file cache
+
+ This function constructs and returns an `arvados.cache.SafeHTTPCache`
+ backed by the filesystem under `~/.cache/arvados/`, or `None` if the
+ directory cannot be set up. The return value can be passed to
+ `httplib2.Http` as the `cache` argument.
+
+ Arguments:
+
+ * data_type: str --- The name of the subdirectory under `~/.cache/arvados`
+ where data is cached.
+ """
try:
homedir = pathlib.Path.home()
except RuntimeError:
@@ -492,3 +483,49 @@ def api_from_config(version=None, apiconfig=None, **kwargs):
docstring for more information about their meaning.
"""
return api(**api_kwargs_from_config(version, apiconfig, **kwargs))
+
+class OrderedJsonModel(apiclient.model.JsonModel):
+ """Model class for JSON that preserves the contents' order
+
+ .. WARNING:: Deprecated
+ This model is redundant now that Python dictionaries preserve insertion
+ ordering. Code that passes this model to API constructors can remove it.
+
+ In Python versions before 3.6, API clients that cared about preserving the
+ order of fields in API server responses could use this model to do so.
+ Typical usage looked like:
+
+ from arvados.api import OrderedJsonModel
+ client = arvados.api('v1', ..., model=OrderedJsonModel())
+ """
+ @util._deprecated(preferred="the default model and rely on Python's built-in dictionary ordering")
+ def __init__(self, data_wrapper=False):
+ return super().__init__(data_wrapper)
+
+
+RETRY_DELAY_INITIAL = 0
+"""
+.. WARNING:: Deprecated
+ This constant was used by retry code in previous versions of the Arvados SDK.
+ Changing the value has no effect anymore.
+ Prefer passing `num_retries` to an API client constructor instead.
+ Refer to the constructor docstrings for details.
+"""
+
+RETRY_DELAY_BACKOFF = 0
+"""
+.. WARNING:: Deprecated
+ This constant was used by retry code in previous versions of the Arvados SDK.
+ Changing the value has no effect anymore.
+ Prefer passing `num_retries` to an API client constructor instead.
+ Refer to the constructor docstrings for details.
+"""
+
+RETRY_COUNT = 0
+"""
+.. WARNING:: Deprecated
+ This constant was used by retry code in previous versions of the Arvados SDK.
+ Changing the value has no effect anymore.
+ Prefer passing `num_retries` to an API client constructor instead.
+ Refer to the constructor docstrings for details.
+"""
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index ef3936e267..7326840896 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -49,7 +49,6 @@ import arvados.commands.keepdocker
import arvados.http_to_keep
import ruamel.yaml as yaml
-from arvados.api import OrderedJsonModel
from arvados._version import __version__
COMMIT_HASH_RE = re.compile(r'^[0-9a-f]{1,40}$')
@@ -208,7 +207,7 @@ def set_src_owner_uuid(resource, uuid, args):
def api_for_instance(instance_name, num_retries):
if not instance_name:
# Use environment
- return arvados.api('v1', model=OrderedJsonModel())
+ return arvados.api('v1')
if '/' in instance_name:
config_file = instance_name
@@ -232,7 +231,6 @@ def api_for_instance(instance_name, num_retries):
host=cfg['ARVADOS_API_HOST'],
token=cfg['ARVADOS_API_TOKEN'],
insecure=api_is_insecure,
- model=OrderedJsonModel(),
num_retries=num_retries,
)
else:
diff --git a/tools/crunchstat-summary/crunchstat_summary/summarizer.py b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
index a876257abc..2bd329719b 100644
--- a/tools/crunchstat-summary/crunchstat_summary/summarizer.py
+++ b/tools/crunchstat-summary/crunchstat_summary/summarizer.py
@@ -15,7 +15,6 @@ import sys
import threading
import _strptime
-from arvados.api import OrderedJsonModel
from crunchstat_summary import logger
# Recommend memory constraints that are this multiple of an integral
@@ -518,7 +517,7 @@ def NewSummarizer(process_or_uuid, **kwargs):
else:
uuid = process_or_uuid
process = None
- arv = arvados.api('v1', model=OrderedJsonModel())
+ arv = arvados.api('v1')
if '-dz642-' in uuid:
if process is None:
@@ -639,7 +638,7 @@ class MultiSummarizer(object):
class JobTreeSummarizer(MultiSummarizer):
"""Summarizes a job and all children listed in its components field."""
def __init__(self, job, label=None, **kwargs):
- arv = arvados.api('v1', model=OrderedJsonModel())
+ arv = arvados.api('v1')
label = label or job.get('name', job['uuid'])
children = collections.OrderedDict()
children[job['uuid']] = JobSummarizer(job, label=label, **kwargs)
@@ -683,7 +682,7 @@ class PipelineSummarizer(MultiSummarizer):
class ContainerRequestTreeSummarizer(MultiSummarizer):
def __init__(self, root, skip_child_jobs=False, **kwargs):
- arv = arvados.api('v1', model=OrderedJsonModel())
+ arv = arvados.api('v1')
label = kwargs.pop('label', None) or root.get('name') or root['uuid']
root['name'] = label
commit de2020ca192fb80da829230008e0bddf93eb43eb
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Oct 11 09:20:29 2023 -0400
Merge branch '20937-arv-copy-http' refs #20937
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index 15c9623224..86b68f2854 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -15,7 +15,7 @@ This tutorial describes how to copy Arvados objects from one cluster to another
h2. arv-copy
- at arv-copy@ allows users to copy collections, workflow definitions and projects from one cluster to another.
+ at arv-copy@ allows users to copy collections, workflow definitions and projects from one cluster to another. You can also use @arv-copy@ to import resources from HTTP URLs into Keep.
For projects, @arv-copy@ will copy all the collections workflow definitions owned by the project, and recursively copy subprojects.
@@ -92,7 +92,7 @@ h3. How to copy a project
We will use the uuid @jutro-j7d0g-xj19djofle3aryq@ as an example project.
<notextile>
-<pre><code>~$ <span class="userinput">peteramstutz at shell:~$ arv-copy --project-uuid pirca-j7d0g-lr8sq3tx3ovn68k jutro-j7d0g-xj19djofle3aryq
+<pre><code>~$ <span class="userinput">~$ arv-copy --project-uuid pirca-j7d0g-lr8sq3tx3ovn68k jutro-j7d0g-xj19djofle3aryq
2021-09-08 21:29:32 arvados.arv-copy[6377] INFO:
2021-09-08 21:29:32 arvados.arv-copy[6377] INFO: Success: created copy with uuid pirca-j7d0g-ig9gvu5piznducp
</code></pre>
@@ -101,3 +101,23 @@ We will use the uuid @jutro-j7d0g-xj19djofle3aryq@ as an example project.
The name and description of the original project will be used for the destination copy. If a project already exists with the same name, collections and workflow definitions will be copied into the project with the same name.
If you would like to copy the project but not its subproject, you can use the @--no-recursive@ flag.
+
+h3. Importing HTTP resources to Keep
+
+You can also use @arv-copy@ to copy the contents of a HTTP URL into Keep. When you do this, Arvados keeps track of the original URL the resource came from. This allows you to refer to the resource by its original URL in Workflow inputs, but actually read from the local copy in Keep.
+
+<notextile>
+<pre><code>~$ <span class="userinput">~$ arv-copy --project-uuid tordo-j7d0g-lr8sq3tx3ovn68k https://example.com/index.html
+tordo-4zz18-dhpb6y9km2byb94
+2023-10-06 10:15:36 arvados.arv-copy[374147] INFO: Success: created copy with uuid tordo-4zz18-dhpb6y9km2byb94
+</code></pre>
+</notextile>
+
+In addition, when importing from HTTP URLs, you may provide a different cluster than the destination in @--src at . This tells @arv-copy@ to search the other cluster for a collection associated with that URL, and if found, copy the collection from that cluster instead of downloading from the original URL.
+
+The following @arv-copy@ command line options affect the behavior of HTTP import.
+
+table(table table-bordered table-condensed).
+|_. Option |_. Description |
+|==--varying-url-params== VARYING_URL_PARAMS|A comma separated list of URL query parameters that should be ignored when storing HTTP URLs in Keep.|
+|==--prefer-cached-downloads==|If a HTTP URL is found in Keep, skip upstream URL freshness check (will not notice if the upstream has changed, but also not error if upstream is unavailable).|
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 539188fddd..448facf776 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -109,9 +109,10 @@ class ArvPathMapper(PathMapper):
# passthrough, we'll download it later.
self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
else:
- keepref = "keep:%s/%s" % http_to_keep(self.arvrunner.api, self.arvrunner.project_uuid, src,
+ results = http_to_keep(self.arvrunner.api, self.arvrunner.project_uuid, src,
varying_url_params=self.arvrunner.toplevel_runtimeContext.varying_url_params,
prefer_cached_downloads=self.arvrunner.toplevel_runtimeContext.prefer_cached_downloads)
+ keepref = "keep:%s/%s" % (results[0], results[1])
logger.info("%s is %s", src, keepref)
self._pathmap[src] = MapperEnt(keepref, keepref, srcobj["class"], True)
except Exception as e:
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 10fe9d7024..ef3936e267 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -36,6 +36,9 @@ import logging
import tempfile
import urllib.parse
import io
+import json
+import queue
+import threading
import arvados
import arvados.config
@@ -43,6 +46,7 @@ import arvados.keep
import arvados.util
import arvados.commands._util as arv_cmd
import arvados.commands.keepdocker
+import arvados.http_to_keep
import ruamel.yaml as yaml
from arvados.api import OrderedJsonModel
@@ -106,6 +110,11 @@ def main():
copy_opts.add_argument(
'--storage-classes', dest='storage_classes',
help='Comma separated list of storage classes to be used when saving data to the destinaton Arvados instance.')
+ copy_opts.add_argument("--varying-url-params", type=str, default="",
+ help="A comma separated list of URL query parameters that should be ignored when storing HTTP URLs in Keep.")
+
+ copy_opts.add_argument("--prefer-cached-downloads", action="store_true", default=False,
+ help="If a HTTP URL is found in Keep, skip upstream URL freshness check (will not notice if the upstream has changed, but also not error if upstream is unavailable).")
copy_opts.add_argument(
'object_uuid',
@@ -126,7 +135,7 @@ def main():
else:
logger.setLevel(logging.INFO)
- if not args.source_arvados:
+ if not args.source_arvados and arvados.util.uuid_pattern.match(args.object_uuid):
args.source_arvados = args.object_uuid[:5]
# Create API clients for the source and destination instances
@@ -138,19 +147,26 @@ def main():
# Identify the kind of object we have been given, and begin copying.
t = uuid_type(src_arv, args.object_uuid)
- if t == 'Collection':
- set_src_owner_uuid(src_arv.collections(), args.object_uuid, args)
- result = copy_collection(args.object_uuid,
- src_arv, dst_arv,
- args)
- elif t == 'Workflow':
- set_src_owner_uuid(src_arv.workflows(), args.object_uuid, args)
- result = copy_workflow(args.object_uuid, src_arv, dst_arv, args)
- elif t == 'Group':
- set_src_owner_uuid(src_arv.groups(), args.object_uuid, args)
- result = copy_project(args.object_uuid, src_arv, dst_arv, args.project_uuid, args)
- else:
- abort("cannot copy object {} of type {}".format(args.object_uuid, t))
+
+ try:
+ if t == 'Collection':
+ set_src_owner_uuid(src_arv.collections(), args.object_uuid, args)
+ result = copy_collection(args.object_uuid,
+ src_arv, dst_arv,
+ args)
+ elif t == 'Workflow':
+ set_src_owner_uuid(src_arv.workflows(), args.object_uuid, args)
+ result = copy_workflow(args.object_uuid, src_arv, dst_arv, args)
+ elif t == 'Group':
+ set_src_owner_uuid(src_arv.groups(), args.object_uuid, args)
+ result = copy_project(args.object_uuid, src_arv, dst_arv, args.project_uuid, args)
+ elif t == 'httpURL':
+ result = copy_from_http(args.object_uuid, src_arv, dst_arv, args)
+ else:
+ abort("cannot copy object {} of type {}".format(args.object_uuid, t))
+ except Exception as e:
+ logger.error("%s", e, exc_info=args.verbose)
+ exit(1)
# Clean up any outstanding temp git repositories.
for d in listvalues(local_repo_dir):
@@ -158,8 +174,9 @@ def main():
# If no exception was thrown and the response does not have an
# error_token field, presume success
- if 'error_token' in result or 'uuid' not in result:
- logger.error("API server returned an error result: {}".format(result))
+ if result is None or 'error_token' in result or 'uuid' not in result:
+ if result:
+ logger.error("API server returned an error result: {}".format(result))
exit(1)
print(result['uuid'])
@@ -567,6 +584,125 @@ def copy_collection(obj_uuid, src, dst, args):
else:
progress_writer = None
+ # go through the words
+ # put each block loc into 'get' queue
+ # 'get' threads get block and put it into 'put' queue
+ # 'put' threads put block and then update dst_locators
+ #
+ # after going through the whole manifest we go back through it
+ # again and build dst_manifest
+
+ lock = threading.Lock()
+
+ # the get queue should be unbounded because we'll add all the
+ # block hashes we want to get, but these are small
+ get_queue = queue.Queue()
+
+ threadcount = 4
+
+ # the put queue contains full data blocks
+ # and if 'get' is faster than 'put' we could end up consuming
+ # a great deal of RAM if it isn't bounded.
+ put_queue = queue.Queue(threadcount)
+ transfer_error = []
+
+ def get_thread():
+ while True:
+ word = get_queue.get()
+ if word is None:
+ put_queue.put(None)
+ get_queue.task_done()
+ return
+
+ blockhash = arvados.KeepLocator(word).md5sum
+ with lock:
+ if blockhash in dst_locators:
+ # Already uploaded
+ get_queue.task_done()
+ continue
+
+ try:
+ logger.debug("Getting block %s", word)
+ data = src_keep.get(word)
+ put_queue.put((word, data))
+ except e:
+ logger.error("Error getting block %s: %s", word, e)
+ transfer_error.append(e)
+ try:
+ # Drain the 'get' queue so we end early
+ while True:
+ get_queue.get(False)
+ get_queue.task_done()
+ except queue.Empty:
+ pass
+ finally:
+ get_queue.task_done()
+
+ def put_thread():
+ nonlocal bytes_written
+ while True:
+ item = put_queue.get()
+ if item is None:
+ put_queue.task_done()
+ return
+
+ word, data = item
+ loc = arvados.KeepLocator(word)
+ blockhash = loc.md5sum
+ with lock:
+ if blockhash in dst_locators:
+ # Already uploaded
+ put_queue.task_done()
+ continue
+
+ try:
+ logger.debug("Putting block %s (%s bytes)", blockhash, loc.size)
+ dst_locator = dst_keep.put(data, classes=(args.storage_classes or []))
+ with lock:
+ dst_locators[blockhash] = dst_locator
+ bytes_written += loc.size
+ if progress_writer:
+ progress_writer.report(obj_uuid, bytes_written, bytes_expected)
+ except e:
+ logger.error("Error putting block %s (%s bytes): %s", blockhash, loc.size, e)
+ try:
+ # Drain the 'get' queue so we end early
+ while True:
+ get_queue.get(False)
+ get_queue.task_done()
+ except queue.Empty:
+ pass
+ transfer_error.append(e)
+ finally:
+ put_queue.task_done()
+
+ for line in manifest.splitlines():
+ words = line.split()
+ for word in words[1:]:
+ try:
+ loc = arvados.KeepLocator(word)
+ except ValueError:
+ # If 'word' can't be parsed as a locator,
+ # presume it's a filename.
+ continue
+
+ get_queue.put(word)
+
+ for i in range(0, threadcount):
+ get_queue.put(None)
+
+ for i in range(0, threadcount):
+ threading.Thread(target=get_thread, daemon=True).start()
+
+ for i in range(0, threadcount):
+ threading.Thread(target=put_thread, daemon=True).start()
+
+ get_queue.join()
+ put_queue.join()
+
+ if len(transfer_error) > 0:
+ return {"error_token": "Failed to transfer blocks"}
+
for line in manifest.splitlines():
words = line.split()
dst_manifest.write(words[0])
@@ -580,16 +716,6 @@ def copy_collection(obj_uuid, src, dst, args):
dst_manifest.write(word)
continue
blockhash = loc.md5sum
- # copy this block if we haven't seen it before
- # (otherwise, just reuse the existing dst_locator)
- if blockhash not in dst_locators:
- logger.debug("Copying block %s (%s bytes)", blockhash, loc.size)
- if progress_writer:
- progress_writer.report(obj_uuid, bytes_written, bytes_expected)
- data = src_keep.get(word)
- dst_locator = dst_keep.put(data, classes=(args.storage_classes or []))
- dst_locators[blockhash] = dst_locator
- bytes_written += loc.size
dst_manifest.write(' ')
dst_manifest.write(dst_locators[blockhash])
dst_manifest.write("\n")
@@ -758,6 +884,10 @@ def git_rev_parse(rev, repo):
def uuid_type(api, object_uuid):
if re.match(arvados.util.keep_locator_pattern, object_uuid):
return 'Collection'
+
+ if object_uuid.startswith("http:") or object_uuid.startswith("https:"):
+ return 'httpURL'
+
p = object_uuid.split('-')
if len(p) == 3:
type_prefix = p[1]
@@ -767,6 +897,27 @@ def uuid_type(api, object_uuid):
return k
return None
+
+def copy_from_http(url, src, dst, args):
+
+ project_uuid = args.project_uuid
+ varying_url_params = args.varying_url_params
+ prefer_cached_downloads = args.prefer_cached_downloads
+
+ cached = arvados.http_to_keep.check_cached_url(src, project_uuid, url, {},
+ varying_url_params=varying_url_params,
+ prefer_cached_downloads=prefer_cached_downloads)
+ if cached[2] is not None:
+ return copy_collection(cached[2], src, dst, args)
+
+ cached = arvados.http_to_keep.http_to_keep(dst, project_uuid, url,
+ varying_url_params=varying_url_params,
+ prefer_cached_downloads=prefer_cached_downloads)
+
+ if cached is not None:
+ return {"uuid": cached[2]}
+
+
def abort(msg, code=1):
logger.info("arv-copy: %s", msg)
exit(code)
diff --git a/sdk/python/arvados/http_to_keep.py b/sdk/python/arvados/http_to_keep.py
index 16c3dc4778..1da8cf4946 100644
--- a/sdk/python/arvados/http_to_keep.py
+++ b/sdk/python/arvados/http_to_keep.py
@@ -182,6 +182,10 @@ class _Downloader(PyCurlHelper):
mt = re.match(r'^HTTP\/(\d(\.\d)?) ([1-5]\d\d) ([^\r\n\x00-\x08\x0b\x0c\x0e-\x1f\x7f]*)\r\n$', self._headers["x-status-line"])
code = int(mt.group(3))
+ if not self.name:
+ logger.error("Cannot determine filename from URL or headers")
+ return
+
if code == 200:
self.target = self.collection.open(self.name, "wb")
@@ -191,6 +195,13 @@ class _Downloader(PyCurlHelper):
self._first_chunk = False
self.count += len(chunk)
+
+ if self.target is None:
+ # "If this number is not equal to the size of the byte
+ # string, this signifies an error and libcurl will abort
+ # the request."
+ return 0
+
self.target.write(chunk)
loopnow = time.time()
if (loopnow - self.checkpoint) < 20:
@@ -238,16 +249,10 @@ def _etag_quote(etag):
return '"' + etag + '"'
-def http_to_keep(api, project_uuid, url,
- utcnow=datetime.datetime.utcnow, varying_url_params="",
- prefer_cached_downloads=False):
- """Download a file over HTTP and upload it to keep, with HTTP headers as metadata.
-
- Before downloading the URL, checks to see if the URL already
- exists in Keep and applies HTTP caching policy, the
- varying_url_params and prefer_cached_downloads flags in order to
- decide whether to use the version in Keep or re-download it.
- """
+def check_cached_url(api, project_uuid, url, etags,
+ utcnow=datetime.datetime.utcnow,
+ varying_url_params="",
+ prefer_cached_downloads=False):
logger.info("Checking Keep for %s", url)
@@ -270,8 +275,6 @@ def http_to_keep(api, project_uuid, url,
now = utcnow()
- etags = {}
-
curldownloader = _Downloader(api)
for item in items:
@@ -287,13 +290,13 @@ def http_to_keep(api, project_uuid, url,
if prefer_cached_downloads or _fresh_cache(cache_url, properties, now):
# HTTP caching rules say we should use the cache
cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
- return (item["portable_data_hash"], next(iter(cr.keys())) )
+ return (item["portable_data_hash"], next(iter(cr.keys())), item["uuid"], clean_url, now)
if not _changed(cache_url, clean_url, properties, now, curldownloader):
# Etag didn't change, same content, just update headers
api.collections().update(uuid=item["uuid"], body={"collection":{"properties": properties}}).execute()
cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
- return (item["portable_data_hash"], next(iter(cr.keys())))
+ return (item["portable_data_hash"], next(iter(cr.keys())), item["uuid"], clean_url, now)
for etagstr in ("Etag", "ETag"):
if etagstr in properties[cache_url] and len(properties[cache_url][etagstr]) > 2:
@@ -301,6 +304,31 @@ def http_to_keep(api, project_uuid, url,
logger.debug("Found ETag values %s", etags)
+ return (None, None, None, clean_url, now)
+
+
+def http_to_keep(api, project_uuid, url,
+ utcnow=datetime.datetime.utcnow, varying_url_params="",
+ prefer_cached_downloads=False):
+ """Download a file over HTTP and upload it to keep, with HTTP headers as metadata.
+
+ Before downloading the URL, checks to see if the URL already
+ exists in Keep and applies HTTP caching policy, the
+ varying_url_params and prefer_cached_downloads flags in order to
+ decide whether to use the version in Keep or re-download it.
+ """
+
+ etags = {}
+ cache_result = check_cached_url(api, project_uuid, url, etags,
+ utcnow, varying_url_params,
+ prefer_cached_downloads)
+
+ if cache_result[0] is not None:
+ return cache_result
+
+ clean_url = cache_result[3]
+ now = cache_result[4]
+
properties = {}
headers = {}
if etags:
@@ -309,6 +337,8 @@ def http_to_keep(api, project_uuid, url,
logger.info("Beginning download of %s", url)
+ curldownloader = _Downloader(api)
+
req = curldownloader.download(url, headers)
c = curldownloader.collection
@@ -326,7 +356,7 @@ def http_to_keep(api, project_uuid, url,
item["properties"].update(properties)
api.collections().update(uuid=item["uuid"], body={"collection":{"properties": item["properties"]}}).execute()
cr = arvados.collection.CollectionReader(item["portable_data_hash"], api_client=api)
- return (item["portable_data_hash"], list(cr.keys())[0])
+ return (item["portable_data_hash"], list(cr.keys())[0], item["uuid"], clean_url, now)
logger.info("Download complete")
@@ -344,4 +374,4 @@ def http_to_keep(api, project_uuid, url,
api.collections().update(uuid=c.manifest_locator(), body={"collection":{"properties": properties}}).execute()
- return (c.portable_data_hash(), curldownloader.name)
+ return (c.portable_data_hash(), curldownloader.name, c.manifest_locator(), clean_url, now)
diff --git a/sdk/python/tests/test_http.py b/sdk/python/tests/test_http.py
index 381a61e2aa..bce57eda61 100644
--- a/sdk/python/tests/test_http.py
+++ b/sdk/python/tests/test_http.py
@@ -97,7 +97,9 @@ class TestHttpToKeep(unittest.TestCase):
utcnow.return_value = datetime.datetime(2018, 5, 15)
r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
- self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt",
+ 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt',
+ datetime.datetime(2018, 5, 15, 0, 0)))
assert mockobj.url == b"http://example.com/file1.txt"
assert mockobj.perform_was_called is True
@@ -146,7 +148,9 @@ class TestHttpToKeep(unittest.TestCase):
utcnow.return_value = datetime.datetime(2018, 5, 16)
r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
- self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt",
+ 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt',
+ datetime.datetime(2018, 5, 16, 0, 0)))
assert mockobj.perform_was_called is False
@@ -185,7 +189,8 @@ class TestHttpToKeep(unittest.TestCase):
utcnow.return_value = datetime.datetime(2018, 5, 16)
r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
- self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3',
+ 'http://example.com/file1.txt', datetime.datetime(2018, 5, 16, 0, 0)))
assert mockobj.perform_was_called is False
@@ -224,7 +229,10 @@ class TestHttpToKeep(unittest.TestCase):
utcnow.return_value = datetime.datetime(2018, 5, 17)
r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
- self.assertEqual(r, ("99999999999999999999999999999997+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999997+99", "file1.txt",
+ 'zzzzz-4zz18-zzzzzzzzzzzzzz4',
+ 'http://example.com/file1.txt', datetime.datetime(2018, 5, 17, 0, 0)))
+
assert mockobj.url == b"http://example.com/file1.txt"
assert mockobj.perform_was_called is True
@@ -278,7 +286,9 @@ class TestHttpToKeep(unittest.TestCase):
utcnow.return_value = datetime.datetime(2018, 5, 17)
r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
- self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt",
+ 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt',
+ datetime.datetime(2018, 5, 17, 0, 0)))
cm.open.assert_not_called()
@@ -315,7 +325,10 @@ class TestHttpToKeep(unittest.TestCase):
utcnow.return_value = datetime.datetime(2018, 5, 15)
r = http_to_keep(api, None, "http://example.com/download?fn=/file1.txt", utcnow=utcnow)
- self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt",
+ 'zzzzz-4zz18-zzzzzzzzzzzzzz3',
+ 'http://example.com/download?fn=/file1.txt',
+ datetime.datetime(2018, 5, 15, 0, 0)))
assert mockobj.url == b"http://example.com/download?fn=/file1.txt"
@@ -369,7 +382,9 @@ class TestHttpToKeep(unittest.TestCase):
utcnow.return_value = datetime.datetime(2018, 5, 17)
r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow)
- self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt",
+ 'zzzzz-4zz18-zzzzzzzzzzzzzz3', 'http://example.com/file1.txt',
+ datetime.datetime(2018, 5, 17, 0, 0)))
print(mockobj.req_headers)
assert mockobj.req_headers == ["Accept: application/octet-stream", "If-None-Match: \"123456\""]
@@ -418,7 +433,8 @@ class TestHttpToKeep(unittest.TestCase):
utcnow.return_value = datetime.datetime(2018, 5, 17)
r = http_to_keep(api, None, "http://example.com/file1.txt", utcnow=utcnow, prefer_cached_downloads=True)
- self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3',
+ 'http://example.com/file1.txt', datetime.datetime(2018, 5, 17, 0, 0)))
assert mockobj.perform_was_called is False
cm.open.assert_not_called()
@@ -465,7 +481,8 @@ class TestHttpToKeep(unittest.TestCase):
r = http_to_keep(api, None, "http://example.com/file1.txt?KeyId=123&Signature=456&Expires=789",
utcnow=utcnow, varying_url_params="KeyId,Signature,Expires")
- self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt"))
+ self.assertEqual(r, ("99999999999999999999999999999998+99", "file1.txt", 'zzzzz-4zz18-zzzzzzzzzzzzzz3',
+ 'http://example.com/file1.txt', datetime.datetime(2018, 5, 17, 0, 0)))
assert mockobj.perform_was_called is True
cm.open.assert_not_called()
commit 6b551f2aa377c627afd30e243cf65fb05fe1635c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Oct 3 09:54:30 2023 -0400
Merge branch '20990-name-btree' refs #20990
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/api/db/migrate/20230922000000_add_btree_name_index_to_collections_and_groups.rb b/services/api/db/migrate/20230922000000_add_btree_name_index_to_collections_and_groups.rb
new file mode 100644
index 0000000000..7e6e725c9b
--- /dev/null
+++ b/services/api/db/migrate/20230922000000_add_btree_name_index_to_collections_and_groups.rb
@@ -0,0 +1,24 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class AddBtreeNameIndexToCollectionsAndGroups < ActiveRecord::Migration[5.2]
+ #
+ # We previously added 'index_groups_on_name' and
+ # 'index_collections_on_name' but those are 'gin_trgm_ops' which is
+ # used with 'ilike' searches but despite documentation suggesting
+ # they would be, experience has shown these indexes don't get used
+ # for '=' (and/or they are much slower than the btree for exact
+ # matches).
+ #
+ # So we want to add a regular btree index.
+ #
+ def up
+ ActiveRecord::Base.connection.execute 'CREATE INDEX index_groups_on_name_btree on groups USING btree (name)'
+ ActiveRecord::Base.connection.execute 'CREATE INDEX index_collections_on_name_btree on collections USING btree (name)'
+ end
+ def down
+ ActiveRecord::Base.connection.execute 'DROP INDEX IF EXISTS index_collections_on_name_btree'
+ ActiveRecord::Base.connection.execute 'DROP INDEX IF EXISTS index_groups_on_name_btree'
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 6e8b128c9e..a26db2e5db 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -2037,6 +2037,13 @@ CREATE INDEX index_collections_on_modified_at_and_uuid ON public.collections USI
CREATE INDEX index_collections_on_name ON public.collections USING gin (name public.gin_trgm_ops);
+--
+-- Name: index_collections_on_name_btree; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_collections_on_name_btree ON public.collections USING btree (name);
+
+
--
-- Name: index_collections_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
@@ -2233,6 +2240,13 @@ CREATE INDEX index_groups_on_modified_at_and_uuid ON public.groups USING btree (
CREATE INDEX index_groups_on_name ON public.groups USING gin (name public.gin_trgm_ops);
+--
+-- Name: index_groups_on_name_btree; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_groups_on_name_btree ON public.groups USING btree (name);
+
+
--
-- Name: index_groups_on_owner_uuid; Type: INDEX; Schema: public; Owner: -
--
@@ -3293,6 +3307,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20230421142716'),
('20230503224107'),
('20230815160000'),
-('20230821000000');
+('20230821000000'),
+('20230922000000');
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list