[ARVADOS] updated: 2.3.2-25-gd09550506
Git user
git at public.arvados.org
Mon Feb 14 19:24:26 UTC 2022
Summary of changes:
.../app/controllers/actions_controller.rb | 2 +-
.../app/controllers/application_controller.rb | 8 +-
.../app/controllers/user_agreements_controller.rb | 12 +-
lib/crunchrun/crunchrun.go | 32 +++-
lib/crunchrun/crunchrun_test.go | 51 +++++-
sdk/cwl/arvados_cwl/__init__.py | 6 +-
sdk/cwl/arvados_cwl/runner.py | 7 +-
sdk/cwl/setup.py | 7 +-
sdk/cwl/tests/arvados-tests.yml | 5 +
sdk/dev-jobs.dockerfile | 6 +-
sdk/python/arvados/arvfile.py | 7 +-
sdk/python/arvados/collection.py | 2 +-
services/fuse/arvados_fuse/__init__.py | 13 +-
services/fuse/arvados_fuse/fusedir.py | 175 ++++++++++++---------
14 files changed, 217 insertions(+), 116 deletions(-)
via d09550506dbd31b92e53b4e861924e49027acabb (commit)
via c364f8b4010adb72c952b7a1c47011675c42fae5 (commit)
via ce4962218956e42058d6084f71d54bfa869b5ed4 (commit)
via dc032a8b37d9360a3ba90752e1a2412c9838e1fa (commit)
from fc415fd3128c5f3772a7f1a1376007c75b588b4c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d09550506dbd31b92e53b4e861924e49027acabb
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Feb 14 13:50:05 2022 -0500
Merge branch '18723-cwl-upload' refs #18723
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 71ef742e3..efac0e097 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -257,7 +257,11 @@ def exit_signal_handler(sigcode, frame):
logger.error(str(u"Caught signal {}, exiting.").format(sigcode))
sys.exit(-sigcode)
-def main(args, stdout, stderr, api_client=None, keep_client=None,
+def main(args=sys.argv[1:],
+ stdout=sys.stdout,
+ stderr=sys.stderr,
+ api_client=None,
+ keep_client=None,
install_sig_handlers=True):
parser = arg_parser()
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 145f1ad7f..7d6d287a2 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -283,9 +283,10 @@ def upload_dependencies(arvrunner, name, document_loader,
metadata = scanobj
sc_result = scandeps(uri, scanobj,
- loadref_fields,
- set(("$include", "$schemas", "location")),
- loadref, urljoin=document_loader.fetcher.urljoin)
+ loadref_fields,
+ set(("$include", "$schemas", "location")),
+ loadref, urljoin=document_loader.fetcher.urljoin,
+ nestdirs=False)
sc = []
uuids = {}
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index f034ca5ab..e73923595 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -31,15 +31,12 @@ setup(name='arvados-cwl-runner',
license='Apache 2.0',
packages=find_packages(),
package_data={'arvados_cwl': ['arv-cwl-schema-v1.0.yml', 'arv-cwl-schema-v1.1.yml', 'arv-cwl-schema-v1.2.yml']},
- scripts=[
- 'bin/cwl-runner',
- 'bin/arvados-cwl-runner',
- ],
+ entry_points={"console_scripts": ["cwl-runner=arvados_cwl:main", "arvados-cwl-runner=arvados_cwl:main"]},
# Note that arvados/build/run-build-packages.sh looks at this
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.1.20211107152837',
+ 'cwltool==3.1.20220210171524',
'schema-salad==8.2.20211116214159',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index ae22d65f4..5282e9392 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -439,3 +439,8 @@
"outstr": "foo woble bar"
tool: 17879-ignore-sbg-fields.cwl
doc: "Test issue 17879 - ignores sbg fields"
+
+- job: chipseq/chip-seq-single.json
+ output: {}
+ tool: chipseq/cwl-packed.json
+ doc: "Test issue 18723 - correctly upload two directories with the same basename"
diff --git a/sdk/dev-jobs.dockerfile b/sdk/dev-jobs.dockerfile
index 1e0068ffd..b55b056b2 100644
--- a/sdk/dev-jobs.dockerfile
+++ b/sdk/dev-jobs.dockerfile
@@ -23,9 +23,7 @@ ARG pipcmd=pip3
RUN apt-get update -q && apt-get install -qy --no-install-recommends \
git ${pythoncmd}-pip ${pythoncmd}-virtualenv ${pythoncmd}-dev libcurl4-gnutls-dev \
- libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential
-
-RUN $pipcmd install -U setuptools six requests
+ libgnutls28-dev nodejs ${pythoncmd}-pyasn1-modules build-essential ${pythoncmd}-setuptools
ARG sdk
ARG runner
@@ -39,7 +37,7 @@ ADD cwl/dist/$runner /tmp/
RUN cd /tmp/arvados-python-client-* && $pipcmd install .
RUN if test -d /tmp/schema-salad-* ; then cd /tmp/schema-salad-* && $pipcmd install . ; fi
-RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install networkx==2.2 && $pipcmd install . ; fi
+RUN if test -d /tmp/cwltool-* ; then cd /tmp/cwltool-* && $pipcmd install . ; fi
RUN cd /tmp/arvados-cwl-runner-* && $pipcmd install .
# Install dependencies and set up system.
commit c364f8b4010adb72c952b7a1c47011675c42fae5
Author: Tom Clegg <tom at curii.com>
Date: Fri Feb 11 13:11:43 2022 -0500
Merge branch '18690-secret-files'
fixes #18690
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 10e5193a8..9c169d19f 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -440,8 +440,8 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
sort.Strings(binds)
for _, bind := range binds {
- mnt, ok := runner.Container.Mounts[bind]
- if !ok {
+ mnt, notSecret := runner.Container.Mounts[bind]
+ if !notSecret {
mnt = runner.SecretMounts[bind]
}
if bind == "stdout" || bind == "stderr" {
@@ -510,8 +510,7 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
}
} else {
src = fmt.Sprintf("%s/tmp%d", runner.ArvMountPoint, tmpcount)
- arvMountCmd = append(arvMountCmd, "--mount-tmp")
- arvMountCmd = append(arvMountCmd, fmt.Sprintf("tmp%d", tmpcount))
+ arvMountCmd = append(arvMountCmd, "--mount-tmp", fmt.Sprintf("tmp%d", tmpcount))
tmpcount++
}
if mnt.Writable {
@@ -571,9 +570,32 @@ func (runner *ContainerRunner) SetupMounts() (map[string]bindmount, error) {
if err != nil {
return nil, fmt.Errorf("writing temp file: %v", err)
}
- if strings.HasPrefix(bind, runner.Container.OutputPath+"/") {
+ if strings.HasPrefix(bind, runner.Container.OutputPath+"/") && (notSecret || runner.Container.Mounts[runner.Container.OutputPath].Kind != "collection") {
+ // In most cases, if the container
+ // specifies a literal file inside the
+ // output path, we copy it into the
+ // output directory (either a mounted
+ // collection or a staging area on the
+ // host fs). If it's a secret, it will
+ // be skipped when copying output from
+ // staging to Keep later.
copyFiles = append(copyFiles, copyFile{tmpfn, runner.HostOutputDir + bind[len(runner.Container.OutputPath):]})
} else {
+ // If a secret is outside OutputPath,
+ // we bind mount the secret file
+ // directly just like other mounts. We
+ // also use this strategy when a
+ // secret is inside OutputPath but
+ // OutputPath is a live collection, to
+ // avoid writing the secret to
+ // Keep. Attempting to remove a
+ // bind-mounted secret file from
+ // inside the container will return a
+ // "Device or resource busy" error
+ // that might not be handled well by
+ // the container, which is why we
+ // don't use this strategy when
+ // OutputPath is a staging directory.
bindmounts[bind] = bindmount{HostPath: tmpfn, ReadOnly: true}
}
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index c28cf73cb..af539c5ac 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -44,6 +44,7 @@ type TestSuite struct {
runner *ContainerRunner
executor *stubExecutor
keepmount string
+ keepmountTmp []string
testDispatcherKeepClient KeepTestClient
testContainerKeepClient KeepTestClient
}
@@ -62,10 +63,20 @@ func (s *TestSuite) SetUpTest(c *C) {
}
s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
s.runner.ArvMountPoint = s.keepmount
+ for i, opt := range cmd {
+ if opt == "--mount-tmp" {
+ err := os.Mkdir(s.keepmount+"/"+cmd[i+1], 0700)
+ if err != nil {
+ return nil, err
+ }
+ s.keepmountTmp = append(s.keepmountTmp, cmd[i+1])
+ }
+ }
return nil, nil
}
s.keepmount = c.MkDir()
err = os.Mkdir(s.keepmount+"/by_id", 0755)
+ s.keepmountTmp = nil
c.Assert(err, IsNil)
err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
c.Assert(err, IsNil)
@@ -638,8 +649,6 @@ func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exi
s.runner.statInterval = 100 * time.Millisecond
s.runner.containerWatchdogInterval = time.Second
- am := &ArvMountCmdLine{}
- s.runner.RunArvMount = am.ArvMountTest
realTemp := c.MkDir()
tempcount := 0
@@ -1968,6 +1977,44 @@ func (s *TestSuite) TestSecretTextMountPoint(c *C) {
c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
+
+ // under secret mounts, output dir is a collection, not captured in output
+ helperRecord = `{
+ "command": ["true"],
+ "container_image": "` + arvadostest.DockerImage112PDH + `",
+ "cwd": "/bin",
+ "mounts": {
+ "/tmp": {"kind": "collection", "writable": true}
+ },
+ "secret_mounts": {
+ "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
+ },
+ "output_path": "/tmp",
+ "priority": 1,
+ "runtime_constraints": {},
+ "state": "Locked"
+ }`
+
+ s.SetUpTest(c)
+ _, _, realtemp := s.fullRunHelper(c, helperRecord, nil, 0, func() {
+ // secret.conf should be provisioned as a separate
+ // bind mount, i.e., it should not appear in the
+ // (fake) fuse filesystem as viewed from the host.
+ content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
+ if !c.Check(errors.Is(err, os.ErrNotExist), Equals, true) {
+ c.Logf("secret.conf: content %q, err %#v", content, err)
+ }
+ err = ioutil.WriteFile(s.runner.HostOutputDir+"/.arvados#collection", []byte(`{"manifest_text":". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"}`), 0700)
+ c.Check(err, IsNil)
+ })
+
+ content, err := ioutil.ReadFile(realtemp + "/text1/mountdata.text")
+ c.Check(err, IsNil)
+ c.Check(string(content), Equals, "mypassword")
+ c.Check(s.executor.created.BindMounts["/tmp/secret.conf"], DeepEquals, bindmount{realtemp + "/text1/mountdata.text", true})
+ c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
+ c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
+ c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n"), NotNil)
}
type FakeProcess struct {
commit ce4962218956e42058d6084f71d54bfa869b5ed4
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Feb 9 16:34:54 2022 -0500
Merge branch '18719-fuse-fixes' refs #18719
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index e915ff2ac..0fcdc1e63 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -760,9 +760,10 @@ class _BlockManager(object):
self._delete_bufferblock(locator)
def _delete_bufferblock(self, locator):
- bb = self._bufferblocks[locator]
- bb.clear()
- del self._bufferblocks[locator]
+ if locator in self._bufferblocks:
+ bb = self._bufferblocks[locator]
+ bb.clear()
+ del self._bufferblocks[locator]
def get_block_contents(self, locator, num_retries, cache_only=False):
"""Fetch a block.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 55be40fa0..a076de6ba 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1395,7 +1395,7 @@ class Collection(RichCollectionBase):
# our tokens.
return
else:
- self._past_versions.add((response.get("modified_at"), response.get("portable_data_hash")))
+ self._remember_api_response(response)
other = CollectionReader(response["manifest_text"])
baseline = CollectionReader(self._manifest_text)
self.apply(baseline.diff(other))
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 1696f856a..14dd7f3f8 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -475,24 +475,13 @@ class Operations(llfuse.Operations):
for item in self.inodes.inode_cache.find_by_uuid(ev["object_uuid"]):
item.invalidate()
- if ev.get("object_kind") == "arvados#collection":
- pdh = new_attrs.get("portable_data_hash")
- # new_attributes.modified_at currently lacks
- # subsecond precision (see #6347) so use event_at
- # which should always be the same.
- stamp = ev.get("event_at")
- if (stamp and pdh and item.writable() and
- item.collection is not None and
- item.collection.modified() and
- new_attrs.get("is_trashed") is not True):
- item.update(to_record_version=(stamp, pdh))
oldowner = old_attrs.get("owner_uuid")
newowner = ev.get("object_owner_uuid")
for parent in (
self.inodes.inode_cache.find_by_uuid(oldowner) +
self.inodes.inode_cache.find_by_uuid(newowner)):
- parent.child_event(ev)
+ parent.invalidate()
@getattr_time.time()
@catch_exceptions
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index a2e33c7b3..7de95a0cb 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -270,10 +270,12 @@ class CollectionDirectoryBase(Directory):
"""
- def __init__(self, parent_inode, inodes, apiconfig, enable_write, collection):
+ def __init__(self, parent_inode, inodes, apiconfig, enable_write, collection, collection_root):
super(CollectionDirectoryBase, self).__init__(parent_inode, inodes, apiconfig, enable_write)
self.apiconfig = apiconfig
self.collection = collection
+ self.collection_root = collection_root
+ self.collection_record_file = None
def new_entry(self, name, item, mtime):
name = self.sanitize_filename(name)
@@ -285,13 +287,17 @@ class CollectionDirectoryBase(Directory):
item.fuse_entry.dead = False
self._entries[name] = item.fuse_entry
elif isinstance(item, arvados.collection.RichCollectionBase):
- self._entries[name] = self.inodes.add_entry(CollectionDirectoryBase(self.inode, self.inodes, self.apiconfig, self._enable_write, item))
+ self._entries[name] = self.inodes.add_entry(CollectionDirectoryBase(self.inode, self.inodes, self.apiconfig, self._enable_write, item, self.collection_root))
self._entries[name].populate(mtime)
else:
self._entries[name] = self.inodes.add_entry(FuseArvadosFile(self.inode, item, mtime, self._enable_write))
item.fuse_entry = self._entries[name]
def on_event(self, event, collection, name, item):
+ # These are events from the Collection object (ADD/DEL/MOD)
+ # emitted by operations on the Collection object (like
+ # "mkdirs" or "remove"), and by "update", which we need to
+ # synchronize with our FUSE objects that are assigned inodes.
if collection == self.collection:
name = self.sanitize_filename(name)
@@ -336,6 +342,10 @@ class CollectionDirectoryBase(Directory):
self.inodes.invalidate_inode(item.fuse_entry)
elif name in self._entries:
self.inodes.invalidate_inode(self._entries[name])
+
+ if self.collection_record_file is not None:
+ self.collection_record_file.invalidate()
+ self.inodes.invalidate_inode(self.collection_record_file)
finally:
while lockcount > 0:
self.collection.lock.acquire()
@@ -353,10 +363,7 @@ class CollectionDirectoryBase(Directory):
@use_counter
def flush(self):
- if not self.writable():
- return
- with llfuse.lock_released:
- self.collection.root_collection().save()
+ self.collection_root.flush()
@use_counter
@check_update
@@ -428,11 +435,9 @@ class CollectionDirectory(CollectionDirectoryBase):
"""Represents the root of a directory tree representing a collection."""
def __init__(self, parent_inode, inodes, api, num_retries, enable_write, collection_record=None, explicit_collection=None):
- super(CollectionDirectory, self).__init__(parent_inode, inodes, api.config, enable_write, None)
+ super(CollectionDirectory, self).__init__(parent_inode, inodes, api.config, enable_write, None, self)
self.api = api
self.num_retries = num_retries
- self.collection_record_file = None
- self.collection_record = None
self._poll = True
try:
self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2) // 2)
@@ -457,62 +462,70 @@ class CollectionDirectory(CollectionDirectoryBase):
def writable(self):
return self._enable_write and (self.collection.writable() if self.collection is not None else self._writable)
+ @use_counter
+ def flush(self):
+ if not self.writable():
+ return
+ with llfuse.lock_released:
+ with self._updating_lock:
+ if self.collection.committed():
+ self.collection.update()
+ else:
+ self.collection.save()
+ self.new_collection_record(self.collection.api_response())
+
def want_event_subscribe(self):
return (uuid_pattern.match(self.collection_locator) is not None)
- # Used by arv-web.py to switch the contents of the CollectionDirectory
- def change_collection(self, new_locator):
- """Switch the contents of the CollectionDirectory.
-
- Must be called with llfuse.lock held.
- """
-
- self.collection_locator = new_locator
- self.collection_record = None
- self.update()
-
def new_collection(self, new_collection_record, coll_reader):
if self.inode:
self.clear()
-
- self.collection_record = new_collection_record
-
- if self.collection_record:
- self._mtime = convertTime(self.collection_record.get('modified_at'))
- self.collection_locator = self.collection_record["uuid"]
- if self.collection_record_file is not None:
- self.collection_record_file.update(self.collection_record)
-
self.collection = coll_reader
+ self.new_collection_record(new_collection_record)
self.populate(self.mtime())
+ def new_collection_record(self, new_collection_record):
+ if not new_collection_record:
+ raise Exception("invalid new_collection_record")
+ self._mtime = convertTime(new_collection_record.get('modified_at'))
+ self._manifest_size = len(new_collection_record["manifest_text"])
+ self.collection_locator = new_collection_record["uuid"]
+ if self.collection_record_file is not None:
+ self.collection_record_file.invalidate()
+ self.inodes.invalidate_inode(self.collection_record_file)
+ _logger.debug("%s invalidated collection record file", self)
+ self.fresh()
+
def uuid(self):
return self.collection_locator
@use_counter
- def update(self, to_record_version=None):
+ def update(self):
try:
- if self.collection_record is not None and portable_data_hash_pattern.match(self.collection_locator):
+ if self.collection is not None and portable_data_hash_pattern.match(self.collection_locator):
+ # It's immutable, nothing to update
return True
if self.collection_locator is None:
+ # No collection locator to retrieve from
self.fresh()
return True
+ new_collection_record = None
try:
with llfuse.lock_released:
self._updating_lock.acquire()
if not self.stale():
- return
+ return True
- _logger.debug("Updating collection %s inode %s to record version %s", self.collection_locator, self.inode, to_record_version)
- new_collection_record = None
+ _logger.debug("Updating collection %s inode %s", self.collection_locator, self.inode)
+ coll_reader = None
if self.collection is not None:
- if self.collection.known_past_version(to_record_version):
- _logger.debug("%s already processed %s", self.collection_locator, to_record_version)
- else:
- self.collection.update()
+ # Already have a collection object
+ self.collection.update()
+ new_collection_record = self.collection.api_response()
else:
+ # Create a new collection object
if uuid_pattern.match(self.collection_locator):
coll_reader = arvados.collection.Collection(
self.collection_locator, self.api, self.api.keep,
@@ -534,14 +547,13 @@ class CollectionDirectory(CollectionDirectoryBase):
new_collection_record['storage_classes_desired'] = coll_reader.storage_classes_desired()
# end with llfuse.lock_released, re-acquire lock
- if (new_collection_record is not None and
- (self.collection_record is None or
- self.collection_record["portable_data_hash"] != new_collection_record.get("portable_data_hash"))):
- self.new_collection(new_collection_record, coll_reader)
- self._manifest_size = len(coll_reader.manifest_text())
- _logger.debug("%s manifest_size %i", self, self._manifest_size)
- self.fresh()
+ if new_collection_record is not None:
+ if coll_reader is not None:
+ self.new_collection(new_collection_record, coll_reader)
+ else:
+ self.new_collection_record(new_collection_record)
+
return True
finally:
self._updating_lock.release()
@@ -549,22 +561,29 @@ class CollectionDirectory(CollectionDirectoryBase):
_logger.error("Error fetching collection '%s': %s", self.collection_locator, e)
except arvados.errors.ArgumentError as detail:
_logger.warning("arv-mount %s: error %s", self.collection_locator, detail)
- if self.collection_record is not None and "manifest_text" in self.collection_record:
- _logger.warning("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
+ if new_collection_record is not None and "manifest_text" in new_collection_record:
+ _logger.warning("arv-mount manifest_text is: %s", new_collection_record["manifest_text"])
except Exception:
_logger.exception("arv-mount %s: error", self.collection_locator)
- if self.collection_record is not None and "manifest_text" in self.collection_record:
- _logger.error("arv-mount manifest_text is: %s", self.collection_record["manifest_text"])
+ if new_collection_record is not None and "manifest_text" in new_collection_record:
+ _logger.error("arv-mount manifest_text is: %s", new_collection_record["manifest_text"])
self.invalidate()
return False
+ @use_counter
+ def collection_record(self):
+ self.flush()
+ return self.collection.api_response()
+
@use_counter
@check_update
def __getitem__(self, item):
if item == '.arvados#collection':
if self.collection_record_file is None:
- self.collection_record_file = ObjectFile(self.inode, self.collection_record)
+ self.collection_record_file = FuncToJSONFile(
+ self.inode, self.collection_record)
self.inodes.add_entry(self.collection_record_file)
+ self.invalidate() # use lookup as a signal to force update
return self.collection_record_file
else:
return super(CollectionDirectory, self).__getitem__(item)
@@ -576,8 +595,9 @@ class CollectionDirectory(CollectionDirectoryBase):
return super(CollectionDirectory, self).__contains__(k)
def invalidate(self):
- self.collection_record = None
- self.collection_record_file = None
+ if self.collection_record_file is not None:
+ self.collection_record_file.invalidate()
+ self.inodes.invalidate_inode(self.collection_record_file)
super(CollectionDirectory, self).invalidate()
def persisted(self):
@@ -626,33 +646,33 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
# This is always enable_write=True because it never tries to
# save to the backend
super(TmpCollectionDirectory, self).__init__(
- parent_inode, inodes, api_client.config, True, collection)
- self.collection_record_file = None
+ parent_inode, inodes, api_client.config, True, collection, self)
self.populate(self.mtime())
def on_event(self, *args, **kwargs):
super(TmpCollectionDirectory, self).on_event(*args, **kwargs)
- if self.collection_record_file:
+ if self.collection_record_file is None:
+ return
- # See discussion in CollectionDirectoryBase.on_event
- lockcount = 0
- try:
- while True:
- self.collection.lock.release()
- lockcount += 1
- except RuntimeError:
- pass
+ # See discussion in CollectionDirectoryBase.on_event
+ lockcount = 0
+ try:
+ while True:
+ self.collection.lock.release()
+ lockcount += 1
+ except RuntimeError:
+ pass
- try:
- with llfuse.lock:
- with self.collection.lock:
- self.collection_record_file.invalidate()
- self.inodes.invalidate_inode(self.collection_record_file)
- _logger.debug("%s invalidated collection record", self)
- finally:
- while lockcount > 0:
- self.collection.lock.acquire()
- lockcount -= 1
+ try:
+ with llfuse.lock:
+ with self.collection.lock:
+ self.collection_record_file.invalidate()
+ self.inodes.invalidate_inode(self.collection_record_file)
+ _logger.debug("%s invalidated collection record", self)
+ finally:
+ while lockcount > 0:
+ self.collection.lock.acquire()
+ lockcount -= 1
def collection_record(self):
with llfuse.lock_released:
@@ -683,6 +703,9 @@ class TmpCollectionDirectory(CollectionDirectoryBase):
def writable(self):
return True
+ def flush(self):
+ pass
+
def want_event_subscribe(self):
return False
@@ -973,11 +996,13 @@ class ProjectDirectory(Directory):
uuid=self.project_uuid,
filters=[["uuid", "is_a", "arvados#group"],
["groups.group_class", "in", ["project","filter"]]]))
- contents.extend(arvados.util.keyset_list_all(self.api.groups().contents,
+ contents.extend(filter(lambda i: i["current_version_uuid"] == i["uuid"],
+ arvados.util.keyset_list_all(self.api.groups().contents,
order_key="uuid",
num_retries=self.num_retries,
uuid=self.project_uuid,
- filters=[["uuid", "is_a", "arvados#collection"]]))
+ filters=[["uuid", "is_a", "arvados#collection"]])))
+
# end with llfuse.lock_released, re-acquire lock
commit dc032a8b37d9360a3ba90752e1a2412c9838e1fa
Author: Ward Vandewege <ward at curii.com>
Date: Fri Jan 7 13:56:25 2022 -0500
Merge branch '18577-fix-back_url'
closes #18577
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/apps/workbench/app/controllers/actions_controller.rb b/apps/workbench/app/controllers/actions_controller.rb
index b0b7a0b64..3667d8aca 100644
--- a/apps/workbench/app/controllers/actions_controller.rb
+++ b/apps/workbench/app/controllers/actions_controller.rb
@@ -49,7 +49,7 @@ class ActionsController < ApplicationController
return self.send(param)
end
end
- redirect_to :back
+ redirect_back(fallback_location: root_path)
end
expose_action :copy_selections_into_project do
diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb
index 04055f848..5312e733f 100644
--- a/apps/workbench/app/controllers/application_controller.rb
+++ b/apps/workbench/app/controllers/application_controller.rb
@@ -417,7 +417,11 @@ class ApplicationController < ActionController::Base
respond_to do |f|
f.json { render json: @object }
f.html {
- redirect_to(params[:return_to] || :back)
+ if params[:return_to]
+ redirect_to(params[:return_to])
+ else
+ redirect_back(fallback_location: root_path)
+ end
}
f.js { render }
end
@@ -519,7 +523,7 @@ class ApplicationController < ActionController::Base
redirect_to arvados_api_client.arvados_login_url(return_to: strip_token_from_path(request.url))
else
flash[:error] = "Either you are not logged in, or your session has timed out. I can't automatically log you in and re-attempt this request."
- redirect_to :back
+ redirect_back(fallback_location: root_path)
end
false # For convenience to return from callbacks
end
diff --git a/apps/workbench/app/controllers/user_agreements_controller.rb b/apps/workbench/app/controllers/user_agreements_controller.rb
index bdfaa2403..5e530a657 100644
--- a/apps/workbench/app/controllers/user_agreements_controller.rb
+++ b/apps/workbench/app/controllers/user_agreements_controller.rb
@@ -9,7 +9,11 @@ class UserAgreementsController < ApplicationController
def index
if unsigned_user_agreements.empty?
- redirect_to(params[:return_to] || :back)
+ if params[:return_to]
+ redirect_to(params[:return_to])
+ else
+ redirect_back(fallback_location: root_path)
+ end
end
end
@@ -24,6 +28,10 @@ class UserAgreementsController < ApplicationController
end
end
current_user.activate
- redirect_to(params[:return_to] || :back)
+ if params[:return_to]
+ redirect_to(params[:return_to])
+ else
+ redirect_back(fallback_location: root_path)
+ end
end
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list