[ARVADOS] updated: 62d8f575a93374a235097b88499fa65eef2f56ec
Git user
git at public.curoverse.com
Tue Dec 13 13:27:47 EST 2016
Summary of changes:
sdk/go/arvados/log.go | 1 +
services/fuse/arvados_fuse/__init__.py | 30 +++++++++++-------------------
services/fuse/arvados_fuse/command.py | 2 ++
services/fuse/tests/mount_test_base.py | 2 ++
services/ws/event.go | 3 ++-
services/ws/session_v0.go | 3 ++-
6 files changed, 20 insertions(+), 21 deletions(-)
via 62d8f575a93374a235097b88499fa65eef2f56ec (commit)
via ba1ec0f0b59ab871b6e4faf5e8ae87809fdb85b6 (commit)
from b9f0177e6a477a518fd5a89156fafde57f2dddf8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 62d8f575a93374a235097b88499fa65eef2f56ec
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Dec 13 12:46:02 2016 -0500
8460: Fix deadlock at shutdown by closing event stream before unmounting.
If llfuse shuts down while a thread is in a "with
llfuse.lock_released" block, the llfuse lock cannot be reacquired, so
the "with" block waits forever instead of exiting. The event listener
thread lands in this state easily because handling a "collection
updated" event makes network requests in a lock_released block.
This deadlock bug started occurring frequently in
tests.test_token_expiry.TokenExpiryTest when using arvados-ws.
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 63a5513..4c1731f 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -399,7 +399,6 @@ class Operations(llfuse.Operations):
parent.invalidate()
parent.update()
-
@catch_exceptions
def getattr(self, inode):
if inode not in self.inodes:
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index f2948f9..ffcfc65 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -126,6 +126,8 @@ class Mount(object):
return self
def __exit__(self, exc_type, exc_value, traceback):
+ if self.operations.events:
+ self.operations.events.close(timeout=self.args.unmount_timeout)
subprocess.call(["fusermount", "-u", "-z", self.args.mountpoint])
self.llfuse_thread.join(timeout=self.args.unmount_timeout)
if self.llfuse_thread.is_alive():
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index 20192f9..1319aeb 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -66,6 +66,8 @@ class MountTestBase(unittest.TestCase):
def tearDown(self):
if self.llfuse_thread:
+ if self.operations.events:
+ self.operations.events.close(timeout=10)
subprocess.call(["fusermount", "-u", "-z", self.mounttmp])
t0 = time.time()
self.llfuse_thread.join(timeout=10)
commit ba1ec0f0b59ab871b6e4faf5e8ae87809fdb85b6
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Dec 11 03:15:54 2016 -0500
8460: Add event_at and props.{new,old}.{pdh,owner} to websocket v0. Fix fuse crash when event_at or props.*.* is missing.
diff --git a/sdk/go/arvados/log.go b/sdk/go/arvados/log.go
index ef56e85..a48f1c6 100644
--- a/sdk/go/arvados/log.go
+++ b/sdk/go/arvados/log.go
@@ -11,6 +11,7 @@ type Log struct {
ObjectUUID string `json:"object_uuid"`
ObjectOwnerUUID string `json:"object_owner_uuid"`
EventType string `json:"event_type"`
+ EventAt *time.Time `json:"event,omitempty"`
Properties map[string]interface{} `json:"properties"`
CreatedAt *time.Time `json:"created_at,omitempty"`
}
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 1828e15..63a5513 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -377,29 +377,22 @@ class Operations(llfuse.Operations):
if 'event_type' not in ev:
return
with llfuse.lock:
+ new_attrs = ev.get("properties", {}).get("new_attributes", {})
+ pdh = new_attrs.get("portable_data_hash")
+ # new_attributes.modified_at currently lacks
+ # subsecond precision (see #6347) so use event_at
+ # which should always be the same.
+ stamp = ev.get("event_at")
+
for item in self.inodes.inode_cache.find_by_uuid(ev["object_uuid"]):
item.invalidate()
- if ev["object_kind"] == "arvados#collection":
- new_attr = (ev.get("properties") and
- ev["properties"].get("new_attributes") and
- ev["properties"]["new_attributes"])
-
- # new_attributes.modified_at currently lacks
- # subsecond precision (see #6347) so use event_at
- # which should always be the same.
- record_version = (
- (ev["event_at"], new_attr["portable_data_hash"])
- if new_attr else None)
-
- item.update(to_record_version=record_version)
+ if stamp and pdh and ev.get("object_kind") == "arvados#collection":
+ item.update(to_record_version=(stamp, pdh))
else:
item.update()
- oldowner = (
- ev.get("properties") and
- ev["properties"].get("old_attributes") and
- ev["properties"]["old_attributes"].get("owner_uuid"))
- newowner = ev["object_owner_uuid"]
+ oldowner = ev.get("properties", {}).get("old_attributes", {}).get("owner_uuid")
+ newowner = ev.get("object_owner_uuid")
for parent in (
self.inodes.inode_cache.find_by_uuid(oldowner) +
self.inodes.inode_cache.find_by_uuid(newowner)):
diff --git a/services/ws/event.go b/services/ws/event.go
index fa2a5df..304f86b 100644
--- a/services/ws/event.go
+++ b/services/ws/event.go
@@ -42,12 +42,13 @@ func (e *event) Detail() *arvados.Log {
}
var logRow arvados.Log
var propYAML []byte
- e.err = e.db.QueryRow(`SELECT id, uuid, object_uuid, COALESCE(object_owner_uuid,''), COALESCE(event_type,''), created_at, properties FROM logs WHERE id = $1`, e.LogID).Scan(
+ e.err = e.db.QueryRow(`SELECT id, uuid, object_uuid, COALESCE(object_owner_uuid,''), COALESCE(event_type,''), event_at, created_at, properties FROM logs WHERE id = $1`, e.LogID).Scan(
&logRow.ID,
&logRow.UUID,
&logRow.ObjectUUID,
&logRow.ObjectOwnerUUID,
&logRow.EventType,
+ &logRow.EventAt,
&logRow.CreatedAt,
&propYAML)
if e.err != nil {
diff --git a/services/ws/session_v0.go b/services/ws/session_v0.go
index 2b108a2..364555a 100644
--- a/services/ws/session_v0.go
+++ b/services/ws/session_v0.go
@@ -16,7 +16,7 @@ var (
errQueueFull = errors.New("client queue full")
errFrameTooBig = errors.New("frame too big")
- sendObjectAttributes = []string{"state", "name"}
+ sendObjectAttributes = []string{"state", "name", "owner_uuid", "portable_data_hash"}
v0subscribeOK = []byte(`{"status":200}`)
v0subscribeFail = []byte(`{"status":400}`)
@@ -97,6 +97,7 @@ func (sess *v0session) EventMessage(e *event) ([]byte, error) {
"object_owner_uuid": detail.ObjectOwnerUUID,
"object_kind": kind,
"event_type": detail.EventType,
+ "event_at": detail.EventAt,
}
if detail.Properties != nil && detail.Properties["text"] != nil {
msg["properties"] = detail.Properties
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list