[ARVADOS] updated: 5894f831aed536618868f2c78fee7c409fd9b0ab
Git user
git at public.curoverse.com
Thu Nov 3 15:45:56 EDT 2016
Summary of changes:
.../test/integration/pipeline_instances_test.rb | 13 +++-
sdk/python/arvados/events.py | 77 ++++++++++++++--------
services/api/app/models/arvados_model.rb | 2 +-
services/api/app/models/log.rb | 2 +-
services/api/test/unit/log_test.rb | 10 +--
services/fuse/arvados_fuse/__init__.py | 7 +-
6 files changed, 72 insertions(+), 39 deletions(-)
via 5894f831aed536618868f2c78fee7c409fd9b0ab (commit)
via 7ea4e4ac71ef2f3dde92ac1468127bfd766add03 (commit)
via ebda23ff4bf42b24dc3c11d9ccb3b053c07276c9 (commit)
via 17ac6b9f8462d63b1162a96f4c182ccc29217c5c (commit)
via a29758043e415cd436b23e61ba813f9c42aedaaa (commit)
from 251dca0361de352e4866f472c4b1b2deb0edd78b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5894f831aed536618868f2c78fee7c409fd9b0ab
Merge: 251dca0 7ea4e4a
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Nov 3 15:45:13 2016 -0400
Merge branch '10224-efficient-event-poll-startup' refs #10224
commit 7ea4e4ac71ef2f3dde92ac1468127bfd766add03
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Nov 3 14:20:44 2016 -0400
10224: Workaround Firefox failure to parse localized dates in metric locales.
diff --git a/apps/workbench/test/integration/pipeline_instances_test.rb b/apps/workbench/test/integration/pipeline_instances_test.rb
index d97850c..171580b 100644
--- a/apps/workbench/test/integration/pipeline_instances_test.rb
+++ b/apps/workbench/test/integration/pipeline_instances_test.rb
@@ -16,14 +16,21 @@ class PipelineInstancesTest < ActionDispatch::IntegrationTest
# Note: Even with all this help, phantomjs seem to behave badly
# when parsing timestamps on the other side of a DST transition.
# See skipped tests below.
+
+ # In some locales (e.g., en_CA.UTF-8) Firefox can't parse what its
+ # own toLocaleString() puts out.
+ t.sub!(/(\d\d\d\d)-(\d\d)-(\d\d)/, '\2/\3/\1')
+
if /(\d+:\d+ [AP]M) (\d+\/\d+\/\d+)/ =~ t
# Currently dates.js renders timestamps as
# '{t.toLocaleTimeString()} {t.toLocaleDateString()}' which even
- # browsers can't make sense of. First we need to flip it around
- # so it looks like what toLocaleString() would have made.
+ # en_US browsers can't make sense of. First we need to flip it
+ # around so it looks like what toLocaleString() would have made.
t = $~[2] + ', ' + $~[1]
end
- DateTime.parse(page.evaluate_script "new Date('#{t}').toUTCString()").to_time
+
+ utc = page.evaluate_script("new Date('#{t}').toUTCString()")
+ DateTime.parse(utc).to_time
end
if false
commit ebda23ff4bf42b24dc3c11d9ccb3b053c07276c9
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Nov 3 10:10:14 2016 -0400
10224: Update tests to expect event_type="delete".
diff --git a/services/api/test/unit/log_test.rb b/services/api/test/unit/log_test.rb
index 632271e..efbb189 100644
--- a/services/api/test/unit/log_test.rb
+++ b/services/api/test/unit/log_test.rb
@@ -6,7 +6,7 @@ class LogTest < ActiveSupport::TestCase
EVENT_TEST_METHODS = {
:create => [:created_at, :assert_nil, :assert_not_nil],
:update => [:modified_at, :assert_not_nil, :assert_not_nil],
- :destroy => [nil, :assert_not_nil, :assert_nil],
+ :delete => [nil, :assert_not_nil, :assert_nil],
}
setup do
@@ -116,7 +116,7 @@ class LogTest < ActiveSupport::TestCase
orig_attrs = auth.attributes
orig_attrs.delete 'api_token'
auth.destroy
- assert_logged(auth, :destroy) do |props|
+ assert_logged(auth, :delete) do |props|
assert_equal(orig_etag, props['old_etag'], "destroyed auth etag mismatch")
assert_equal(orig_attrs, props['old_attributes'],
"destroyed auth attributes mismatch")
@@ -230,7 +230,7 @@ class LogTest < ActiveSupport::TestCase
auth.save!
assert_logged_with_clean_properties(auth, :update, 'api_token')
auth.destroy
- assert_logged_with_clean_properties(auth, :destroy, 'api_token')
+ assert_logged_with_clean_properties(auth, :delete, 'api_token')
end
test "use ownership and permission links to determine which logs a user can see" do
@@ -283,7 +283,7 @@ class LogTest < ActiveSupport::TestCase
coll.save!
assert_logged_with_clean_properties(coll, :update, 'manifest_text')
coll.destroy
- assert_logged_with_clean_properties(coll, :destroy, 'manifest_text')
+ assert_logged_with_clean_properties(coll, :delete, 'manifest_text')
end
end
@@ -302,7 +302,7 @@ class LogTest < ActiveSupport::TestCase
assert_equal(txt, props['new_attributes']['manifest_text'])
end
coll.destroy
- assert_logged(coll, :destroy) do |props|
+ assert_logged(coll, :delete) do |props|
assert_equal(txt, props['old_attributes']['manifest_text'])
end
end
commit 17ac6b9f8462d63b1162a96f4c182ccc29217c5c
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 1 11:28:49 2016 -0400
10224: Choose a recent-event threshold without querying the entire event history.
diff --git a/sdk/python/arvados/events.py b/sdk/python/arvados/events.py
index a1b4638..cf26f9e 100644
--- a/sdk/python/arvados/events.py
+++ b/sdk/python/arvados/events.py
@@ -155,39 +155,66 @@ class PollClient(threading.Thread):
self._closing_lock = threading.RLock()
def run(self):
- self.id = 0
if self.last_log_id != None:
- self.id = self.last_log_id
+ # Caller supplied the last-seen event ID from a previous
+ # connection
+ skip_old_events = [["id", ">", str(self.last_log_id)]]
else:
- for f in self.filters:
- for tries_left in RetryLoop(num_retries=25, backoff_start=.1, max_wait=self.poll_time):
- try:
- items = self.api.logs().list(limit=1, order="id desc", filters=f).execute()['items']
- break
- except errors.ApiError as error:
- pass
- else:
- tries_left = 0
- break
- if tries_left == 0:
- _logger.exception("PollClient thread could not contact API server.")
- with self._closing_lock:
- self._closing.set()
- thread.interrupt_main()
- return
- if items:
- if items[0]['id'] > self.id:
- self.id = items[0]['id']
+ # We need to do a reverse-order query to find the most
+ # recent event ID (see "if not skip_old_events" below).
+ skip_old_events = False
self.on_event({'status': 200})
while not self._closing.is_set():
- max_id = self.id
moreitems = False
for f in self.filters:
for tries_left in RetryLoop(num_retries=25, backoff_start=.1, max_wait=self.poll_time):
try:
- items = self.api.logs().list(order="id asc", filters=f+[["id", ">", str(self.id)]]).execute()
+ if not skip_old_events:
+ # If the caller didn't provide a known
+ # recent ID, our first request will ask
+ # for the single most recent event from
+ # the last 2 hours (the time restriction
+ # avoids doing an expensive database
+ # query, and leaves a big enough margin to
+ # account for clock skew). If we do find a
+ # recent event, we remember its ID but
+ # then discard it (we are supposed to be
+ # returning new/current events, not old
+ # ones).
+ #
+ # Subsequent requests will get multiple
+ # events in chronological order, and
+ # filter on that same cutoff time, or
+ # (once we see our first matching event)
+ # the ID of the last-seen event.
+ skip_old_events = [[
+ "created_at", ">=",
+ time.strftime(
+ "%Y-%m-%dT%H:%M:%SZ",
+ time.gmtime(time.time()-7200))]]
+ items = self.api.logs().list(
+ order="id desc",
+ limit=1,
+ filters=f+skip_old_events).execute()
+ if items["items"]:
+ skip_old_events = [
+ ["id", ">", str(items["items"][0]["id"])]]
+ items = {
+ "items": [],
+ "items_available": 0,
+ }
+ else:
+ # In this case, either we know the most
+ # recent matching ID, or we know there
+ # were no matching events in the 2-hour
+ # window before subscribing. Either way we
+ # can safely ask for events in ascending
+ # order.
+ items = self.api.logs().list(
+ order="id asc",
+ filters=f+skip_old_events).execute()
break
except errors.ApiError as error:
pass
@@ -201,8 +228,7 @@ class PollClient(threading.Thread):
thread.interrupt_main()
return
for i in items["items"]:
- if i['id'] > max_id:
- max_id = i['id']
+ skip_old_events = [["id", ">", str(i["id"])]]
with self._closing_lock:
if self._closing.is_set():
return
@@ -213,7 +239,6 @@ class PollClient(threading.Thread):
thread.interrupt_main()
if items["items_available"] > len(items["items"]):
moreitems = True
- self.id = max_id
if not moreitems:
self._closing.wait(self.poll_time)
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index f32a5db..1828e15 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -367,9 +367,10 @@ class Operations(llfuse.Operations):
return True
def listen_for_events(self):
- self.events = arvados.events.subscribe(self._api_client,
- [["event_type", "in", ["create", "update", "delete"]]],
- self.on_event)
+ self.events = arvados.events.subscribe(
+ self._api_client,
+ [["event_type", "in", ["create", "update", "delete"]]],
+ self.on_event)
@catch_exceptions
def on_event(self, ev):
commit a29758043e415cd436b23e61ba813f9c42aedaaa
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Nov 1 11:29:42 2016 -0400
10224: Change event_type to "delete" to match HTTP verb and Python SDK code.
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 672374b..18d5647 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -636,7 +636,7 @@ class ArvadosModel < ActiveRecord::Base
end
def log_destroy
- log_change('destroy') do |log|
+ log_change('delete') do |log|
log.fill_properties('old', etag(@old_attributes), @old_logged_attributes)
log.update_to nil
end
diff --git a/services/api/app/models/log.rb b/services/api/app/models/log.rb
index f8d624a..7eab402 100644
--- a/services/api/app/models/log.rb
+++ b/services/api/app/models/log.rb
@@ -47,7 +47,7 @@ class Log < ArvadosModel
self.event_at = thing.created_at
when "update"
self.event_at = thing.modified_at
- when "destroy"
+ when "delete"
self.event_at = db_current_time
end
self
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list