[ARVADOS] updated: 1.3.0-1103-g8a9d909c5
Git user
git at public.curoverse.com
Tue Jun 18 15:11:14 UTC 2019
Summary of changes:
services/fuse/arvados_fuse/__init__.py | 33 ++++++++++++++++++++------------
services/fuse/arvados_fuse/command.py | 9 ++++++---
services/fuse/arvados_fuse/crunchstat.py | 12 +++++++-----
services/fuse/arvados_fuse/fresh.py | 1 +
services/fuse/arvados_fuse/fusedir.py | 20 +++++++++++--------
services/fuse/arvados_fuse/fusefile.py | 3 ++-
services/fuse/arvados_version.py | 2 ++
services/fuse/setup.py | 3 +--
8 files changed, 52 insertions(+), 31 deletions(-)
via 8a9d909c5a407a9aedcdc333ec4cfa3f1762b3c3 (commit)
via 8dfb0e3f8ca8154f4c16ab26e03aa6c6b114b5ad (commit)
from acb06e145fc50740496a34d2804e3438b2ddb2b1 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8a9d909c5a407a9aedcdc333ec4cfa3f1762b3c3
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Mon Jun 17 17:34:16 2019 -0400
15295: Updates fuse code with futurize stage2 and fixes tests
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 72ccdb5f9..f944a9197 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -50,6 +50,15 @@ an object that is live in the inode cache, that object is immediately updated.
"""
from __future__ import absolute_import
+from __future__ import division
+from future.utils import viewitems
+from future.utils import native
+from future import standard_library
+standard_library.install_aliases()
+from builtins import next
+from builtins import str
+from builtins import object
+from builtins import dict
import os
import sys
import llfuse
@@ -73,8 +82,7 @@ import collections
import functools
import arvados.keep
from prometheus_client import Summary
-
-import Queue
+import queue
# Default _notify_queue has a limit of 1000 items, but it really needs to be
# unlimited to avoid deadlocks, see https://arvados.org/issues/3198#note-43 for
@@ -82,10 +90,10 @@ import Queue
if hasattr(llfuse, 'capi'):
# llfuse < 0.42
- llfuse.capi._notify_queue = Queue.Queue()
+ llfuse.capi._notify_queue = queue.Queue()
else:
# llfuse >= 0.42
- llfuse._notify_queue = Queue.Queue()
+ llfuse._notify_queue = queue.Queue()
LLFUSE_VERSION_0 = llfuse.__version__.startswith('0')
@@ -262,7 +270,7 @@ class Inodes(object):
self._entries[key] = item
def __iter__(self):
- return self._entries.iterkeys()
+ return iter(self._entries.keys())
def items(self):
return self._entries.items()
@@ -303,12 +311,12 @@ class Inodes(object):
if entry.has_ref(False):
# Only necessary if the kernel has previously done a lookup on this
# inode and hasn't yet forgotten about it.
- llfuse.invalidate_entry(entry.inode, name.encode(self.encoding))
+ llfuse.invalidate_entry(entry.inode, native(name.encode(self.encoding)))
def clear(self):
self.inode_cache.clear()
- for k,v in self._entries.items():
+ for k,v in viewitems(self._entries):
try:
v.finalize()
except Exception as e:
@@ -526,7 +534,7 @@ class Operations(llfuse.Operations):
entry.st_size = e.size()
entry.st_blksize = 512
- entry.st_blocks = (entry.st_size/512)+1
+ entry.st_blocks = (entry.st_size // 512) + 1
if hasattr(entry, 'st_atime_ns'):
# llfuse >= 0.42
entry.st_atime_ns = int(e.atime() * 1000000000)
@@ -567,7 +575,7 @@ class Operations(llfuse.Operations):
@lookup_time.time()
@catch_exceptions
def lookup(self, parent_inode, name, ctx=None):
- name = unicode(name, self.inodes.encoding)
+ name = str(name, self.inodes.encoding)
inode = None
if name == '.':
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index f174d1bb0..14b1327d9 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -2,6 +2,9 @@
#
# SPDX-License-Identifier: AGPL-3.0
+from builtins import range
+from builtins import str
+from builtins import object
import argparse
import arvados
import daemon
@@ -134,12 +137,12 @@ class Mount(object):
if self.args.replace:
unmount(path=self.args.mountpoint,
timeout=self.args.unmount_timeout)
- llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+ llfuse.init(self.operations, self.args.mountpoint.encode('utf-8'), self._fuse_options())
if self.daemon:
daemon.DaemonContext(
working_directory=os.path.dirname(self.args.mountpoint),
- files_preserve=range(
- 3, resource.getrlimit(resource.RLIMIT_NOFILE)[1])
+ files_preserve=list(range(
+ 3, resource.getrlimit(resource.RLIMIT_NOFILE)[1]))
).open()
if self.listen_for_events and not self.args.disable_event_listening:
self.operations.listen_for_events()
diff --git a/services/fuse/arvados_fuse/crunchstat.py b/services/fuse/arvados_fuse/crunchstat.py
index e99573752..0cb585a6f 100644
--- a/services/fuse/arvados_fuse/crunchstat.py
+++ b/services/fuse/arvados_fuse/crunchstat.py
@@ -2,13 +2,15 @@
#
# SPDX-License-Identifier: AGPL-3.0
+from builtins import str
+from builtins import object
import sys
import time
from collections import namedtuple
Stat = namedtuple("Stat", ['name', 'get'])
-class StatWriter(object):
+class StatWriter(object):
def __init__(self, prefix, interval, stats):
self.prefix = prefix
self.interval = interval
@@ -40,7 +42,7 @@ class StatWriter(object):
def statlogger(interval, keep, ops):
calls = StatWriter("keepcalls", interval, [
- Stat("put", keep.put_counter.get),
+ Stat("put", keep.put_counter.get),
Stat("get", keep.get_counter.get)
])
net = StatWriter("net:keep0", interval, [
@@ -48,15 +50,15 @@ def statlogger(interval, keep, ops):
Stat("rx", keep.download_counter.get)
])
cache = StatWriter("keepcache", interval, [
- Stat("hit", keep.hits_counter.get),
+ Stat("hit", keep.hits_counter.get),
Stat("miss", keep.misses_counter.get)
])
fuseops = StatWriter("fuseops", interval, [
- Stat("write", ops.write_ops_counter.get),
+ Stat("write", ops.write_ops_counter.get),
Stat("read", ops.read_ops_counter.get)
])
fusetimes = []
- for cur_op in ops.metric_op_names():
+ for cur_op in ops.metric_op_names():
name = "fuseop:{0}".format(cur_op)
fusetimes.append(StatWriter(name, interval, [
Stat("count", ops.metric_count_func(cur_op)),
diff --git a/services/fuse/arvados_fuse/fresh.py b/services/fuse/arvados_fuse/fresh.py
index 2e7a2a818..53214ee94 100644
--- a/services/fuse/arvados_fuse/fresh.py
+++ b/services/fuse/arvados_fuse/fresh.py
@@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: AGPL-3.0
+from builtins import object
import time
import ciso8601
import calendar
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 75fa2762d..9d53df950 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -3,6 +3,9 @@
# SPDX-License-Identifier: AGPL-3.0
from __future__ import absolute_import
+from __future__ import division
+from future.utils import viewitems
+from builtins import dict
import logging
import re
import time
@@ -164,7 +167,7 @@ class Directory(FreshBase):
def in_use(self):
if super(Directory, self).in_use():
return True
- for v in self._entries.itervalues():
+ for v in self._entries.values():
if v.in_use():
return True
return False
@@ -172,7 +175,7 @@ class Directory(FreshBase):
def has_ref(self, only_children):
if super(Directory, self).has_ref(only_children):
return True
- for v in self._entries.itervalues():
+ for v in self._entries.values():
if v.has_ref(False):
return True
return False
@@ -194,7 +197,7 @@ class Directory(FreshBase):
# Find self on the parent in order to invalidate this path.
# Calling the public items() method might trigger a refresh,
# which we definitely don't want, so read the internal dict directly.
- for k,v in parent._entries.items():
+ for k,v in viewitems(parent._entries):
if v is self:
self.inodes.invalidate_entry(parent, k)
break
@@ -283,7 +286,7 @@ class CollectionDirectoryBase(Directory):
def populate(self, mtime):
self._mtime = mtime
self.collection.subscribe(self.on_event)
- for entry, item in self.collection.items():
+ for entry, item in viewitems(self.collection):
self.new_entry(entry, item, self.mtime())
def writable(self):
@@ -360,7 +363,7 @@ class CollectionDirectory(CollectionDirectoryBase):
self.collection_record = None
self._poll = True
try:
- self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2)/2)
+ self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2) // 2)
except:
_logger.debug("Error getting blobSignatureTtl from discovery document: %s", sys.exc_info()[0])
self._poll_time = 60*60
@@ -1117,7 +1120,7 @@ class SharedDirectory(Directory):
# end with llfuse.lock_released, re-acquire lock
- self.merge(contents.items(),
+ self.merge(viewitems(contents),
lambda i: i[0],
lambda a, i: a.uuid() == i[1]['uuid'],
lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time))
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 2e6484cab..4253a09f6 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -2,6 +2,8 @@
#
# SPDX-License-Identifier: Apache-2.0
+from builtins import str
+from builtins import next
import subprocess
import time
import os
diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index cbc9cb23e..28f0c5417 100644
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -39,8 +39,7 @@ setup(name='arvados_fuse',
],
install_requires=[
'arvados-python-client >= 0.1.20151118035730',
- # llfuse 1.3.4 fails to install via pip
- 'llfuse >=1.2, <1.3.4',
+ 'llfuse >= 1.3.6',
'python-daemon',
'ciso8601 >= 2.0.0',
'setuptools',
commit 8dfb0e3f8ca8154f4c16ab26e03aa6c6b114b5ad
Author: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
Date: Mon Jun 17 15:14:35 2019 -0400
14965: Updates fuse code with futurize stage1
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index b37ef695b..72ccdb5f9 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -49,6 +49,7 @@ an object that is live in the inode cache, that object is immediately updated.
"""
+from __future__ import absolute_import
import os
import sys
import llfuse
@@ -88,8 +89,8 @@ else:
LLFUSE_VERSION_0 = llfuse.__version__.startswith('0')
-from fusedir import sanitize_filename, Directory, CollectionDirectory, TmpCollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
-from fusefile import StringFile, FuseArvadosFile
+from .fusedir import sanitize_filename, Directory, CollectionDirectory, TmpCollectionDirectory, MagicDirectory, TagsDirectory, ProjectDirectory, SharedDirectory, CollectionDirectoryBase
+from .fusefile import StringFile, FuseArvadosFile
_logger = logging.getLogger('arvados.arvados_fuse')
@@ -414,7 +415,7 @@ class Operations(llfuse.Operations):
self.initlock.set()
def metric_samples(self):
- return self.fuse_time.collect()[0].samples
+ return self.fuse_time.collect()[0].samples
def metric_op_names(self):
ops = []
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 2d58012fa..75fa2762d 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: AGPL-3.0
+from __future__ import absolute_import
import logging
import re
import time
@@ -14,8 +15,8 @@ from apiclient import errors as apiclient_errors
import errno
import time
-from fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
-from fresh import FreshBase, convertTime, use_counter, check_update
+from .fusefile import StringFile, ObjectFile, FuncToJSONFile, FuseArvadosFile
+from .fresh import FreshBase, convertTime, use_counter, check_update
import arvados.collection
from arvados.util import portable_data_hash_pattern, uuid_pattern, collection_uuid_pattern, group_uuid_pattern, user_uuid_pattern, link_uuid_pattern
diff --git a/services/fuse/arvados_fuse/fusefile.py b/services/fuse/arvados_fuse/fusefile.py
index cedb4fb45..1b12d7e41 100644
--- a/services/fuse/arvados_fuse/fusefile.py
+++ b/services/fuse/arvados_fuse/fusefile.py
@@ -2,13 +2,14 @@
#
# SPDX-License-Identifier: AGPL-3.0
+from __future__ import absolute_import
import json
import llfuse
import logging
import re
import time
-from fresh import FreshBase, convertTime
+from .fresh import FreshBase, convertTime
_logger = logging.getLogger('arvados.arvados_fuse')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list