[arvados] updated: 2.7.0-35-g318684addb
git repository hosting
git at public.arvados.org
Wed Dec 6 16:48:56 UTC 2023
Summary of changes:
services/api/app/models/link.rb | 11 ++++--
services/api/app/models/user.rb | 8 ++--
services/api/lib/update_permissions.rb | 67 +++++++++++++++++++++++++++++++-
services/fuse/arvados_fuse/command.py | 31 ++++++++++++---
services/fuse/tests/test_command_args.py | 45 +++++++++++++++++++++
5 files changed, 147 insertions(+), 15 deletions(-)
via 318684addb994eae446bdc9907bd5d44c8f58e2a (commit)
via 29e6998044b508d5a221c06ab6b784779cd299c6 (commit)
from d53b38d1fa91725b10c7359278e52a7f2f6a15db (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 318684addb994eae446bdc9907bd5d44c8f58e2a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Dec 6 09:31:40 2023 -0500
Merge branch '21223-arv-mount-nofile' refs #21223
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/fuse/arvados_fuse/command.py b/services/fuse/arvados_fuse/command.py
index c2242eb2bf..9c607c7f0c 100644
--- a/services/fuse/arvados_fuse/command.py
+++ b/services/fuse/arvados_fuse/command.py
@@ -134,21 +134,40 @@ class Mount(object):
if self.args.logfile:
self.args.logfile = os.path.realpath(self.args.logfile)
+ try:
+ self._setup_logging()
+ except Exception as e:
+ self.logger.exception("exception during setup: %s", e)
+ exit(1)
+
try:
nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
- if nofile_limit[0] < 10240:
- resource.setrlimit(resource.RLIMIT_NOFILE, (min(10240, nofile_limit[1]), nofile_limit[1]))
+
+ minlimit = 10240
+ if self.args.file_cache:
+ # Adjust the file handle limit so it can meet
+ # the desired cache size. Multiply by 8 because the
+ # number of 64 MiB cache slots that keepclient
+ # allocates is RLIMIT_NOFILE / 8
+ minlimit = int((self.args.file_cache/(64*1024*1024)) * 8)
+
+ if nofile_limit[0] < minlimit:
+ resource.setrlimit(resource.RLIMIT_NOFILE, (min(minlimit, nofile_limit[1]), nofile_limit[1]))
+
+ if minlimit > nofile_limit[1]:
+ self.logger.warning("file handles required to meet --file-cache (%s) exceeds hard file handle limit (%s), cache size will be smaller than requested", minlimit, nofile_limit[1])
+
except Exception as e:
- self.logger.warning("arv-mount: unable to adjust file handle limit: %s", e)
+ self.logger.warning("unable to adjust file handle limit: %s", e)
- self.logger.debug("arv-mount: file handle limit is %s", resource.getrlimit(resource.RLIMIT_NOFILE))
+ nofile_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+ self.logger.info("file cache capped at %s bytes or less based on available disk (RLIMIT_NOFILE is %s)", ((nofile_limit[0]//8)*64*1024*1024), nofile_limit)
try:
- self._setup_logging()
self._setup_api()
self._setup_mount()
except Exception as e:
- self.logger.exception("arv-mount: exception during setup: %s", e)
+ self.logger.exception("exception during setup: %s", e)
exit(1)
def __enter__(self):
diff --git a/services/fuse/tests/test_command_args.py b/services/fuse/tests/test_command_args.py
index 600bb0fe22..b08ab19335 100644
--- a/services/fuse/tests/test_command_args.py
+++ b/services/fuse/tests/test_command_args.py
@@ -20,6 +20,7 @@ from . import run_test_server
import sys
import tempfile
import unittest
+import resource
def noexit(func):
"""If argparse or arvados_fuse tries to exit, fail the test instead"""
@@ -261,6 +262,50 @@ class MountArgsTest(unittest.TestCase):
'--foreground', self.mntdir])
arvados_fuse.command.Mount(args)
+ @noexit
+ @mock.patch('resource.setrlimit')
+ @mock.patch('resource.getrlimit')
+ def test_default_file_cache(self, getrlimit, setrlimit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', self.mntdir])
+ self.assertEqual(args.mode, None)
+ getrlimit.return_value = (1024, 1048576)
+ self.mnt = arvados_fuse.command.Mount(args)
+ setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (10240, 1048576))
+
+ @noexit
+ @mock.patch('resource.setrlimit')
+ @mock.patch('resource.getrlimit')
+ def test_small_file_cache(self, getrlimit, setrlimit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', '--file-cache=256000000', self.mntdir])
+ self.assertEqual(args.mode, None)
+ getrlimit.return_value = (1024, 1048576)
+ self.mnt = arvados_fuse.command.Mount(args)
+ setrlimit.assert_not_called()
+
+ @noexit
+ @mock.patch('resource.setrlimit')
+ @mock.patch('resource.getrlimit')
+ def test_large_file_cache(self, getrlimit, setrlimit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', '--file-cache=256000000000', self.mntdir])
+ self.assertEqual(args.mode, None)
+ getrlimit.return_value = (1024, 1048576)
+ self.mnt = arvados_fuse.command.Mount(args)
+ setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (30517, 1048576))
+
+ @noexit
+ @mock.patch('resource.setrlimit')
+ @mock.patch('resource.getrlimit')
+ def test_file_cache_hard_limit(self, getrlimit, setrlimit):
+ args = arvados_fuse.command.ArgumentParser().parse_args([
+ '--foreground', '--file-cache=256000000000', self.mntdir])
+ self.assertEqual(args.mode, None)
+ getrlimit.return_value = (1024, 2048)
+ self.mnt = arvados_fuse.command.Mount(args)
+ setrlimit.assert_called_with(resource.RLIMIT_NOFILE, (2048, 2048))
+
class MountErrorTest(unittest.TestCase):
def setUp(self):
self.mntdir = tempfile.mkdtemp()
commit 29e6998044b508d5a221c06ab6b784779cd299c6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Dec 5 16:37:14 2023 -0500
Merge branch '21160-user-activation' refs #21160
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/api/app/models/link.rb b/services/api/app/models/link.rb
index 4d4c2832bb..2eb6b88a0c 100644
--- a/services/api/app/models/link.rb
+++ b/services/api/app/models/link.rb
@@ -17,11 +17,12 @@ class Link < ArvadosModel
before_update :apply_max_overlapping_permissions
before_create :apply_max_overlapping_permissions
after_update :delete_overlapping_permissions
- after_update :call_update_permissions
- after_create :call_update_permissions
+ after_update :call_update_permissions, :if => Proc.new { @need_update_permissions }
+ after_create :call_update_permissions, :if => Proc.new { @need_update_permissions }
before_destroy :clear_permissions
after_destroy :delete_overlapping_permissions
after_destroy :check_permissions
+ before_save :check_need_update_permissions
api_accessible :user, extend: :common do |t|
t.add :tail_uuid
@@ -189,11 +190,13 @@ class Link < ArvadosModel
'can_manage' => 3,
}
+ def check_need_update_permissions
+ @need_update_permissions = self.link_class == 'permission' && (name != name_was || new_record?)
+ end
+
def call_update_permissions
- if self.link_class == 'permission'
update_permissions tail_uuid, head_uuid, PERM_LEVEL[name], self.uuid
current_user.forget_cached_group_perms
- end
end
def clear_permissions
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index d4d4a82458..c73f31a99d 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -512,11 +512,11 @@ SELECT target_uuid, perm_level
update_attributes!(redirect_to_user_uuid: new_user.uuid, username: nil)
end
skip_check_permissions_against_full_refresh do
- update_permissions self.uuid, self.uuid, CAN_MANAGE_PERM
- update_permissions new_user.uuid, new_user.uuid, CAN_MANAGE_PERM
- update_permissions new_user.owner_uuid, new_user.uuid, CAN_MANAGE_PERM
+ update_permissions self.uuid, self.uuid, CAN_MANAGE_PERM, nil, true
+ update_permissions new_user.uuid, new_user.uuid, CAN_MANAGE_PERM, nil, true
+ update_permissions new_user.owner_uuid, new_user.uuid, CAN_MANAGE_PERM, nil, true
end
- update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
+ update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM, nil, true
end
end
diff --git a/services/api/lib/update_permissions.rb b/services/api/lib/update_permissions.rb
index 5c8072b48a..7e946b4311 100644
--- a/services/api/lib/update_permissions.rb
+++ b/services/api/lib/update_permissions.rb
@@ -7,7 +7,7 @@ require '20200501150153_permission_table_constants'
REVOKE_PERM = 0
CAN_MANAGE_PERM = 3
-def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
+def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil, update_all_users=false
return if Thread.current[:suppress_update_permissions]
#
@@ -109,6 +109,70 @@ def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
# Disable merge join for just this query (also local for this transaction), then reenable it.
ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to false;"
+ if perm_origin_uuid[5..11] == '-tpzed-' && !update_all_users
+ # Modifying permission granted to a user, recompute the all permissions for that user
+
+ ActiveRecord::Base.connection.exec_query %{
+with origin_user_perms as (
+ select pq.origin_uuid as user_uuid, target_uuid, pq.val, pq.traverse_owned from (
+ #{PERM_QUERY_TEMPLATE % {:base_case => %{
+ select '#{perm_origin_uuid}'::varchar(255), '#{perm_origin_uuid}'::varchar(255), 3, true, true
+ where exists (select uuid from users where uuid='#{perm_origin_uuid}')
+},
+:edge_perm => %{
+case (edges.edge_id = '#{edge_id}')
+ when true then #{perm_level}
+ else edges.val
+ end
+}
+} }) as pq),
+
+/*
+ Because users always have permission on themselves, this
+ query also makes sure those permission rows are always
+ returned.
+*/
+temptable_perms as (
+ select * from origin_user_perms
+ union all
+ select target_uuid as user_uuid, target_uuid, 3, true
+ from origin_user_perms
+ where origin_user_perms.target_uuid like '_____-tpzed-_______________' and
+ origin_user_perms.target_uuid != '#{perm_origin_uuid}'
+),
+
+/*
+ Now that we have recomputed a set of permissions, delete any
+ rows from the materialized_permissions table where (target_uuid,
+ user_uuid) is not present or has perm_level=0 in the recomputed
+ set.
+*/
+delete_rows as (
+ delete from #{PERMISSION_VIEW} where
+ user_uuid='#{perm_origin_uuid}' and
+ not exists (select 1 from temptable_perms
+ where target_uuid=#{PERMISSION_VIEW}.target_uuid and
+ user_uuid='#{perm_origin_uuid}' and
+ val>0)
+)
+
+/*
+ Now insert-or-update permissions in the recomputed set. The
+ WHERE clause is important to avoid redundantly updating rows
+ that haven't actually changed.
+*/
+insert into #{PERMISSION_VIEW} (user_uuid, target_uuid, perm_level, traverse_owned)
+ select user_uuid, target_uuid, val as perm_level, traverse_owned from temptable_perms where val>0
+on conflict (user_uuid, target_uuid) do update
+set perm_level=EXCLUDED.perm_level, traverse_owned=EXCLUDED.traverse_owned
+where #{PERMISSION_VIEW}.user_uuid=EXCLUDED.user_uuid and
+ #{PERMISSION_VIEW}.target_uuid=EXCLUDED.target_uuid and
+ (#{PERMISSION_VIEW}.perm_level != EXCLUDED.perm_level or
+ #{PERMISSION_VIEW}.traverse_owned != EXCLUDED.traverse_owned);
+
+}
+ else
+ # Modifying permission granted to a group, recompute permissions for everything accessible through that group
ActiveRecord::Base.connection.exec_query %{
with temptable_perms as (
select * from compute_permission_subgraph($1, $2, $3, $4)),
@@ -147,6 +211,7 @@ where #{PERMISSION_VIEW}.user_uuid=EXCLUDED.user_uuid and
[nil, starting_uuid],
[nil, perm_level],
[nil, edge_id]]
+ end
if perm_level>0
check_permissions_against_full_refresh
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list