[arvados] updated: 2.1.0-3092-g2cddd3c4e
git repository hosting
git at public.arvados.org
Tue Nov 22 22:02:29 UTC 2022
Summary of changes:
.gitignore | 4 -
apps/workbench/Gemfile.lock | 4 +-
apps/workbench/app/helpers/version_helper.rb | 2 +-
apps/workbench/app/views/users/_tables.html.erb | 2 +-
.../test/integration/report_issue_test.rb | 2 +-
build/package-build-dockerfiles/centos7/Dockerfile | 2 +-
build/run-build-packages-one-target.sh | 1 -
build/run-build-packages.sh | 5 -
build/run-library.sh | 34 ---
build/run-tests.sh | 24 +-
doc/_config.yml | 5 +-
doc/admin/diagnostics.html.textile.liquid | 83 ++++++
doc/admin/federation.html.textile.liquid | 12 +-
doc/admin/health-checks.html.textile.liquid | 37 ++-
.../maintenance-and-upgrading.html.textile.liquid | 7 +-
doc/admin/upgrading.html.textile.liquid | 22 +-
doc/api/methods/groups.html.textile.liquid | 2 +-
doc/api/projects.html.textile.liquid | 2 +-
doc/api/properties.html.textile.liquid | 50 ++++
doc/architecture/federation.html.textile.liquid | 6 +-
...configure-s3-object-storage.html.textile.liquid | 5 +-
.../install-dispatch-cloud.html.textile.liquid | 38 +--
.../install-dispatch.html.textile.liquid | 25 ++
.../crunch2-slurm/install-test.html.textile.liquid | 30 +-
doc/sdk/index.html.textile.liquid | 1 -
doc/sdk/perl/example.html.textile.liquid | 77 -----
doc/sdk/perl/index.html.textile.liquid | 66 -----
lib/config/config.default.yml | 42 ++-
lib/config/export.go | 3 +-
lib/controller/dblock/dblock.go | 77 +++--
lib/controller/dblock/dblock_test.go | 91 ++++++
lib/controller/federation.go | 4 +-
lib/controller/federation/conn.go | 20 ++
lib/controller/federation/generate.go | 2 +-
lib/controller/federation/generated.go | 41 +++
lib/controller/federation/list.go | 32 +--
lib/controller/federation/login_test.go | 12 +-
lib/controller/handler.go | 41 +--
lib/controller/handler_test.go | 41 ++-
lib/controller/localdb/conn.go | 21 ++
lib/controller/localdb/login.go | 37 +++
lib/controller/localdb/login_oidc.go | 3 +
lib/controller/localdb/login_oidc_test.go | 86 +++++-
lib/controller/localdb/login_testuser_test.go | 3 +-
lib/controller/localdb/logout.go | 2 +
lib/controller/router/router.go | 35 +++
lib/controller/rpc/conn.go | 35 +++
lib/controller/trash.go | 65 ++++-
lib/crunchrun/crunchrun.go | 9 +
lib/crunchstat/crunchstat.go | 88 +++++-
lib/crunchstat/crunchstat_test.go | 103 ++++---
lib/ctrlctx/db.go | 31 +++
lib/diagnostics/cmd.go | 89 ++++--
lib/dispatchcloud/dispatcher.go | 6 +
lib/dispatchcloud/dispatcher_test.go | 25 +-
lib/dispatchcloud/node_size.go | 7 +-
lib/dispatchcloud/node_size_test.go | 30 +-
lib/install/deps.go | 18 +-
lib/lsf/dispatch.go | 6 +
lib/lsf/dispatch_test.go | 1 +
lib/pam/testclient.go | 2 +-
sdk/cwl/arvados_cwl/__init__.py | 18 ++
sdk/cwl/arvados_cwl/arvcontainer.py | 26 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 93 ++++++-
sdk/cwl/arvados_cwl/context.py | 3 +
sdk/cwl/arvados_cwl/executor.py | 31 ++-
sdk/cwl/arvados_cwl/fsaccess.py | 9 +-
sdk/cwl/arvados_cwl/http.py | 106 +++++--
sdk/cwl/arvados_cwl/pathmapper.py | 15 +-
sdk/cwl/arvados_cwl/runner.py | 51 ++--
sdk/cwl/setup.py | 5 +-
sdk/cwl/tests/19678-name-id.cwl | 26 ++
sdk/cwl/tests/arvados-tests.yml | 10 +
.../collection_per_tool_wrapper.cwl | 35 +++
sdk/cwl/tests/test_container.py | 79 +++++-
sdk/cwl/tests/test_http.py | 165 ++++++++++-
sdk/cwl/tests/test_submit.py | 232 +++++++++------
...upload_packed.cwl => expect_upload_wrapper.cwl} | 51 ++--
...acked.cwl => expect_upload_wrapper_altname.cwl} | 51 ++--
sdk/dev-jobs.dockerfile | 6 +
sdk/go/arvados/api.go | 10 +
sdk/go/arvados/client.go | 12 +-
sdk/go/arvados/config.go | 22 +-
sdk/go/arvados/duration.go | 8 +
sdk/go/arvados/duration_test.go | 10 +
sdk/go/arvados/fs_collection.go | 6 +-
sdk/go/arvados/log.go | 7 +-
sdk/go/arvados/vocabulary.go | 2 +
sdk/go/arvados/vocabulary_test.go | 2 +
sdk/go/arvadostest/api.go | 20 ++
sdk/go/health/aggregator.go | 4 +-
sdk/perl/.gitignore | 1 -
sdk/perl/Makefile.PL | 18 --
sdk/perl/lib/Arvados.pm | 165 -----------
sdk/perl/lib/Arvados/Request.pm | 104 -------
sdk/perl/lib/Arvados/ResourceAccessor.pm | 25 --
sdk/perl/lib/Arvados/ResourceMethod.pm | 124 ---------
sdk/perl/lib/Arvados/ResourceProxy.pm | 61 ----
sdk/perl/lib/Arvados/ResourceProxyList.pm | 24 --
sdk/python/README.rst | 2 +-
sdk/python/arvados/collection.py | 5 +
sdk/python/arvados/diskcache.py | 56 +++-
sdk/python/arvados/keep.py | 17 +-
sdk/python/arvados/retry.py | 148 +++++++---
sdk/python/setup.py | 7 +-
sdk/python/tests/arvados_testutil.py | 12 +
sdk/python/tests/run_test_server.py | 3 +
sdk/python/tests/test_keep_client.py | 310 +++++++++++++++++----
sdk/ruby/lib/arvados/keep.rb | 2 +-
services/api/Gemfile.lock | 2 +-
services/api/app/models/api_client.rb | 9 +-
services/api/app/models/arvados_model.rb | 9 +-
services/api/app/models/user.rb | 23 +-
.../api/lib/tasks/delete_old_container_logs.rake | 6 +-
.../arvados/v1/collections_controller_test.rb | 18 ++
services/api/test/integration/users_test.rb | 18 ++
.../test/tasks/delete_old_container_logs_test.rb | 54 ----
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 14 +-
services/crunchstat/crunchstat.go | 44 ++-
services/fuse/README.rst | 2 +-
services/fuse/tests/mount_test_base.py | 4 +-
services/keep-balance/balance.go | 17 +-
services/keep-balance/balance_run_test.go | 20 +-
services/keep-balance/integration_test.go | 3 +-
services/keep-balance/main.go | 2 +-
services/keep-balance/server.go | 29 +-
services/keepstore/s3_volume.go | 7 +-
services/keepstore/s3aws_volume.go | 16 +-
tools/arvbox/lib/arvbox/docker/common.sh | 5 +
129 files changed, 2665 insertions(+), 1492 deletions(-)
create mode 100644 doc/admin/diagnostics.html.textile.liquid
create mode 100644 doc/api/properties.html.textile.liquid
delete mode 100644 doc/sdk/perl/example.html.textile.liquid
delete mode 100644 doc/sdk/perl/index.html.textile.liquid
create mode 100644 lib/controller/dblock/dblock_test.go
create mode 100644 sdk/cwl/tests/19678-name-id.cwl
create mode 100644 sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl
copy sdk/cwl/tests/wf/{expect_upload_packed.cwl => expect_upload_wrapper.cwl} (69%)
copy sdk/cwl/tests/wf/{expect_upload_packed.cwl => expect_upload_wrapper_altname.cwl} (69%)
delete mode 100644 sdk/perl/.gitignore
delete mode 100644 sdk/perl/Makefile.PL
delete mode 100644 sdk/perl/lib/Arvados.pm
delete mode 100644 sdk/perl/lib/Arvados/Request.pm
delete mode 100644 sdk/perl/lib/Arvados/ResourceAccessor.pm
delete mode 100644 sdk/perl/lib/Arvados/ResourceMethod.pm
delete mode 100644 sdk/perl/lib/Arvados/ResourceProxy.pm
delete mode 100644 sdk/perl/lib/Arvados/ResourceProxyList.pm
delete mode 100644 services/api/test/tasks/delete_old_container_logs_test.rb
via 2cddd3c4e531e37c6d960c08da91a552bf75a1cc (commit)
via 2693a197ea37490a891c54bbb279937a7b4e897c (commit)
via 09cbdc3074b3f1e69c9c537875146f6da0a6ed8f (commit)
via 2b7d05cdf4e054024607b859bd6fb41e04855bfa (commit)
via 1b75afdd0c278d34c1a99ed41814eb5119a254a9 (commit)
via 9b976ee4d3e0c58eaa81f28f13dc4d112dbf804b (commit)
via 2c2a0b36cd0ec30d755ac2b4dd4f01e67ba058f9 (commit)
via bb406742b50b90d3796ec86c4c7f806004cc3933 (commit)
via 273d4dda75bad4b1ba18bc3616f16082b95c0467 (commit)
via 37180569bd74f4cbe1797898436634a5f686d832 (commit)
via 5a33764685ef7fa9578b255e78ad2ed77abc1ded (commit)
via 5e3a6c32deb39429cd85898b63562b5ae49a9524 (commit)
via 531fd553a1b83c546066c1d2a2619f86e17b6d20 (commit)
via 75d0bce4f378efc488b67b178ace50301f9ad8ff (commit)
via d760c01e23eb731235aee6e47c5a4b0aeed1d275 (commit)
via 9e988394278b9c0c072c27107b67669875b8fca7 (commit)
via f080dcbd0d27b7e830a1c0c544f049212ed79e61 (commit)
via 699bc133ef79768575e811ec058653cad521ac26 (commit)
via 91fbb52cbe2637e1a86bc0f615545f5e909f7317 (commit)
via 0515919a3da565c7bf5087eca38f47bc4422c260 (commit)
via 7a233da0326bee6f4f6448528707c0cf8925d2ea (commit)
via 4d0ab09acfd9aed9c4b2cf6c1a85a9538e9c969d (commit)
via 924f8f6c13c06afc8a83168929b249e0e8fa7d18 (commit)
via 8a0527fcb8948720a873aa35fd1800076c3859a2 (commit)
via 1c40d138c5de3ed5e27437edc5cbb6b84da29fc3 (commit)
via 18e79caebbda5a4350462a2b22e5af36f8e4b699 (commit)
via 8851415af9ae0bd1a6362f1877e34d6c5ff0c46f (commit)
via c31c6528cac695bc86d4244516e07ea316cac979 (commit)
via 878ec61b535740ebeb40dcc4a330557698c66417 (commit)
via 35a14f916699ad73c65b07973afa3dfac3ac9a98 (commit)
via 5b09754d58268e0d98cd144fcc1847c606f5029e (commit)
via c6428570be58b01ce80c257adce06114cea1d88f (commit)
via 8a6ca096b41dcbbb9854abddae03997afd2ddf06 (commit)
via 34aa58ae5b28ddf0ef5f37c70e761c763bf6a431 (commit)
via cc686b375dc80b5e8e7d471b4bb663fb879f221b (commit)
via 34f3229a26dcc9d9b6c94e207eb8f58bb6555acb (commit)
via 12aba9fd35f8813ff30cd6e892574505a24fcea3 (commit)
via 1c5a32c1279fc04c1c7e4fb9d8e1e23190b408b9 (commit)
via fb16ecd1f19f543341d5cf1e9ce35bf5ac68a19a (commit)
via 89dbe5bbfd6802022b1da67033cf8ea429f7aec1 (commit)
via 0b39c68ee38afbfec9f7d6d082a52cc2681edbea (commit)
via e49a01cb75572deacf6878ad53d7965ea1869d2b (commit)
via ee532d8ffb857bea75712fd93d450eaef161521b (commit)
via 988db9dc5af7e2c06e2a6106d2ce32275e3da7a4 (commit)
via 88d70b6d852d2347a958b7e76de3f47df2699e70 (commit)
via c80603fb6b953bedc60f1deda274bfa4d437b597 (commit)
via fd6802c8181b172fb7c500a01d1343836d6a6809 (commit)
via 686f881614885a6566a566105539934ead80466f (commit)
via 43bcf5878e33e31eb6d28bbb868696ec467b1008 (commit)
via 9cd2fc2cd84000e706d73d1ff8316ce46b1be54d (commit)
via 9f39acdc1d65d8b2714ffccd51a0311e6df9ec4a (commit)
via 5ee9b22d0777d7ef42fa6acc05b666fd30e1ab2b (commit)
via 29a6508d68cc93c376a8ba28559b7404933b5546 (commit)
via 5b8f61eb57fc511b4110bb28c627f5761ecf349e (commit)
via bd25fd95831117fe70864f1d03a9504b68c85ba8 (commit)
via f68ba06c5e85b748f13f723373e1fbe79fa8e563 (commit)
via e2149a153e3432c24320b7574934a5f1f4040df7 (commit)
via 710dc7f830f65232389cf191028edfdfe4cefe77 (commit)
via 11809f6cf62877291c06ab763ea764772a468a38 (commit)
via ff779b9f1306420462ec5898424188c88bb927a9 (commit)
via 266512db4befdfa2bf7562e0e10a5c86ffa15ec9 (commit)
via 21c7799069b13380913d0dfdd9ecc441d3af7298 (commit)
via 578c505d74e0e4daf680c5c39fb1619bb073a592 (commit)
via d4fb93d5adb02b5508e004fb328d532ce5b9f4bd (commit)
via 00087e9458f9ede8d4c2bb9621d782b8c2fadf29 (commit)
via 8c97b429803908fe13052dc788dfc313001c83b2 (commit)
via c0626f50e9ff14d008ae93de48a04318dd93eef2 (commit)
via d0f3483739a0140802374e6a9f5d0ab5972bd951 (commit)
via 78a6558918cbc38d01907682d480602a1351fa97 (commit)
via d6a05be901e501bd10c00702fc8540dc1efba68a (commit)
via d825b0330a1b51d8ccbb25e7dc7d9aac26e781e0 (commit)
via 9828e9218084856240fdeafa2d388d8bf322e655 (commit)
via 9e764431dc5e3b42eba5f5495b008625bf11d777 (commit)
via f6c7741002f3855663bf9ac4aef4cbf86f3b5d51 (commit)
via 3070a6df84012867058b93c3941eb7feec353cc5 (commit)
via bb506312949465d4503d7f0b4434cfcd435cda0a (commit)
via 4e1c08989f8f44bf5e253cd74ad7af5898c4ddd8 (commit)
via 07baa0ed049746514495d1648c1aef0c40545141 (commit)
via 4529d84afb3549ccb4ae9005a8f64f558c2bbe5c (commit)
via b4b304a3a73a147cb85e9e4ec171d7f350620d30 (commit)
via e2267bd99209651c61425f335230e515421b2ef4 (commit)
via a5f492d001f158a01abfd79b65c5bf41708da7f9 (commit)
via 15043a6825ecd62ccb2272025384474a235b30cc (commit)
via 8a8f27d60f97047483072539837785ea917b286d (commit)
via 08bf53a3396ab74e805d468ccbb9c0cea86a3d5a (commit)
via 65b12213f740b117fb14822bce0dbb415257c355 (commit)
via e33a15001d7a94a805a5d0d4c77544d959974193 (commit)
via aa9507e1633819259794ea4d6cf391dc88621dac (commit)
via dee23e534ab1c84fbe2e34730c0327989cc263ac (commit)
via d5df19fdff62724f9faeb3bee17201363071bf9e (commit)
via b6da0bcc634615e8db9d4546a33b97bb69583afc (commit)
via 8860ada0a6afd6adb2b26d5b0ab161bfe66c6019 (commit)
via bef451193ffe6c7239c545ed1c93769b50c9e0a8 (commit)
via e55bd0d3b54494061d54853c4b613ad680ffb6a9 (commit)
via 533ee50c604e3900c390b77fafac1455b28a15d0 (commit)
via 53d3f0bd7ed8ddb597f8615d87ec4691cce62f51 (commit)
via ca69f0bebc31124d9b61cec4b790d45a94bff379 (commit)
via c634fdaf04b09ccd64d418c37dddc1eb7ff1c490 (commit)
via 9b1154beb90c7d174053e7487a3f9b7dc937dcf8 (commit)
via 368bab079ea02d274da28ed0d233d87140a139af (commit)
via bd1e6372a91fe31dd0a5d9b112b1026013f9a887 (commit)
via f6ea5e67a71df12bd665de083d6fdc35e64b6b27 (commit)
via 28ccc316a2eabe38e81627484e3109d6be932ac8 (commit)
via 0596129229750c593066e414d9315f643585bc3e (commit)
via f3203e42412d5b5216b2c70caae47b73b712d18c (commit)
via c235da62cb5425e3906f58f398e84e70fec9ae87 (commit)
via 14f6625379992bc3ee054ad419095b476a1c4284 (commit)
via b89ab7f9270acfabe9139d14d7071cf117b39bd4 (commit)
via 7d2ec2603a42f6cae5552ab6dfd2a37a01e43ea8 (commit)
via 99d3381ef145368617ecf92532201d79c9bb0ea7 (commit)
via 5c0d99ac1c8371910676b6375282a4fcfb5d213d (commit)
via 3282886a07b17773d4f8ad47209978e50bed7e92 (commit)
via cbf20e690f71440eb725e7280eb14b827032d9a4 (commit)
via 1f73a01c0d895811228025fefd430d9c40590175 (commit)
via 67f1f536ad087ca9976f093e1f3477cec57f4985 (commit)
via 61ee61895a33008c70e5a294407cf55efc19622c (commit)
via 58afa4202a5ff084fe7eee1a5274e02071858ba5 (commit)
via 451fd7eccd17e07555b3544d8ed1dc85f5d67875 (commit)
via a7caca5452e7ece2ea3f6efd3ab4bb992c1f31fb (commit)
via 88f3630af39229d3d66621f98c775823313c21f8 (commit)
via 210f068ee8ba5b6f56459ba2f14c86b59385b29b (commit)
from 858a8ce2394b57540f95fa794e894e3315508c8b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2cddd3c4e531e37c6d960c08da91a552bf75a1cc
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 22 17:02:01 2022 -0500
18842: Added tests for specific disk cache behavior
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/python/arvados/diskcache.py b/sdk/python/arvados/diskcache.py
index 9734d93a7..939ca3bb3 100644
--- a/sdk/python/arvados/diskcache.py
+++ b/sdk/python/arvados/diskcache.py
@@ -9,6 +9,7 @@ import traceback
import stat
import tempfile
import fcntl
+import time
import errno
import logging
@@ -143,7 +144,7 @@ class DiskCacheSlot(object):
# The locking strategy ensures that cache blocks
# in use remain visible.
#
- fcntl.flock(filehandle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
os.remove(final)
return True
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 7a227f122..570710a58 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -18,6 +18,7 @@ import re
import shutil
import socket
import sys
+import tempfile
import time
import unittest
import urllib.parse
@@ -568,6 +569,9 @@ class KeepClientCacheTestCase(unittest.TestCase, tutil.ApiClientMock, DiskCacheB
# First reponse was not cached because it was from a HEAD request.
self.assertNotEqual(head_resp, get_resp)
+
+
+
@tutil.skip_sleep
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class KeepStorageClassesTestCase(unittest.TestCase, tutil.ApiClientMock, DiskCacheBase):
@@ -1519,3 +1523,159 @@ class KeepClientAPIErrorTest(unittest.TestCase, DiskCacheBase):
keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8+3")
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.get("acbd18db4cc2f85cedef654fccc4a4d8+3")
+
+
+class KeepDiskCacheTestCase(unittest.TestCase, tutil.ApiClientMock):
+ def setUp(self):
+ self.api_client = self.mock_keep_services(count=2)
+ self.data = b'xyzzy'
+ self.locator = '1271ed5ef305aadabc605b1609e24c52'
+ self.disk_cache_dir = tempfile.mkdtemp()
+
+ def tearDown(self):
+ shutil.rmtree(self.disk_cache_dir)
+
+
+ @mock.patch('arvados.KeepClient.KeepService.get')
+ def test_disk_cache_read(self, get_mock):
+ # confirm it finds an existing cache block when the cache is
+ # initialized.
+
+ os.makedirs(os.path.join(self.disk_cache_dir, self.locator[0:3]))
+ with open(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock"), "wb") as f:
+ f.write(self.data)
+
+ # block cache should have found the existing block
+ block_cache = arvados.keep.KeepBlockCache(disk_cache=True,
+ disk_cache_dir=self.disk_cache_dir)
+ keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=block_cache)
+
+ self.assertTrue(tutil.binary_compare(keep_client.get(self.locator), self.data))
+
+ get_mock.assert_not_called()
+
+
+ @mock.patch('arvados.KeepClient.KeepService.get')
+ def test_disk_cache_share(self, get_mock):
+ # confirm it finds a cache block written after the disk cache
+ # was initialized.
+
+ block_cache = arvados.keep.KeepBlockCache(disk_cache=True,
+ disk_cache_dir=self.disk_cache_dir)
+ keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=block_cache)
+
+ os.makedirs(os.path.join(self.disk_cache_dir, self.locator[0:3]))
+ with open(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock"), "wb") as f:
+ f.write(self.data)
+
+ # when we try to get the block, it'll check the disk and find it.
+ self.assertTrue(tutil.binary_compare(keep_client.get(self.locator), self.data))
+
+ get_mock.assert_not_called()
+
+
+ def test_disk_cache_write(self):
+ # confirm the cache block was created
+
+ block_cache = arvados.keep.KeepBlockCache(disk_cache=True,
+ disk_cache_dir=self.disk_cache_dir)
+ keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=block_cache)
+
+ with tutil.mock_keep_responses(self.data, 200) as mock:
+ self.assertTrue(tutil.binary_compare(keep_client.get(self.locator), self.data))
+
+ with open(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock"), "rb") as f:
+ self.assertTrue(tutil.binary_compare(f.read(), self.data))
+
+
+ def test_disk_cache_clean(self):
+ # confirm that a tmp file in the cache is cleaned up
+
+ os.makedirs(os.path.join(self.disk_cache_dir, self.locator[0:3]))
+ with open(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC.keepcacheblock"), "wb") as f:
+ f.write(b"abc1")
+
+ with open(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC"), "wb") as f:
+ f.write(b"abc2")
+
+ with open(os.path.join(self.disk_cache_dir, self.locator[0:3], "XYZABC"), "wb") as f:
+ f.write(b"abc3")
+
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC.keepcacheblock")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "XYZABC")))
+
+ block_cache = arvados.keep.KeepBlockCache(disk_cache=True,
+ disk_cache_dir=self.disk_cache_dir)
+
+ # The tmp still hasn't been deleted because it was created in the last 60 seconds
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC.keepcacheblock")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "XYZABC")))
+
+ # Set the mtime to 61s in the past
+ os.utime(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC.keepcacheblock"), times=(time.time()-61, time.time()-61))
+ os.utime(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC"), times=(time.time()-61, time.time()-61))
+ os.utime(os.path.join(self.disk_cache_dir, self.locator[0:3], "XYZABC"), times=(time.time()-61, time.time()-61))
+
+ block_cache2 = arvados.keep.KeepBlockCache(disk_cache=True,
+ disk_cache_dir=self.disk_cache_dir)
+
+ # Tmp should be gone but the other ones are safe.
+ self.assertFalse(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC.keepcacheblock")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "tmpXYZABC")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], "XYZABC")))
+
+
+ @mock.patch('arvados.KeepClient.KeepService.get')
+ def test_disk_cache_cap(self, get_mock):
+ # confirm that the cache is kept to the desired limit
+
+ os.makedirs(os.path.join(self.disk_cache_dir, self.locator[0:3]))
+ with open(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock"), "wb") as f:
+ f.write(self.data)
+
+ os.makedirs(os.path.join(self.disk_cache_dir, "acb"))
+ with open(os.path.join(self.disk_cache_dir, "acb", "acbd18db4cc2f85cedef654fccc4a4d8.keepcacheblock"), "wb") as f:
+ f.write(b"foo")
+
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, "acb", "acbd18db4cc2f85cedef654fccc4a4d8.keepcacheblock")))
+
+ block_cache = arvados.keep.KeepBlockCache(disk_cache=True,
+ disk_cache_dir=self.disk_cache_dir,
+ max_slots=1)
+
+ self.assertFalse(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, "acb", "acbd18db4cc2f85cedef654fccc4a4d8.keepcacheblock")))
+
+
+ @mock.patch('arvados.KeepClient.KeepService.get')
+ def test_disk_cache_share(self, get_mock):
+ # confirm that a second cache doesn't delete files that belong to the first cache.
+
+ os.makedirs(os.path.join(self.disk_cache_dir, self.locator[0:3]))
+ with open(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock"), "wb") as f:
+ f.write(self.data)
+
+ os.makedirs(os.path.join(self.disk_cache_dir, "acb"))
+ with open(os.path.join(self.disk_cache_dir, "acb", "acbd18db4cc2f85cedef654fccc4a4d8.keepcacheblock"), "wb") as f:
+ f.write(b"foo")
+
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, "acb", "acbd18db4cc2f85cedef654fccc4a4d8.keepcacheblock")))
+
+ block_cache = arvados.keep.KeepBlockCache(disk_cache=True,
+ disk_cache_dir=self.disk_cache_dir,
+ max_slots=2)
+
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, "acb", "acbd18db4cc2f85cedef654fccc4a4d8.keepcacheblock")))
+
+
+ block_cache2 = arvados.keep.KeepBlockCache(disk_cache=True,
+ disk_cache_dir=self.disk_cache_dir,
+ max_slots=1)
+
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, self.locator[0:3], self.locator+".keepcacheblock")))
+ self.assertTrue(os.path.exists(os.path.join(self.disk_cache_dir, "acb", "acbd18db4cc2f85cedef654fccc4a4d8.keepcacheblock")))
commit 2693a197ea37490a891c54bbb279937a7b4e897c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 22 16:12:20 2022 -0500
18842: Use tempdir for cache directory in tests
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index 3772761b8..003565979 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -295,3 +295,15 @@ def make_block_cache(disk_cache):
shutil.rmtree(disk_cache_dir, ignore_errors=True)
block_cache = arvados.keep.KeepBlockCache(disk_cache=disk_cache)
return block_cache
+
+
+class DiskCacheBase:
+ def make_block_cache(self, disk_cache):
+ self.disk_cache_dir = tempfile.mkdtemp() if disk_cache else None
+ block_cache = arvados.keep.KeepBlockCache(disk_cache=disk_cache,
+ disk_cache_dir=self.disk_cache_dir)
+ return block_cache
+
+ def tearDown(self):
+ if self.disk_cache_dir:
+ shutil.rmtree(self.disk_cache_dir)
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 126116393..7a227f122 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -31,22 +31,29 @@ from . import arvados_testutil as tutil
from . import keepstub
from . import run_test_server
-from .arvados_testutil import make_block_cache
+from .arvados_testutil import DiskCacheBase
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepTestCase(run_test_server.TestCaseWithServers):
+class KeepTestCase(run_test_server.TestCaseWithServers, DiskCacheBase):
disk_cache = False
MAIN_SERVER = {}
KEEP_SERVER = {}
+ block_cache_test = None
@classmethod
def setUpClass(cls):
super(KeepTestCase, cls).setUpClass()
run_test_server.authorize_with("admin")
cls.api_client = arvados.api('v1')
+ cls.block_cache_test = DiskCacheBase()
cls.keep_client = arvados.KeepClient(api_client=cls.api_client,
proxy='', local_store='',
- block_cache=make_block_cache(cls.disk_cache))
+ block_cache=cls.block_cache_test.make_block_cache(cls.disk_cache))
+
+ @classmethod
+ def tearDownClass(cls):
+ super(KeepTestCase, cls).setUpClass()
+ cls.block_cache_test.tearDown()
def test_KeepBasicRWTest(self):
self.assertEqual(0, self.keep_client.upload_counter.get())
@@ -137,14 +144,17 @@ class KeepTestCase(run_test_server.TestCaseWithServers):
'wrong content from Keep.get for "test_head"')
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
+class KeepPermissionTestCase(run_test_server.TestCaseWithServers, DiskCacheBase):
disk_cache = False
MAIN_SERVER = {}
KEEP_SERVER = {'blob_signing': True}
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def test_KeepBasicRWTest(self):
run_test_server.authorize_with('active')
- keep_client = arvados.KeepClient(block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(block_cache=self.make_block_cache(self.disk_cache))
foo_locator = keep_client.put('foo')
self.assertRegex(
foo_locator,
@@ -182,7 +192,7 @@ class KeepPermissionTestCase(run_test_server.TestCaseWithServers):
unsigned_bar_locator)
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepProxyTestCase(run_test_server.TestCaseWithServers):
+class KeepProxyTestCase(run_test_server.TestCaseWithServers, DiskCacheBase):
disk_cache = False
MAIN_SERVER = {}
KEEP_SERVER = {}
@@ -196,12 +206,13 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
def tearDown(self):
super(KeepProxyTestCase, self).tearDown()
+ DiskCacheBase.tearDown(self)
def test_KeepProxyTest1(self):
# Will use ARVADOS_KEEP_SERVICES environment variable that
# is set by setUpClass().
keep_client = arvados.KeepClient(api_client=self.api_client,
- local_store='', block_cache=make_block_cache(self.disk_cache))
+ local_store='', block_cache=self.make_block_cache(self.disk_cache))
baz_locator = keep_client.put('baz')
self.assertRegex(
baz_locator,
@@ -218,7 +229,7 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
arvados.config.settings()['ARVADOS_KEEP_SERVICES'] = 'http://10.0.0.1 https://foo.example.org:1234/'
keep_client = arvados.KeepClient(api_client=self.api_client,
local_store='',
- block_cache=make_block_cache(self.disk_cache))
+ block_cache=self.make_block_cache(self.disk_cache))
uris = [x['_service_root'] for x in keep_client._keep_services]
self.assertEqual(uris, ['http://10.0.0.1/',
'https://foo.example.org:1234/'])
@@ -228,14 +239,17 @@ class KeepProxyTestCase(run_test_server.TestCaseWithServers):
with self.assertRaises(arvados.errors.ArgumentError):
keep_client = arvados.KeepClient(api_client=self.api_client,
local_store='',
- block_cache=make_block_cache(self.disk_cache))
+ block_cache=self.make_block_cache(self.disk_cache))
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
+class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock, DiskCacheBase):
disk_cache = False
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def get_service_roots(self, api_client):
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
services = keep_client.weighted_service_roots(arvados.KeepLocator('0'*32))
return [urllib.parse.urlparse(url) for url in sorted(services)]
@@ -255,7 +269,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
def test_recognize_proxy_services_in_controller_response(self):
keep_client = arvados.KeepClient(api_client=self.mock_keep_services(
service_type='proxy', service_host='localhost', service_port=9, count=1),
- block_cache=make_block_cache(self.disk_cache))
+ block_cache=self.make_block_cache(self.disk_cache))
try:
# this will fail, but it ensures we get the service
# discovery response
@@ -270,7 +284,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client.insecure = True
with tutil.mock_keep_responses(b'foo', 200) as mock:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3')
self.assertEqual(
mock.responses[0].getopt(pycurl.SSL_VERIFYPEER),
@@ -281,7 +295,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client.insecure = False
with tutil.mock_keep_responses(b'foo', 200) as mock:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
keep_client.get('acbd18db4cc2f85cedef654fccc4a4d8+3')
# getopt()==None here means we didn't change the
# default. If we were using real pycurl instead of a mock,
@@ -302,7 +316,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
headers = {'X-Keep-Locator':local_loc}
with tutil.mock_keep_responses('', 200, **headers):
# Check that the translated locator gets returned
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
self.assertEqual(local_loc, keep_client.refresh_signature(remote_loc))
# Check that refresh_signature() uses the correct method and headers
keep_client._get_or_head = mock.MagicMock()
@@ -321,7 +335,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client = self.mock_keep_services(count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.get('ffffffffffffffffffffffffffffffff')
self.assertEqual(
@@ -338,7 +352,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client = self.mock_keep_services(count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
with self.assertRaises(arvados.errors.KeepWriteError):
keep_client.put(b'foo')
self.assertEqual(
@@ -355,7 +369,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client = self.mock_keep_services(count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.head('ffffffffffffffffffffffffffffffff')
self.assertEqual(
@@ -372,7 +386,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client = self.mock_keep_services(service_type='proxy', count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.get('ffffffffffffffffffffffffffffffff')
self.assertEqual(
@@ -389,7 +403,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client = self.mock_keep_services(service_type='proxy', count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
with self.assertRaises(arvados.errors.KeepReadError):
keep_client.head('ffffffffffffffffffffffffffffffff')
self.assertEqual(
@@ -403,6 +417,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
None)
def test_proxy_put_timeout(self):
+ self.disk_cache_dir = None
api_client = self.mock_keep_services(service_type='proxy', count=1)
force_timeout = socket.timeout("timed out")
with tutil.mock_keep_responses(force_timeout, 0) as mock:
@@ -423,7 +438,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client = mock.MagicMock(name='api_client')
api_client.keep_services().accessible().execute.side_effect = (
arvados.errors.ApiError)
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
with self.assertRaises(exc_class) as err_check:
getattr(keep_client, verb)('d41d8cd98f00b204e9800998ecf8427e+0')
self.assertEqual(0, len(err_check.exception.request_errors()))
@@ -443,7 +458,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
"retry error reporting test", 500, 500, 500, 500, 500, 500, 502, 502)
with req_mock, tutil.skip_sleep, \
self.assertRaises(exc_class) as err_check:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
getattr(keep_client, verb)('d41d8cd98f00b204e9800998ecf8427e+0',
num_retries=3)
self.assertEqual([502, 502], [
@@ -466,7 +481,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client = self.mock_keep_services(count=3)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
keep_client.put(data)
self.assertEqual(2, len(exc_check.exception.request_errors()))
@@ -476,7 +491,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
api_client = self.mock_keep_services(service_type='proxy', read_only=True, count=1)
with tutil.mock_keep_responses(data_loc, 200, 500, 500) as req_mock, \
self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
keep_client.put(data)
self.assertEqual(True, ("no Keep services available" in str(exc_check.exception)))
self.assertEqual(0, len(exc_check.exception.request_errors()))
@@ -485,7 +500,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
body = b'oddball service get'
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(body, 200):
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
actual = keep_client.get(tutil.str_keep_locator(body))
self.assertEqual(body, actual)
@@ -494,7 +509,7 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
pdh = tutil.str_keep_locator(body)
api_client = self.mock_keep_services(service_type='fancynewblobstore')
with tutil.mock_keep_responses(pdh, 200):
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
actual = keep_client.put(body, copies=1)
self.assertEqual(pdh, actual)
@@ -506,22 +521,25 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
headers = {'x-keep-replicas-stored': 3}
with tutil.mock_keep_responses(pdh, 200, 418, 418, 418,
**headers) as req_mock:
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
actual = keep_client.put(body, copies=2)
self.assertEqual(pdh, actual)
self.assertEqual(1, req_mock.call_count)
@tutil.skip_sleep
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientCacheTestCase(unittest.TestCase, tutil.ApiClientMock):
+class KeepClientCacheTestCase(unittest.TestCase, tutil.ApiClientMock, DiskCacheBase):
disk_cache = False
def setUp(self):
self.api_client = self.mock_keep_services(count=2)
- self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=make_block_cache(self.disk_cache))
+ self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=self.make_block_cache(self.disk_cache))
self.data = b'xyzzy'
self.locator = '1271ed5ef305aadabc605b1609e24c52'
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
@mock.patch('arvados.KeepClient.KeepService.get')
def test_get_request_cache(self, get_mock):
with tutil.mock_keep_responses(self.data, 200, 200):
@@ -552,15 +570,18 @@ class KeepClientCacheTestCase(unittest.TestCase, tutil.ApiClientMock):
@tutil.skip_sleep
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepStorageClassesTestCase(unittest.TestCase, tutil.ApiClientMock):
+class KeepStorageClassesTestCase(unittest.TestCase, tutil.ApiClientMock, DiskCacheBase):
disk_cache = False
def setUp(self):
self.api_client = self.mock_keep_services(count=2)
- self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=make_block_cache(self.disk_cache))
+ self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=self.make_block_cache(self.disk_cache))
self.data = b'xyzzy'
self.locator = '1271ed5ef305aadabc605b1609e24c52'
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def test_multiple_default_storage_classes_req_header(self):
api_mock = self.api_client_mock()
api_mock.config.return_value = {
@@ -571,7 +592,7 @@ class KeepStorageClassesTestCase(unittest.TestCase, tutil.ApiClientMock):
}
}
api_client = self.mock_keep_services(api_mock=api_mock, count=2)
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
resp_hdr = {
'x-keep-storage-classes-confirmed': 'foo=1, bar=1',
'x-keep-replicas-stored': 1
@@ -666,12 +687,12 @@ class KeepStorageClassesTestCase(unittest.TestCase, tutil.ApiClientMock):
@tutil.skip_sleep
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock):
+class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock, DiskCacheBase):
disk_cache = False
def setUp(self):
self.api_client = self.mock_keep_services(count=2)
- self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=make_block_cache(self.disk_cache))
+ self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=self.make_block_cache(self.disk_cache))
self.data = b'xyzzy'
self.locator = '1271ed5ef305aadabc605b1609e24c52'
self.test_id = arvados.util.new_request_id()
@@ -681,6 +702,9 @@ class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock):
# id='123456789'>:
self.api_client.request_id = None
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def test_default_to_api_client_request_id(self):
self.api_client.request_id = self.test_id
with tutil.mock_keep_responses(self.locator, 200, 200) as mock:
@@ -757,7 +781,7 @@ class KeepXRequestIdTestCase(unittest.TestCase, tutil.ApiClientMock):
@tutil.skip_sleep
#@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
+class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock, DiskCacheBase):
disk_cache = False
def setUp(self):
@@ -781,7 +805,10 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
hashlib.md5(self.blocks[x]).hexdigest()
for x in range(len(self.expected_order))]
self.api_client = self.mock_keep_services(count=self.services)
- self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=make_block_cache(self.disk_cache))
+ self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=self.make_block_cache(self.disk_cache))
+
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
def test_weighted_service_roots_against_reference_set(self):
# Confirm weighted_service_roots() returns the correct order
@@ -854,12 +881,12 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
hashlib.md5("{:064x}".format(x).encode()).hexdigest() for x in range(100)]
initial_services = 12
self.api_client = self.mock_keep_services(count=initial_services)
- self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=make_block_cache(self.disk_cache))
+ self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=self.make_block_cache(self.disk_cache))
probes_before = [
self.keep_client.weighted_service_roots(arvados.KeepLocator(hash)) for hash in hashes]
for added_services in range(1, 12):
api_client = self.mock_keep_services(count=initial_services+added_services)
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
total_penalty = 0
for hash_index in range(len(hashes)):
probe_after = keep_client.weighted_service_roots(
@@ -895,7 +922,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
# Arbitrary port number:
aport = random.randint(1024,65535)
api_client = self.mock_keep_services(service_port=aport, count=self.services)
- keep_client = arvados.KeepClient(api_client=api_client, block_cache=make_block_cache(self.disk_cache))
+ keep_client = arvados.KeepClient(api_client=api_client, block_cache=self.make_block_cache(self.disk_cache))
with mock.patch('pycurl.Curl') as curl_mock, \
self.assertRaises(exc_class) as err_check:
curl_mock.return_value = tutil.FakeCurl.make(code=500, body=b'')
@@ -912,7 +939,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
self.check_64_zeros_error_order('put', arvados.errors.KeepWriteError)
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
+class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase, DiskCacheBase):
disk_cache = False
# BANDWIDTH_LOW_LIM must be less than len(DATA) so we can transfer
@@ -922,6 +949,9 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
BANDWIDTH_LOW_LIM = 1024
TIMEOUT_TIME = 1.0
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
class assertTakesBetween(unittest.TestCase):
def __init__(self, tmin, tmax):
self.tmin = tmin
@@ -951,7 +981,7 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
def keepClient(self, timeouts=(0.1, TIMEOUT_TIME, BANDWIDTH_LOW_LIM)):
return arvados.KeepClient(
api_client=self.api_client,
- timeout=timeouts, block_cache=make_block_cache(self.disk_cache))
+ timeout=timeouts, block_cache=self.make_block_cache(self.disk_cache))
def test_timeout_slow_connect(self):
# Can't simulate TCP delays with our own socket. Leave our
@@ -1056,9 +1086,12 @@ class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
kc.put(self.DATA, copies=1, num_retries=0)
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
+class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock, DiskCacheBase):
disk_cache = False
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def mock_disks_and_gateways(self, disks=3, gateways=1):
self.gateways = [{
'uuid': 'zzzzz-bi6l4-gateway{:08d}'.format(i),
@@ -1073,7 +1106,7 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
for gw in self.gateways]
self.api_client = self.mock_keep_services(
count=disks, additional_services=self.gateways)
- self.keepClient = arvados.KeepClient(api_client=self.api_client, block_cache=make_block_cache(self.disk_cache))
+ self.keepClient = arvados.KeepClient(api_client=self.api_client, block_cache=self.make_block_cache(self.disk_cache))
@mock.patch('pycurl.Curl')
def test_get_with_gateway_hint_first(self, MockCurl):
@@ -1181,7 +1214,7 @@ class KeepClientRetryTestMixin(object):
def new_client(self, **caller_kwargs):
kwargs = self.client_kwargs.copy()
kwargs.update(caller_kwargs)
- kwargs['block_cache'] = make_block_cache(self.disk_cache)
+ kwargs['block_cache'] = self.make_block_cache(self.disk_cache)
return arvados.KeepClient(**kwargs)
def run_method(self, *args, **kwargs):
@@ -1232,12 +1265,15 @@ class KeepClientRetryTestMixin(object):
@tutil.skip_sleep
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientRetryGetTestCase(KeepClientRetryTestMixin, unittest.TestCase):
+class KeepClientRetryGetTestCase(KeepClientRetryTestMixin, unittest.TestCase, DiskCacheBase):
DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_DATA
DEFAULT_EXCEPTION = arvados.errors.KeepReadError
HINTED_LOCATOR = KeepClientRetryTestMixin.TEST_LOCATOR + '+K at xyzzy'
TEST_PATCHER = staticmethod(tutil.mock_keep_responses)
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def run_method(self, locator=KeepClientRetryTestMixin.TEST_LOCATOR,
*args, **kwargs):
return self.new_client().get(locator, *args, **kwargs)
@@ -1277,12 +1313,15 @@ class KeepClientRetryGetTestCase(KeepClientRetryTestMixin, unittest.TestCase):
@tutil.skip_sleep
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientRetryHeadTestCase(KeepClientRetryTestMixin, unittest.TestCase):
+class KeepClientRetryHeadTestCase(KeepClientRetryTestMixin, unittest.TestCase, DiskCacheBase):
DEFAULT_EXPECT = True
DEFAULT_EXCEPTION = arvados.errors.KeepReadError
HINTED_LOCATOR = KeepClientRetryTestMixin.TEST_LOCATOR + '+K at xyzzy'
TEST_PATCHER = staticmethod(tutil.mock_keep_responses)
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def run_method(self, locator=KeepClientRetryTestMixin.TEST_LOCATOR,
*args, **kwargs):
return self.new_client().head(locator, *args, **kwargs)
@@ -1316,11 +1355,14 @@ class KeepClientRetryHeadTestCase(KeepClientRetryTestMixin, unittest.TestCase):
@tutil.skip_sleep
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientRetryPutTestCase(KeepClientRetryTestMixin, unittest.TestCase):
+class KeepClientRetryPutTestCase(KeepClientRetryTestMixin, unittest.TestCase, DiskCacheBase):
DEFAULT_EXPECT = KeepClientRetryTestMixin.TEST_LOCATOR
DEFAULT_EXCEPTION = arvados.errors.KeepWriteError
TEST_PATCHER = staticmethod(tutil.mock_keep_responses)
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def run_method(self, data=KeepClientRetryTestMixin.TEST_DATA,
copies=1, *args, **kwargs):
return self.new_client().put(data, copies, *args, **kwargs)
@@ -1405,7 +1447,7 @@ class AvoidOverreplication(unittest.TestCase, tutil.ApiClientMock):
@tutil.skip_sleep
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class RetryNeedsMultipleServices(unittest.TestCase, tutil.ApiClientMock):
+class RetryNeedsMultipleServices(unittest.TestCase, tutil.ApiClientMock, DiskCacheBase):
block_cache = False
# Test put()s that need two distinct servers to succeed, possibly
@@ -1413,7 +1455,10 @@ class RetryNeedsMultipleServices(unittest.TestCase, tutil.ApiClientMock):
def setUp(self):
self.api_client = self.mock_keep_services(count=2)
- self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=make_block_cache(self.disk_cache))
+ self.keep_client = arvados.KeepClient(api_client=self.api_client, block_cache=self.make_block_cache(self.disk_cache))
+
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
def test_success_after_exception(self):
with tutil.mock_keep_responses(
@@ -1441,9 +1486,12 @@ class RetryNeedsMultipleServices(unittest.TestCase, tutil.ApiClientMock):
self.assertEqual(2, req_mock.call_count)
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
-class KeepClientAPIErrorTest(unittest.TestCase):
+class KeepClientAPIErrorTest(unittest.TestCase, DiskCacheBase):
disk_cache = False
+ def tearDown(self):
+ DiskCacheBase.tearDown(self)
+
def test_api_fail(self):
class ApiMock(object):
def __getattr__(self, r):
@@ -1457,7 +1505,7 @@ class KeepClientAPIErrorTest(unittest.TestCase):
raise arvados.errors.KeepReadError()
keep_client = arvados.KeepClient(api_client=ApiMock(),
proxy='', local_store='',
- block_cache=make_block_cache(self.disk_cache))
+ block_cache=self.make_block_cache(self.disk_cache))
# The bug this is testing for is that if an API (not
# keepstore) exception is thrown as part of a get(), the next
commit 09cbdc3074b3f1e69c9c537875146f6da0a6ed8f
Merge: 9828e9218 2b7d05cdf
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 22 13:49:01 2022 -0500
Merge branch 'main' into 18842-arv-mount-disk-config
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
commit 9828e9218084856240fdeafa2d388d8bf322e655
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Nov 1 17:05:42 2022 -0400
18842: Respond to review comments
* Fix test typo
* Add suffix .keepcache
* Delete tmp file if an error is thrown before it is renamed
* Default disk cache size accounts for both free space and total disk size
* Handle errors and fall back to RAM caching
* Delete old tmp blocks if we find them
* Collection class gets keep client if initialized with ThreadSafeApiCache
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/admin/maintenance-and-upgrading.html.textile.liquid b/doc/admin/maintenance-and-upgrading.html.textile.liquid
index 3cc80a356..2b634fb9e 100644
--- a/doc/admin/maintenance-and-upgrading.html.textile.liquid
+++ b/doc/admin/maintenance-and-upgrading.html.textile.liquid
@@ -60,7 +60,9 @@ Upgrading Arvados typically involves the following steps:
# Wait for the cluster to be idle and stop Arvados services.
# Make a backup of your database, as a precaution.
# update the configuration file for the new release, if necessary (see "Maintaining Arvados":#maintaining above)
-# rebuild and deploy the "compute node image":{{site.baseurl}}/install/crunch2-cloud/install-compute-node.html (cloud only)
+# Update compute nodes
+## (cloud) Rebuild and deploy the "compute node image":{{site.baseurl}}/install/crunch2-cloud/install-compute-node.html
+## (slurm/LSF) Upgrade the @python3-arvados-fuse@ package used on your compute nodes
# Install new packages using @apt-get upgrade@ or @yum upgrade at .
# Wait for package installation scripts as they perform any necessary data migrations.
# Run @arvados-server config-check@ to detect configuration errors or deprecated entries.
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 998481ab6..e1138910a 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1308,6 +1308,11 @@ class Collection(RichCollectionBase):
super(Collection, self).__init__(parent)
self._api_client = api_client
self._keep_client = keep_client
+
+ # Use the keep client from ThreadSafeApiCache
+ if self._keep_client is None and isinstance(self._api_client, ThreadSafeApiCache):
+ self._keep_client = self._api_client.keep
+
self._block_manager = block_manager
self.replication_desired = replication_desired
self._storage_classes_desired = storage_classes_desired
diff --git a/sdk/python/arvados/diskcache.py b/sdk/python/arvados/diskcache.py
index 6f1ccb97e..9734d93a7 100644
--- a/sdk/python/arvados/diskcache.py
+++ b/sdk/python/arvados/diskcache.py
@@ -9,6 +9,12 @@ import traceback
import stat
import tempfile
import fcntl
+import errno
+import logging
+
+_logger = logging.getLogger('arvados.keep')
+
+cacheblock_suffix = ".keepcacheblock"
class DiskCacheSlot(object):
__slots__ = ("locator", "ready", "content", "cachedir")
@@ -24,6 +30,7 @@ class DiskCacheSlot(object):
return self.content
def set(self, value):
+ tmpfile = None
try:
if value is None:
self.content = None
@@ -41,9 +48,9 @@ class DiskCacheSlot(object):
blockdir = os.path.join(self.cachedir, self.locator[0:3])
os.makedirs(blockdir, mode=0o700, exist_ok=True)
- final = os.path.join(blockdir, self.locator)
+ final = os.path.join(blockdir, self.locator) + cacheblock_suffix
- f = tempfile.NamedTemporaryFile(dir=blockdir, delete=False)
+ f = tempfile.NamedTemporaryFile(dir=blockdir, delete=False, prefix="tmp", suffix=cacheblock_suffix)
tmpfile = f.name
os.chmod(tmpfile, stat.S_IRUSR | stat.S_IWUSR)
@@ -54,11 +61,32 @@ class DiskCacheSlot(object):
f.write(value)
f.flush()
os.rename(tmpfile, final)
+ tmpfile = None
self.content = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
+ except OSError as e:
+ if e.errno == errno.ENODEV:
+ _logger.error("Unable to use disk cache: The underlying filesystem does not support memory mapping.")
+ elif e.errno == errno.ENOMEM:
+ _logger.error("Unable to use disk cache: The process's maximum number of mappings would have been exceeded.")
+ elif e.errno == errno.ENOSPC:
+ _logger.error("Unable to use disk cache: Out of disk space.")
+ else:
+ traceback.print_exc()
except Exception as e:
traceback.print_exc()
finally:
+ if tmpfile is not None:
+ # If the tempfile hasn't been renamed on disk yet, try to delete it.
+ try:
+ os.remove(tmpfile)
+ except:
+ pass
+ if self.content is None:
+ # Something went wrong with the disk cache, fall back
+ # to RAM cache behavior (the alternative is to cache
+ # nothing and return a read error).
+ self.content = value
self.ready.set()
def size(self):
@@ -89,11 +117,12 @@ class DiskCacheSlot(object):
# gone.
blockdir = os.path.join(self.cachedir, self.locator[0:3])
- final = os.path.join(blockdir, self.locator)
+ final = os.path.join(blockdir, self.locator) + cacheblock_suffix
try:
with open(final, "rb") as f:
- # unlock,
+ # unlock
fcntl.flock(f, fcntl.LOCK_UN)
+ self.content = None
# try to get an exclusive lock, this ensures other
# processes are not using the block. It is
@@ -125,7 +154,7 @@ class DiskCacheSlot(object):
@staticmethod
def get_from_disk(locator, cachedir):
blockdir = os.path.join(cachedir, locator[0:3])
- final = os.path.join(blockdir, locator)
+ final = os.path.join(blockdir, locator) + cacheblock_suffix
try:
filehandle = open(final, "rb")
@@ -156,9 +185,21 @@ class DiskCacheSlot(object):
blocks = []
for root, dirs, files in os.walk(cachedir):
for name in files:
+ if not name.endswith(cacheblock_suffix):
+ continue
+
blockpath = os.path.join(root, name)
res = os.stat(blockpath)
- blocks.append((name, res.st_atime))
+
+ if len(name) == (32+len(cacheblock_suffix)) and not name.startswith("tmp"):
+ blocks.append((name[0:32], res.st_atime))
+ elif name.startswith("tmp") and ((time.time() - res.st_mtime) > 60):
+ # found a temporary file more than 1 minute old,
+ # try to delete it.
+ try:
+ os.remove(blockpath)
+ except:
+ pass
# sort by access time (atime), going from most recently
# accessed (highest timestamp) to least recently accessed
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index b6c98d143..dd99e8b92 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -190,17 +190,24 @@ class KeepBlockCache(object):
if self._max_slots == 0:
if self._disk_cache:
- # default set max slots to half of maximum file handles
+ # default max slots to half of maximum file handles
+ # NOFILE typically defaults to 1024 on Linux so this
+ # will be 512 slots.
self._max_slots = resource.getrlimit(resource.RLIMIT_NOFILE)[0] / 2
else:
- self._max_slots = 1024
+ # RAM cache slots
+ self._max_slots = 512
if self.cache_max == 0:
if self._disk_cache:
fs = os.statvfs(self._disk_cache_dir)
- avail = (fs.f_bavail * fs.f_bsize) / 2
- # Half the available space or max_slots * 64 MiB
- self.cache_max = min(avail, (self._max_slots * 64 * 1024 * 1024))
+ avail = (fs.f_bavail * fs.f_bsize) / 4
+ maxdisk = int((fs.f_blocks * fs.f_bsize) * 0.10)
+ # pick smallest of:
+ # 10% of total disk size
+ # 25% of available space
+ # max_slots * 64 MiB
+ self.cache_max = min(min(maxdisk, avail), (self._max_slots * 64 * 1024 * 1024))
else:
# 256 GiB in RAM
self.cache_max = (256 * 1024 * 1024)
diff --git a/services/fuse/tests/mount_test_base.py b/services/fuse/tests/mount_test_base.py
index b1383d36b..e82660408 100644
--- a/services/fuse/tests/mount_test_base.py
+++ b/services/fuse/tests/mount_test_base.py
@@ -33,7 +33,7 @@ def make_block_cache(disk_cache):
return block_cache
class MountTestBase(unittest.TestCase):
- block_cache = False
+ disk_cache = False
def setUp(self, api=None, local_store=True):
# The underlying C implementation of open() makes a fstat() syscall
@@ -54,7 +54,7 @@ class MountTestBase(unittest.TestCase):
run_test_server.run()
run_test_server.authorize_with("admin")
- self.api = api if api else arvados.safeapi.ThreadSafeApiCache(arvados.config.settings(), keep_params={"block_cache": make_block_cache(self.block_cache)})
+ self.api = api if api else arvados.safeapi.ThreadSafeApiCache(arvados.config.settings(), keep_params={"block_cache": make_block_cache(self.disk_cache)})
self.llfuse_thread = None
# This is a copy of Mount's method. TODO: Refactor MountTestBase
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list