[arvados] created: 2.7.0-6546-gb8751a1a22
git repository hosting
git at public.arvados.org
Fri May 3 20:55:38 UTC 2024
at b8751a1a220ea451d44303268ea9207c2152aed2 (commit)
commit b8751a1a220ea451d44303268ea9207c2152aed2
Author: Brett Smith <brett.smith at curii.com>
Date: Fri May 3 16:27:32 2024 -0400
21207: Run Python tests with pytest
This gets us away from a deprecated setup.py invocation, and provides
richer test authoring and result reporting facilities.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/.licenseignore b/.licenseignore
index 1e1c12a53a..926aff75a4 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -43,6 +43,7 @@ docker/jobs/1078ECD7.key
*.min.js
*.png
*/proc_stat
+*/pytest.ini
*/README
*/robots.txt
*/runit-docker/*
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 3ab571ff7f..eb2fecf74d 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -566,13 +566,14 @@ install_env() {
setup_virtualenv "$VENV3DIR"
. "$VENV3DIR/bin/activate"
- # wheel modernizes the venv (as of early 2024) and makes it more closely
- # match our package build environment.
+ # parameterized and pytest are direct dependencies of Python tests.
# PyYAML is a test requirement used by run_test_server.py and needed for
# other, non-Python tests.
# pdoc is needed to build PySDK documentation.
+ # wheel modernizes the venv (as of early 2024) and makes it more closely
+ # match our package build environment.
# We run `setup.py build` first to generate _version.py.
- pip install PyYAML pdoc wheel \
+ pip install parameterized pytest PyYAML pdoc wheel \
&& env -C "$WORKSPACE/sdk/python" python3 setup.py build \
&& pip install "$WORKSPACE/sdk/python" \
|| fatal "installing Python SDK and related dependencies failed"
@@ -701,7 +702,7 @@ do_test_once() {
elif [[ "$2" == "pip" ]]
then
tries=0
- cd "$WORKSPACE/$1" && while :
+ cd "$WORKSPACE/$1" && pip install . && while :
do
tries=$((${tries}+1))
# $3 can name a path directory for us to use, including trailing
@@ -709,7 +710,7 @@ do_test_once() {
if [[ -e "${3}activate" ]]; then
. "${3}activate"
fi
- python setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+ python3 -m pytest ${testargs[$1]}
result=$?
if [[ ${tries} < 3 && ${result} == 137 ]]
then
diff --git a/sdk/cwl/pytest.ini b/sdk/cwl/pytest.ini
new file mode 120000
index 0000000000..05a82dbfef
--- /dev/null
+++ b/sdk/cwl/pytest.ini
@@ -0,0 +1 @@
+../../sdk/python/pytest.ini
\ No newline at end of file
diff --git a/sdk/python/pytest.ini b/sdk/python/pytest.ini
new file mode 100644
index 0000000000..9b1bbfd50a
--- /dev/null
+++ b/sdk/python/pytest.ini
@@ -0,0 +1,8 @@
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+[pytest]
+testpaths =
+ tests
diff --git a/services/dockercleaner/pytest.ini b/services/dockercleaner/pytest.ini
new file mode 120000
index 0000000000..05a82dbfef
--- /dev/null
+++ b/services/dockercleaner/pytest.ini
@@ -0,0 +1 @@
+../../sdk/python/pytest.ini
\ No newline at end of file
diff --git a/services/fuse/pytest.ini b/services/fuse/pytest.ini
new file mode 120000
index 0000000000..05a82dbfef
--- /dev/null
+++ b/services/fuse/pytest.ini
@@ -0,0 +1 @@
+../../sdk/python/pytest.ini
\ No newline at end of file
diff --git a/tools/crunchstat-summary/pytest.ini b/tools/crunchstat-summary/pytest.ini
new file mode 120000
index 0000000000..05a82dbfef
--- /dev/null
+++ b/tools/crunchstat-summary/pytest.ini
@@ -0,0 +1 @@
+../../sdk/python/pytest.ini
\ No newline at end of file
diff --git a/tools/user-activity/pytest.ini b/tools/user-activity/pytest.ini
new file mode 120000
index 0000000000..05a82dbfef
--- /dev/null
+++ b/tools/user-activity/pytest.ini
@@ -0,0 +1 @@
+../../sdk/python/pytest.ini
\ No newline at end of file
commit c58300cc5eff7a227d8a0499ac3dc2c125ad4c3e
Author: Brett Smith <brett.smith at curii.com>
Date: Fri May 3 16:34:38 2024 -0400
21207: Remove Python slow/short tests support
After the previous commit there are no more users of this, and this
interface will no longer be supported after our migration to pytest. If
we want this functionality in the future we should set up a custom
"slow" mark in pytest and use pytest's built-in mark filtering.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index 794b6afe42..ca20de4c58 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -72,14 +72,6 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
-def short_tests_only(arglist=sys.argv):
- try:
- arglist.remove('--short-tests-only')
- except ValueError:
- return False
- else:
- return True
-
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 794b6afe42..ca20de4c58 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -72,14 +72,6 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
-def short_tests_only(arglist=sys.argv):
- try:
- arglist.remove('--short-tests-only')
- except ValueError:
- return False
- else:
- return True
-
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 257968d89e..dc10269060 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -13,7 +13,6 @@ from setuptools.command import build_py
import arvados_version
version = arvados_version.get_version()
-short_tests_only = arvados_version.short_tests_only()
README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
class BuildPython(build_py.build_py):
diff --git a/sdk/python/tests/slow_test.py b/sdk/python/tests/slow_test.py
deleted file mode 100644
index ae46f4e1bf..0000000000
--- a/sdk/python/tests/slow_test.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import __main__
-import os
-import unittest
-
-slow_test = lambda _: unittest.skipIf(
- __main__.short_tests_only,
- "running --short tests only")
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 794b6afe42..ca20de4c58 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -72,14 +72,6 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
-def short_tests_only(arglist=sys.argv):
- try:
- arglist.remove('--short-tests-only')
- except ValueError:
- return False
- else:
- return True
-
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index b20b776ebf..cb0fc8d88a 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -11,7 +11,6 @@ from setuptools import setup, find_packages
import arvados_version
version = arvados_version.get_version()
-short_tests_only = arvados_version.short_tests_only()
README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name="arvados-docker-cleaner",
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 794b6afe42..ca20de4c58 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -72,14 +72,6 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
-def short_tests_only(arglist=sys.argv):
- try:
- arglist.remove('--short-tests-only')
- except ValueError:
- return False
- else:
- return True
-
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index 002afc22c9..f9b0fcf91d 100644
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -11,7 +11,6 @@ from setuptools import setup, find_packages
import arvados_version
version = arvados_version.get_version()
-short_tests_only = arvados_version.short_tests_only()
README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='arvados_fuse',
diff --git a/services/fuse/tests/slow_test.py b/services/fuse/tests/slow_test.py
deleted file mode 120000
index c7e1f7fada..0000000000
--- a/services/fuse/tests/slow_test.py
+++ /dev/null
@@ -1 +0,0 @@
-../../../sdk/python/tests/slow_test.py
\ No newline at end of file
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index 794b6afe42..ca20de4c58 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -72,14 +72,6 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
-def short_tests_only(arglist=sys.argv):
- try:
- arglist.remove('--short-tests-only')
- except ValueError:
- return False
- else:
- return True
-
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 95c534c90b..d5adc92367 100755
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -11,7 +11,6 @@ from setuptools import setup, find_packages
import arvados_version
version = arvados_version.get_version()
-short_tests_only = arvados_version.short_tests_only()
README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='crunchstat_summary',
diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
index 794b6afe42..ca20de4c58 100644
--- a/tools/user-activity/arvados_version.py
+++ b/tools/user-activity/arvados_version.py
@@ -72,14 +72,6 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
-def short_tests_only(arglist=sys.argv):
- try:
- arglist.remove('--short-tests-only')
- except ValueError:
- return False
- else:
- return True
-
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
commit 9de62c160f7cc9c9c994e4ced447cd5bef88333e
Author: Brett Smith <brett.smith at curii.com>
Date: Fri May 3 16:13:36 2024 -0400
21207: Remove FUSE performance tests
None of these tests have been running as intended. The `slow_test`
decorator is written incorrectly such that the return value of
`unittest.skipIf` *replaces* the test method rather than *wrapping* it.
We just did a round of serious performance improvement on FUSE and
literally nobody looked at this code or tried to track down the
profiling results it produces.
I'm not opposed to performance tests on principle. But these tests
aren't that, they just generate some profiling results and then spend a
bunch of time running various checks on the manifest that arv-mount
generated.
Remove them to save runtime and make it easier to run tests under
pytest. We can always write better performance tests later if we want.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/services/fuse/tests/performance/__init__.py b/services/fuse/tests/performance/__init__.py
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/services/fuse/tests/performance/performance_profiler.py b/services/fuse/tests/performance/performance_profiler.py
deleted file mode 120000
index 01a6805b32..0000000000
--- a/services/fuse/tests/performance/performance_profiler.py
+++ /dev/null
@@ -1 +0,0 @@
-../../../../sdk/python/tests/performance/performance_profiler.py
\ No newline at end of file
diff --git a/services/fuse/tests/performance/test_collection_performance.py b/services/fuse/tests/performance/test_collection_performance.py
deleted file mode 100644
index 525a779ca4..0000000000
--- a/services/fuse/tests/performance/test_collection_performance.py
+++ /dev/null
@@ -1,488 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-import arvados
-import arvados_fuse as fuse
-import llfuse
-import logging
-import os
-import sys
-import unittest
-
-from .. import run_test_server
-from ..mount_test_base import MountTestBase
-from ..slow_test import slow_test
-
-logger = logging.getLogger('arvados.arv-mount')
-
-from .performance_profiler import profiled
-
-def fuse_createCollectionWithMultipleBlocks(mounttmp, streams=1, files_per_stream=1, data='x'):
- class Test(unittest.TestCase):
- def runTest(self):
- self.createCollectionWithMultipleBlocks()
-
- @profiled
- def createCollectionWithMultipleBlocks(self):
- for i in range(0, streams):
- os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
-
- # Create files
- for j in range(0, files_per_stream):
- with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f:
- f.write(data)
-
- Test().runTest()
-
-def fuse_readContentsFromCollectionWithMultipleBlocks(mounttmp, streams=1, files_per_stream=1, data='x'):
- class Test(unittest.TestCase):
- def runTest(self):
- self.readContentsFromCollectionWithMultipleBlocks()
-
- @profiled
- def readContentsFromCollectionWithMultipleBlocks(self):
- for i in range(0, streams):
- d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i)))
- for j in range(0, files_per_stream):
- with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f:
- self.assertEqual(data, f.read())
-
- Test().runTest()
-
-def fuse_moveFileFromCollectionWithMultipleBlocks(mounttmp, stream, filename):
- class Test(unittest.TestCase):
- def runTest(self):
- self.moveFileFromCollectionWithMultipleBlocks()
-
- @profiled
- def moveFileFromCollectionWithMultipleBlocks(self):
- d1 = llfuse.listdir(os.path.join(mounttmp, stream))
- self.assertIn(filename, d1)
-
- os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename))
-
- d1 = llfuse.listdir(os.path.join(mounttmp))
- self.assertIn('moved_from_'+stream+'_'+filename, d1)
-
- d1 = llfuse.listdir(os.path.join(mounttmp, stream))
- self.assertNotIn(filename, d1)
-
- Test().runTest()
-
-def fuse_deleteFileFromCollectionWithMultipleBlocks(mounttmp, stream, filename):
- class Test(unittest.TestCase):
- def runTest(self):
- self.deleteFileFromCollectionWithMultipleBlocks()
-
- @profiled
- def deleteFileFromCollectionWithMultipleBlocks(self):
- os.remove(os.path.join(mounttmp, stream, filename))
-
- Test().runTest()
-
-# Create a collection with 2 streams, 3 files_per_stream, 2 blocks_per_file, 2**26 bytes_per_block
-class CreateCollectionWithMultipleBlocksAndMoveAndDeleteFile(MountTestBase):
- def setUp(self):
- super(CreateCollectionWithMultipleBlocksAndMoveAndDeleteFile, self).setUp()
-
- @slow_test
- def test_CreateCollectionWithManyBlocksAndMoveAndDeleteFile(self):
- collection = arvados.collection.Collection(api_client=self.api)
- collection.save_new()
-
- m = self.make_mount(fuse.CollectionDirectory)
- with llfuse.lock:
- m.new_collection(collection.api_response(), collection)
- self.assertTrue(m.writable())
-
- streams = 2
- files_per_stream = 3
- blocks_per_file = 2
- bytes_per_block = 2**26
-
- data = 'x' * blocks_per_file * bytes_per_block
-
- self.pool.apply(fuse_createCollectionWithMultipleBlocks, (self.mounttmp, streams, files_per_stream, data,))
-
- collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
-
- for i in range(0, streams):
- self.assertIn('./stream' + str(i), collection2["manifest_text"])
-
- for i in range(0, files_per_stream):
- self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"])
-
- # Read file contents
- self.pool.apply(fuse_readContentsFromCollectionWithMultipleBlocks, (self.mounttmp, streams, files_per_stream, data,))
-
- # Move file0.txt out of the streams into .
- for i in range(0, streams):
- self.pool.apply(fuse_moveFileFromCollectionWithMultipleBlocks, (self.mounttmp, 'stream'+str(i), 'file0.txt',))
-
- collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
-
- manifest_streams = collection2['manifest_text'].split('\n')
- self.assertEqual(4, len(manifest_streams))
-
- for i in range(0, streams):
- self.assertIn('file0.txt', manifest_streams[0])
-
- for i in range(0, streams):
- self.assertNotIn('file0.txt', manifest_streams[i+1])
-
- for i in range(0, streams):
- for j in range(1, files_per_stream):
- self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
-
- # Delete 'file1.txt' from all the streams
- for i in range(0, streams):
- self.pool.apply(fuse_deleteFileFromCollectionWithMultipleBlocks, (self.mounttmp, 'stream'+str(i), 'file1.txt'))
-
- collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
-
- manifest_streams = collection2['manifest_text'].split('\n')
- self.assertEqual(4, len(manifest_streams))
-
- for i in range(0, streams):
- self.assertIn('file0.txt', manifest_streams[0])
-
- self.assertNotIn('file1.txt', collection2['manifest_text'])
-
- for i in range(0, streams):
- for j in range(2, files_per_stream):
- self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
-
-
-def fuse_createCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, data='x'):
- class Test(unittest.TestCase):
- def runTest(self):
- self.createCollectionWithManyFiles()
-
- @profiled
- def createCollectionWithManyFiles(self):
- for i in range(0, streams):
- os.mkdir(os.path.join(mounttmp, "./stream" + str(i)))
-
- # Create files
- for j in range(0, files_per_stream):
- with open(os.path.join(mounttmp, "./stream" + str(i), "file" + str(j) +".txt"), "w") as f:
- f.write(data)
-
- Test().runTest()
-
-def fuse_readContentsFromCollectionWithManyFiles(mounttmp, streams=1, files_per_stream=1, data='x'):
- class Test(unittest.TestCase):
- def runTest(self):
- self.readContentsFromCollectionWithManyFiles()
-
- @profiled
- def readContentsFromCollectionWithManyFiles(self):
- for i in range(0, streams):
- d1 = llfuse.listdir(os.path.join(mounttmp, 'stream'+str(i)))
- for j in range(0, files_per_stream):
- with open(os.path.join(mounttmp, 'stream'+str(i), 'file'+str(i)+'.txt')) as f:
- self.assertEqual(data, f.read())
-
- Test().runTest()
-
-def fuse_moveFileFromCollectionWithManyFiles(mounttmp, stream, filename):
- class Test(unittest.TestCase):
- def runTest(self):
- self.moveFileFromCollectionWithManyFiles()
-
- @profiled
- def moveFileFromCollectionWithManyFiles(self):
- d1 = llfuse.listdir(os.path.join(mounttmp, stream))
- self.assertIn(filename, d1)
-
- os.rename(os.path.join(mounttmp, stream, filename), os.path.join(mounttmp, 'moved_from_'+stream+'_'+filename))
-
- d1 = llfuse.listdir(os.path.join(mounttmp))
- self.assertIn('moved_from_'+stream+'_'+filename, d1)
-
- d1 = llfuse.listdir(os.path.join(mounttmp, stream))
- self.assertNotIn(filename, d1)
-
- Test().runTest()
-
-def fuse_deleteFileFromCollectionWithManyFiles(mounttmp, stream, filename):
- class Test(unittest.TestCase):
- def runTest(self):
- self.deleteFileFromCollectionWithManyFiles()
-
- @profiled
- def deleteFileFromCollectionWithManyFiles(self):
- os.remove(os.path.join(mounttmp, stream, filename))
-
- Test().runTest()
-
-# Create a collection with two streams, each with 200 files
-class CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
- def setUp(self):
- super(CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
-
- @slow_test
- def test_CreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
- collection = arvados.collection.Collection(api_client=self.api)
- collection.save_new()
-
- m = self.make_mount(fuse.CollectionDirectory)
- with llfuse.lock:
- m.new_collection(collection.api_response(), collection)
- self.assertTrue(m.writable())
-
- streams = 2
- files_per_stream = 200
- data = 'x'
-
- self.pool.apply(fuse_createCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, data,))
-
- collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
-
- for i in range(0, streams):
- self.assertIn('./stream' + str(i), collection2["manifest_text"])
-
- for i in range(0, files_per_stream):
- self.assertIn('file' + str(i) + '.txt', collection2["manifest_text"])
-
- # Read file contents
- self.pool.apply(fuse_readContentsFromCollectionWithManyFiles, (self.mounttmp, streams, files_per_stream, data,))
-
- # Move file0.txt out of the streams into .
- for i in range(0, streams):
- self.pool.apply(fuse_moveFileFromCollectionWithManyFiles, (self.mounttmp, 'stream'+str(i), 'file0.txt',))
-
- collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
-
- manifest_streams = collection2['manifest_text'].split('\n')
- self.assertEqual(4, len(manifest_streams))
-
- for i in range(0, streams):
- self.assertIn('file0.txt', manifest_streams[0])
-
- for i in range(0, streams):
- self.assertNotIn('file0.txt', manifest_streams[i+1])
-
- for i in range(0, streams):
- for j in range(1, files_per_stream):
- self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
-
- # Delete 'file1.txt' from all the streams
- for i in range(0, streams):
- self.pool.apply(fuse_deleteFileFromCollectionWithManyFiles, (self.mounttmp, 'stream'+str(i), 'file1.txt'))
-
- collection2 = self.api.collections().get(uuid=collection.manifest_locator()).execute()
-
- manifest_streams = collection2['manifest_text'].split('\n')
- self.assertEqual(4, len(manifest_streams))
-
- for i in range(0, streams):
- self.assertIn('file0.txt', manifest_streams[0])
-
- self.assertNotIn('file1.txt', collection2['manifest_text'])
-
- for i in range(0, streams):
- for j in range(2, files_per_stream):
- self.assertIn('file' + str(j) + '.txt', manifest_streams[i+1])
-
-
-def magicDirTest_MoveFileFromCollection(mounttmp, collection1, collection2, stream, filename):
- class Test(unittest.TestCase):
- def runTest(self):
- self.magicDirTest_moveFileFromCollection()
-
- @profiled
- def magicDirTest_moveFileFromCollection(self):
- os.rename(os.path.join(mounttmp, collection1, filename), os.path.join(mounttmp, collection2, filename))
-
- Test().runTest()
-
-def magicDirTest_RemoveFileFromCollection(mounttmp, collection1, stream, filename):
- class Test(unittest.TestCase):
- def runTest(self):
- self.magicDirTest_removeFileFromCollection()
-
- @profiled
- def magicDirTest_removeFileFromCollection(self):
- os.remove(os.path.join(mounttmp, collection1, filename))
-
- Test().runTest()
-
-class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile(MountTestBase):
- def setUp(self):
- super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAndDeleteFile, self).setUp()
-
- @profiled
- def magicDirTest_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, data='x'):
- # Create collection
- collection = arvados.collection.Collection(api_client=self.api)
- for j in range(0, files_per_stream):
- with collection.open("file"+str(j)+".txt", "w") as f:
- f.write(data)
- collection.save_new()
- return collection
-
- @profiled
- def magicDirTest_readCollectionContents(self, collection, streams=1, files_per_stream=1, data='x'):
- mount_ls = os.listdir(os.path.join(self.mounttmp, collection))
-
- files = {}
- for j in range(0, files_per_stream):
- files[os.path.join(self.mounttmp, collection, 'file'+str(j)+'.txt')] = data
-
- for k, v in files.items():
- with open(os.path.join(self.mounttmp, collection, k)) as f:
- self.assertEqual(v, f.read())
-
- @slow_test
- def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAndDeleteFile(self):
- streams = 2
- files_per_stream = 200
- data = 'x'
-
- collection1 = self.magicDirTest_createCollectionWithManyFiles()
- # Create collection with multiple files
- collection2 = self.magicDirTest_createCollectionWithManyFiles(streams, files_per_stream, data)
-
- # Mount FuseMagicDir
- self.make_mount(fuse.MagicDirectory)
-
- self.magicDirTest_readCollectionContents(collection2.manifest_locator(), streams, files_per_stream, data)
-
- # Move file0.txt out of the collection2 into collection1
- self.pool.apply(magicDirTest_MoveFileFromCollection, (self.mounttmp, collection2.manifest_locator(),
- collection1.manifest_locator(), 'stream0', 'file0.txt',))
- updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
- self.assertFalse('file0.txt' in updated_collection['manifest_text'])
- self.assertTrue('file1.txt' in updated_collection['manifest_text'])
-
- # Delete file1.txt from collection2
- self.pool.apply(magicDirTest_RemoveFileFromCollection, (self.mounttmp, collection2.manifest_locator(), 'stream0', 'file1.txt',))
- updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
- self.assertFalse('file1.txt' in updated_collection['manifest_text'])
- self.assertTrue('file2.txt' in updated_collection['manifest_text'])
-
-
-def magicDirTest_MoveAllFilesFromCollection(mounttmp, from_collection, to_collection, stream, files_per_stream):
- class Test(unittest.TestCase):
- def runTest(self):
- self.magicDirTest_moveAllFilesFromCollection()
-
- @profiled
- def magicDirTest_moveAllFilesFromCollection(self):
- for j in range(0, files_per_stream):
- os.rename(os.path.join(mounttmp, from_collection, 'file'+str(j)+'.txt'), os.path.join(mounttmp, to_collection, 'file'+str(j)+'.txt'))
-
- Test().runTest()
-
-class UsingMagicDir_CreateCollectionWithManyFilesAndMoveAllFilesIntoAnother(MountTestBase):
- def setUp(self):
- super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveAllFilesIntoAnother, self).setUp()
-
- @profiled
- def magicDirTestMoveAllFiles_createCollectionWithManyFiles(self, streams=0, files_per_stream=0,
- blocks_per_file=0, bytes_per_block=0, data='x'):
- # Create collection
- collection = arvados.collection.Collection(api_client=self.api)
- for j in range(0, files_per_stream):
- with collection.open("file"+str(j)+".txt", "w") as f:
- f.write(data)
- collection.save_new()
- return collection
-
- @slow_test
- def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveAllFilesIntoAnother(self):
- streams = 2
- files_per_stream = 200
- data = 'x'
-
- collection1 = self.magicDirTestMoveAllFiles_createCollectionWithManyFiles()
- # Create collection with multiple files
- collection2 = self.magicDirTestMoveAllFiles_createCollectionWithManyFiles(streams, files_per_stream, data)
-
- # Mount FuseMagicDir
- self.make_mount(fuse.MagicDirectory)
-
- # Move all files from collection2 into collection1
- self.pool.apply(magicDirTest_MoveAllFilesFromCollection, (self.mounttmp, collection2.manifest_locator(),
- collection1.manifest_locator(), 'stream0', files_per_stream,))
-
- updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
- file_names = ["file%i.txt" % i for i in range(0, files_per_stream)]
- for name in file_names:
- self.assertFalse(name in updated_collection['manifest_text'])
-
- updated_collection = self.api.collections().get(uuid=collection1.manifest_locator()).execute()
- for name in file_names:
- self.assertTrue(name in updated_collection['manifest_text'])
-
-
-# Move one file at a time from one collection into another
-class UsingMagicDir_CreateCollectionWithManyFilesAndMoveEachFileIntoAnother(MountTestBase):
- def setUp(self):
- super(UsingMagicDir_CreateCollectionWithManyFilesAndMoveEachFileIntoAnother, self).setUp()
-
- @profiled
- def magicDirTestMoveFiles_createCollectionWithManyFiles(self, streams=0, files_per_stream=0, data='x'):
- # Create collection
- collection = arvados.collection.Collection(api_client=self.api)
- for j in range(0, files_per_stream):
- with collection.open("file"+str(j)+".txt", "w") as f:
- f.write(data)
- collection.save_new()
- return collection
-
- def magicDirTestMoveFiles_oneEachIntoAnother(self, from_collection, to_collection, files_per_stream):
- for j in range(0, files_per_stream):
- self.pool.apply(magicDirTest_MoveFileFromCollection, (self.mounttmp, from_collection.manifest_locator(),
- to_collection.manifest_locator(), 'stream0', 'file'+str(j)+'.txt',))
-
- @slow_test
- def test_UsingMagicDirCreateCollectionWithManyFilesAndMoveEachFileIntoAnother(self):
- streams = 2
- files_per_stream = 200
- data = 'x'
-
- collection1 = self.magicDirTestMoveFiles_createCollectionWithManyFiles()
- # Create collection with multiple files
- collection2 = self.magicDirTestMoveFiles_createCollectionWithManyFiles(streams, files_per_stream, data)
-
- # Mount FuseMagicDir
- self.make_mount(fuse.MagicDirectory)
-
- # Move all files from collection2 into collection1
- self.magicDirTestMoveFiles_oneEachIntoAnother(collection2, collection1, files_per_stream)
-
- updated_collection = self.api.collections().get(uuid=collection2.manifest_locator()).execute()
- file_names = ["file%i.txt" % i for i in range(0, files_per_stream)]
- for name in file_names:
- self.assertFalse(name in updated_collection['manifest_text'])
-
- updated_collection = self.api.collections().get(uuid=collection1.manifest_locator()).execute()
- for name in file_names:
- self.assertTrue(name in updated_collection['manifest_text'])
-
-class FuseListLargeProjectContents(MountTestBase):
- @profiled
- def getProjectWithManyCollections(self):
- project_contents = llfuse.listdir(self.mounttmp)
- self.assertEqual(201, len(project_contents))
- self.assertIn('Collection_1', project_contents)
-
- @profiled
- def listContentsInProjectWithManyCollections(self):
- project_contents = llfuse.listdir(self.mounttmp)
- self.assertEqual(201, len(project_contents))
- self.assertIn('Collection_1', project_contents)
-
- for collection_name in project_contents:
- collection_contents = llfuse.listdir(os.path.join(self.mounttmp, collection_name))
- self.assertIn('baz', collection_contents)
-
- @slow_test
- def test_listLargeProjectContents(self):
- self.make_mount(fuse.ProjectDirectory,
- project_object=run_test_server.fixture('groups')['project_with_201_collections'])
- self.getProjectWithManyCollections()
- self.listContentsInProjectWithManyCollections()
commit 2f7673c45f9945ba04eabba97a4bdf4d3d340688
Author: Brett Smith <brett.smith at curii.com>
Date: Fri May 3 12:02:56 2024 -0400
21207: Rename CWL standalone test functions
These functions are not picked up by the unittest runner (since they're
not TestCase methods) but are picked up by pytest (since they start with
test_*). Rename them to retain existing behavior across test runners.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/tests/test_copy_deps.py b/sdk/cwl/tests/test_copy_deps.py
index 28a5915b11..8ad735fddc 100644
--- a/sdk/cwl/tests/test_copy_deps.py
+++ b/sdk/cwl/tests/test_copy_deps.py
@@ -73,7 +73,7 @@ def check_contents(group, wf_uuid):
raise Exception("Couldn't find collection containing expected "+expect_file)
-def test_create():
+def check_create():
group = api.groups().create(body={"group": {"name": "test-19070-project-1", "group_class": "project"}}, ensure_unique_name=True).execute()
try:
contents = api.groups().contents(uuid=group["uuid"]).execute()
@@ -90,7 +90,7 @@ def test_create():
api.groups().delete(uuid=group["uuid"]).execute()
-def test_update():
+def check_update():
group = api.groups().create(body={"group": {"name": "test-19070-project-2", "group_class": "project"}}, ensure_unique_name=True).execute()
try:
contents = api.groups().contents(uuid=group["uuid"]).execute()
@@ -132,7 +132,7 @@ def test_update():
api.groups().delete(uuid=group["uuid"]).execute()
-def test_execute():
+def check_execute():
group = api.groups().create(body={"group": {"name": "test-19070-project-3", "group_class": "project"}}, ensure_unique_name=True).execute()
try:
contents = api.groups().contents(uuid=group["uuid"]).execute()
@@ -193,6 +193,6 @@ def test_execute():
api.groups().delete(uuid=group["uuid"]).execute()
if __name__ == '__main__':
- test_create()
- test_update()
- test_execute()
+ check_create()
+ check_update()
+ check_execute()
diff --git a/sdk/cwl/tests/test_set_output_prop.py b/sdk/cwl/tests/test_set_output_prop.py
index 3219eac989..0e829eeb92 100644
--- a/sdk/cwl/tests/test_set_output_prop.py
+++ b/sdk/cwl/tests/test_set_output_prop.py
@@ -7,7 +7,7 @@ import subprocess
api = arvados.api()
-def test_execute():
+def check_execute():
group = api.groups().create(body={"group": {"name": "test-17004-project", "group_class": "project"}}, ensure_unique_name=True).execute()
try:
contents = api.groups().contents(uuid=group["uuid"]).execute()
@@ -34,4 +34,4 @@ def test_execute():
api.groups().delete(uuid=group["uuid"]).execute()
if __name__ == '__main__':
- test_execute()
+ check_execute()
commit 6ff9889d875d23780114ad74d90e07c890fce8d6
Author: Brett Smith <brett.smith at curii.com>
Date: Fri May 3 09:55:15 2024 -0400
21207: Set up MockedPut._state with a deepcopy
Otherwise its internal state could get into EMPTY_STATE and contaminate
other tests.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index e3f7c9d446..e8a3e65bdc 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -6,6 +6,7 @@
import apiclient
import ciso8601
+import copy
import datetime
import json
import logging
@@ -568,7 +569,7 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
class CachedManifestValidationTest(ArvadosBaseTestCase):
class MockedPut(arv_put.ArvPutUploadJob):
def __init__(self, cached_manifest=None):
- self._state = arv_put.ArvPutUploadJob.EMPTY_STATE
+ self._state = copy.deepcopy(arv_put.ArvPutUploadJob.EMPTY_STATE)
self._state['manifest'] = cached_manifest
self._api_client = mock.MagicMock()
self.logger = mock.MagicMock()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list