[arvados] created: 2.7.0-6207-ge512d73d1f
git repository hosting
git at public.arvados.org
Sun Mar 17 00:09:34 UTC 2024
at e512d73d1fc576f468510fcc67b185f01a70c0c2 (commit)
commit e512d73d1fc576f468510fcc67b185f01a70c0c2
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Mar 16 19:23:44 2024 -0400
21601: Specify Python interdependencies from Git instead of <=
This helps ensure that when we install packages from Git (which we do
for various builds) we get the package from the same commit, and don't
accidentally rely on an older version than intended.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index b594f88a7c..1ec7c5487a 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -123,6 +123,18 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ if ENV_VERSION or (REPO_PATH is None):
+ dep_fmt = f'{{}}=={version}'.format
+ else:
+ def dep_fmt(pkg):
+ pkg_path = REPO_PATH / PACKAGE_SRCPATH_MAP[pkg]
+ return f'{pkg} @ {pkg_path.as_uri()}'
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield dep_fmt(dep_pkg)
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 043b52cb81..551bd964b1 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -9,16 +9,9 @@ import sys
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_cwl")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
- pysdk_dep = "=={}".format(version)
-else:
- # On dev releases, arvados-python-client may have a different timestamp
- pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='arvados-cwl-runner',
version=version,
@@ -36,10 +29,9 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
+ *arvados_version.iter_dependencies(version),
'cwltool==3.1.20230601100705',
'schema-salad==8.4.20230601112322',
- 'arvados-python-client{}'.format(pysdk_dep),
- 'crunchstat-summary{}'.format(pysdk_dep),
'ciso8601 >= 2.0.0',
'networkx < 2.6',
'msgpack==1.0.3',
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index b594f88a7c..1ec7c5487a 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -123,6 +123,18 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ if ENV_VERSION or (REPO_PATH is None):
+ dep_fmt = f'{{}}=={version}'.format
+ else:
+ def dep_fmt(pkg):
+ pkg_path = REPO_PATH / PACKAGE_SRCPATH_MAP[pkg]
+ return f'{pkg} @ {pkg_path.as_uri()}'
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield dep_fmt(dep_pkg)
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index 00429de844..e13e51609a 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -12,12 +12,10 @@ from pathlib import Path
from setuptools import setup, find_packages
from setuptools.command import build_py
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados")
+version = arvados_version.get_version()
short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
class BuildPython(build_py.build_py):
"""Extend setuptools `build_py` to generate API documentation
@@ -111,6 +109,7 @@ setup(name='arvados-python-client',
('share/doc/arvados-python-client', ['LICENSE-2.0.txt', 'README.rst']),
],
install_requires=[
+ *arvados_version.iter_dependencies(version),
'ciso8601 >=2.0.0',
'future',
'google-api-core <2.11.0', # 2.11.0rc1 is incompatible with google-auth<2
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index b594f88a7c..1ec7c5487a 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -123,6 +123,18 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ if ENV_VERSION or (REPO_PATH is None):
+ dep_fmt = f'{{}}=={version}'.format
+ else:
+ def dep_fmt(pkg):
+ pkg_path = REPO_PATH / PACKAGE_SRCPATH_MAP[pkg]
+ return f'{pkg} @ {pkg_path.as_uri()}'
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield dep_fmt(dep_pkg)
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 565fdcd1ea..9c69879b45 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -10,12 +10,10 @@ import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_docker")
+version = arvados_version.get_version()
short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name="arvados-docker-cleaner",
version=version,
@@ -33,6 +31,7 @@ setup(name="arvados-docker-cleaner",
('share/doc/arvados-docker-cleaner', ['agpl-3.0.txt', 'arvados-docker-cleaner.service']),
],
install_requires=[
+ *arvados_version.iter_dependencies(version),
'docker>=6.1.0',
'setuptools',
],
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index b594f88a7c..1ec7c5487a 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -123,6 +123,18 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ if ENV_VERSION or (REPO_PATH is None):
+ dep_fmt = f'{{}}=={version}'.format
+ else:
+ def dep_fmt(pkg):
+ pkg_path = REPO_PATH / PACKAGE_SRCPATH_MAP[pkg]
+ return f'{pkg} @ {pkg_path.as_uri()}'
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield dep_fmt(dep_pkg)
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index 80a26980f1..07c9dcf5e1 100644
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -10,17 +10,10 @@ import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_fuse")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
- pysdk_dep = "=={}".format(version)
-else:
- # On dev releases, arvados-python-client may have a different timestamp
- pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='arvados_fuse',
version=version,
@@ -39,7 +32,7 @@ setup(name='arvados_fuse',
('share/doc/arvados_fuse', ['agpl-3.0.txt', 'README.rst']),
],
install_requires=[
- 'arvados-python-client{}'.format(pysdk_dep),
+ *arvados_version.iter_dependencies(version),
'llfuse >= 1.3.6',
'future',
'python-daemon',
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index b594f88a7c..1ec7c5487a 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -123,6 +123,18 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ if ENV_VERSION or (REPO_PATH is None):
+ dep_fmt = f'{{}}=={version}'.format
+ else:
+ def dep_fmt(pkg):
+ pkg_path = REPO_PATH / PACKAGE_SRCPATH_MAP[pkg]
+ return f'{pkg} @ {pkg_path.as_uri()}'
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield dep_fmt(dep_pkg)
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 4bb9df2ba9..24a6bf5e4f 100755
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -10,17 +10,10 @@ import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "crunchstat_summary")
-if os.environ.get('ARVADOS_BUILDING_VERSION', False):
- pysdk_dep = "=={}".format(version)
-else:
- # On dev releases, arvados-python-client may have a different timestamp
- pysdk_dep = "<={}".format(version)
+version = arvados_version.get_version()
short_tests_only = arvados_version.short_tests_only()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='crunchstat_summary',
version=version,
@@ -39,7 +32,7 @@ setup(name='crunchstat_summary',
('share/doc/crunchstat_summary', ['agpl-3.0.txt']),
],
install_requires=[
- 'arvados-python-client{}'.format(pysdk_dep),
+ *arvados_version.iter_dependencies(version),
],
python_requires="~=3.8",
test_suite='tests',
diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
index b594f88a7c..1ec7c5487a 100644
--- a/tools/user-activity/arvados_version.py
+++ b/tools/user-activity/arvados_version.py
@@ -123,6 +123,18 @@ def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
save_version(setup_dir, module, version)
return version
+def iter_dependencies(version=None):
+ if version is None:
+ version = get_version()
+ if ENV_VERSION or (REPO_PATH is None):
+ dep_fmt = f'{{}}=={version}'.format
+ else:
+ def dep_fmt(pkg):
+ pkg_path = REPO_PATH / PACKAGE_SRCPATH_MAP[pkg]
+ return f'{pkg} @ {pkg_path.as_uri()}'
+ for dep_pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ()):
+ yield dep_fmt(dep_pkg)
+
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
print(get_version())
diff --git a/tools/user-activity/setup.py b/tools/user-activity/setup.py
index 4b7ec16b93..8611fa47a1 100755
--- a/tools/user-activity/setup.py
+++ b/tools/user-activity/setup.py
@@ -10,11 +10,9 @@ import re
from setuptools import setup, find_packages
-SETUP_DIR = os.path.dirname(__file__) or '.'
-README = os.path.join(SETUP_DIR, 'README.rst')
-
import arvados_version
-version = arvados_version.get_version(SETUP_DIR, "arvados_user_activity")
+version = arvados_version.get_version()
+README = os.path.join(arvados_version.SETUP_DIR, 'README.rst')
setup(name='arvados-user-activity',
version=version,
@@ -31,7 +29,7 @@ setup(name='arvados-user-activity',
('share/doc/arvados_user_activity', ['agpl-3.0.txt']),
],
install_requires=[
- 'arvados-python-client >= 2.2.0.dev20201118185221',
+ *arvados_version.iter_dependencies(version),
],
python_requires="~=3.8",
zip_safe=True,
commit e3da491220eccd56c5de72b84713e3b911d5db85
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Mar 16 19:10:15 2024 -0400
21601: Move --short-tests-only check to arvados_version
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index e3d66aa472..00429de844 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -17,11 +17,7 @@ README = os.path.join(SETUP_DIR, 'README.rst')
import arvados_version
version = arvados_version.get_version(SETUP_DIR, "arvados")
-
-short_tests_only = False
-if '--short-tests-only' in sys.argv:
- short_tests_only = True
- sys.argv.remove('--short-tests-only')
+short_tests_only = arvados_version.short_tests_only()
class BuildPython(build_py.build_py):
"""Extend setuptools `build_py` to generate API documentation
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/services/dockercleaner/setup.py b/services/dockercleaner/setup.py
index 2b386c70b4..565fdcd1ea 100644
--- a/services/dockercleaner/setup.py
+++ b/services/dockercleaner/setup.py
@@ -15,11 +15,7 @@ README = os.path.join(SETUP_DIR, 'README.rst')
import arvados_version
version = arvados_version.get_version(SETUP_DIR, "arvados_docker")
-
-short_tests_only = False
-if '--short-tests-only' in sys.argv:
- short_tests_only = True
- sys.argv.remove('--short-tests-only')
+short_tests_only = arvados_version.short_tests_only()
setup(name="arvados-docker-cleaner",
version=version,
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/services/fuse/setup.py b/services/fuse/setup.py
index b04829652e..80a26980f1 100644
--- a/services/fuse/setup.py
+++ b/services/fuse/setup.py
@@ -20,11 +20,7 @@ if os.environ.get('ARVADOS_BUILDING_VERSION', False):
else:
# On dev releases, arvados-python-client may have a different timestamp
pysdk_dep = "<={}".format(version)
-
-short_tests_only = False
-if '--short-tests-only' in sys.argv:
- short_tests_only = True
- sys.argv.remove('--short-tests-only')
+short_tests_only = arvados_version.short_tests_only()
setup(name='arvados_fuse',
version=version,
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 98be9f2702..4bb9df2ba9 100755
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -20,11 +20,7 @@ if os.environ.get('ARVADOS_BUILDING_VERSION', False):
else:
# On dev releases, arvados-python-client may have a different timestamp
pysdk_dep = "<={}".format(version)
-
-short_tests_only = False
-if '--short-tests-only' in sys.argv:
- short_tests_only = True
- sys.argv.remove('--short-tests-only')
+short_tests_only = arvados_version.short_tests_only()
setup(name='crunchstat_summary',
version=version,
diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
index 020ec38738..b594f88a7c 100644
--- a/tools/user-activity/arvados_version.py
+++ b/tools/user-activity/arvados_version.py
@@ -71,6 +71,14 @@ if REPO_PATH is None:
if (SETUP_DIR / mod_name).is_dir()
)
+def short_tests_only(arglist=sys.argv):
+ try:
+ arglist.remove('--short-tests-only')
+ except ValueError:
+ return False
+ else:
+ return True
+
def git_log_output(path, *args):
return subprocess.check_output(
['git', '-C', str(REPO_PATH),
commit 581897ebb4b912049fce721a825f63910b61017c
Author: Brett Smith <brett.smith at curii.com>
Date: Sat Mar 16 19:02:59 2024 -0400
21601: Make arvados_version.py more declarative
The main goal of this change is to introduce the metadata maps near the
top of the file, which we will use to build additional
functionality. The rest of the changes are just modernization or
clean-up based on that.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/cwl/arvados_version.py b/sdk/cwl/arvados_version.py
index a78dbfcf2b..020ec38738 100644
--- a/sdk/cwl/arvados_version.py
+++ b/sdk/cwl/arvados_version.py
@@ -1,63 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../python")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../tools/crunchstat-summary")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
-
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
-
- return read_version(setup_dir, module)
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
- print(get_version(SETUP_DIR, "arvados_cwl"))
+ print(get_version())
diff --git a/sdk/python/arvados_version.py b/sdk/python/arvados_version.py
index 092131d930..020ec38738 100644
--- a/sdk/python/arvados_version.py
+++ b/sdk/python/arvados_version.py
@@ -1,61 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
-
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
-
- return read_version(setup_dir, module)
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
if __name__ == '__main__':
- print(get_version(SETUP_DIR, "arvados"))
+ print(get_version())
diff --git a/services/dockercleaner/arvados_version.py b/services/dockercleaner/arvados_version.py
index 38e6f564e7..020ec38738 100644
--- a/services/dockercleaner/arvados_version.py
+++ b/services/dockercleaner/arvados_version.py
@@ -1,57 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
-
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
- return read_version(setup_dir, module)
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version())
diff --git a/services/fuse/arvados_version.py b/services/fuse/arvados_version.py
index d8eec3d9ee..020ec38738 100644
--- a/services/fuse/arvados_version.py
+++ b/services/fuse/arvados_version.py
@@ -1,58 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
-
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
- return read_version(setup_dir, module)
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version())
diff --git a/tools/crunchstat-summary/arvados_version.py b/tools/crunchstat-summary/arvados_version.py
index d8eec3d9ee..020ec38738 100644
--- a/tools/crunchstat-summary/arvados_version.py
+++ b/tools/crunchstat-summary/arvados_version.py
@@ -1,58 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
-
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
- return read_version(setup_dir, module)
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version())
diff --git a/tools/user-activity/arvados_version.py b/tools/user-activity/arvados_version.py
index d8eec3d9ee..020ec38738 100644
--- a/tools/user-activity/arvados_version.py
+++ b/tools/user-activity/arvados_version.py
@@ -1,58 +1,120 @@
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+#
+# This file runs in one of three modes:
+#
+# 1. If the ARVADOS_BUILDING_VERSION environment variable is set, it writes
+# _version.py and generates dependencies based on that value.
+# 2. If running from an arvados Git checkout, it writes _version.py
+# and generates dependencies from Git.
+# 3. Otherwise, we expect this is source previously generated from Git, and
+# it reads _version.py and generates dependencies from it.
-import subprocess
-import time
import os
-import re
+import runpy
+import subprocess
import sys
-SETUP_DIR = os.path.dirname(os.path.abspath(__file__))
-VERSION_PATHS = {
- SETUP_DIR,
- os.path.abspath(os.path.join(SETUP_DIR, "../../sdk/python")),
- os.path.abspath(os.path.join(SETUP_DIR, "../../build/version-at-commit.sh"))
- }
+from pathlib import Path
+
+# These maps explain the relationships between different Python modules in
+# the arvados repository. We use these to help generate setup.py.
+PACKAGE_DEPENDENCY_MAP = {
+ 'arvados-cwl-runner': ['arvados-python-client', 'crunchstat_summary'],
+ 'arvados-user-activity': ['arvados-python-client'],
+ 'arvados_fuse': ['arvados-python-client'],
+ 'crunchstat_summary': ['arvados-python-client'],
+}
+PACKAGE_MODULE_MAP = {
+ 'arvados-cwl-runner': 'arvados_cwl',
+ 'arvados-docker-cleaner': 'arvados_docker',
+ 'arvados-python-client': 'arvados',
+ 'arvados-user-activity': 'arvados_user_activity',
+ 'arvados_fuse': 'arvados_fuse',
+ 'crunchstat_summary': 'crunchstat_summary',
+}
+PACKAGE_SRCPATH_MAP = {
+ 'arvados-cwl-runner': Path('sdk', 'cwl'),
+ 'arvados-docker-cleaner': Path('services', 'dockercleaner'),
+ 'arvados-python-client': Path('sdk', 'python'),
+ 'arvados-user-activity': Path('tools', 'user-activity'),
+ 'arvados_fuse': Path('services', 'fuse'),
+ 'crunchstat_summary': Path('tools', 'crunchstat-summary'),
+}
+
+ENV_VERSION = os.environ.get("ARVADOS_BUILDING_VERSION")
+SETUP_DIR = Path(__file__).absolute().parent
+try:
+ REPO_PATH = Path(subprocess.check_output(
+ ['git', '-C', str(SETUP_DIR), 'rev-parse', '--show-toplevel'],
+ stderr=subprocess.DEVNULL,
+ text=True,
+ ).rstrip('\n'))
+except (subprocess.CalledProcessError, OSError):
+ REPO_PATH = None
+else:
+ # Verify this is the arvados monorepo
+ if all((REPO_PATH / path).exists() for path in PACKAGE_SRCPATH_MAP.values()):
+ PACKAGE_NAME, = (
+ pkg_name for pkg_name, path in PACKAGE_SRCPATH_MAP.items()
+ if (REPO_PATH / path) == SETUP_DIR
+ )
+ MODULE_NAME = PACKAGE_MODULE_MAP[PACKAGE_NAME]
+ VERSION_SCRIPT_PATH = Path(REPO_PATH, 'build', 'version-at-commit.sh')
+ else:
+ REPO_PATH = None
+if REPO_PATH is None:
+ (PACKAGE_NAME, MODULE_NAME), = (
+ (pkg_name, mod_name)
+ for pkg_name, mod_name in PACKAGE_MODULE_MAP.items()
+ if (SETUP_DIR / mod_name).is_dir()
+ )
+
+def git_log_output(path, *args):
+ return subprocess.check_output(
+ ['git', '-C', str(REPO_PATH),
+ 'log', '--first-parent', '--max-count=1',
+ *args, str(path)],
+ text=True,
+ ).rstrip('\n')
def choose_version_from():
- ts = {}
- for path in VERSION_PATHS:
- ts[subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%ct', path]).strip()] = path
-
- sorted_ts = sorted(ts.items())
- getver = sorted_ts[-1][1]
- print("Using "+getver+" for version number calculation of "+SETUP_DIR, file=sys.stderr)
+ ver_paths = [SETUP_DIR, VERSION_SCRIPT_PATH, *(
+ PACKAGE_SRCPATH_MAP[pkg]
+ for pkg in PACKAGE_DEPENDENCY_MAP.get(PACKAGE_NAME, ())
+ )]
+ getver = max(ver_paths, key=lambda path: git_log_output(path, '--format=format:%ct'))
+ print(f"Using {getver} for version number calculation of {SETUP_DIR}", file=sys.stderr)
return getver
def git_version_at_commit():
curdir = choose_version_from()
- myhash = subprocess.check_output(['git', 'log', '-n1', '--first-parent',
- '--format=%H', curdir]).strip()
- myversion = subprocess.check_output([SETUP_DIR+'/../../build/version-at-commit.sh', myhash]).strip().decode()
- return myversion
+ myhash = git_log_output(curdir, '--format=%H')
+ return subprocess.check_output(
+ [str(VERSION_SCRIPT_PATH), myhash],
+ text=True,
+ ).rstrip('\n')
def save_version(setup_dir, module, v):
- v = v.replace("~dev", ".dev").replace("~rc", "rc")
- with open(os.path.join(setup_dir, module, "_version.py"), 'wt') as fp:
- return fp.write("__version__ = '%s'\n" % v)
+ with Path(setup_dir, module, '_version.py').open('w') as fp:
+ print(f"__version__ = {v!r}", file=fp)
def read_version(setup_dir, module):
- with open(os.path.join(setup_dir, module, "_version.py"), 'rt') as fp:
- return re.match("__version__ = '(.*)'$", fp.read()).groups()[0]
-
-def get_version(setup_dir, module):
- env_version = os.environ.get("ARVADOS_BUILDING_VERSION")
+ file_vars = runpy.run_path(Path(setup_dir, module, '_version.py'))
+ return file_vars['__version__']
- if env_version:
- save_version(setup_dir, module, env_version)
+def get_version(setup_dir=SETUP_DIR, module=MODULE_NAME):
+ if ENV_VERSION:
+ version = ENV_VERSION
+ elif REPO_PATH is None:
+ return read_version(setup_dir, module)
else:
- try:
- save_version(setup_dir, module, git_version_at_commit())
- except (subprocess.CalledProcessError, OSError) as err:
- print("ERROR: {0}".format(err), file=sys.stderr)
- pass
+ version = git_version_at_commit()
+ version = version.replace("~dev", ".dev").replace("~rc", "rc")
+ save_version(setup_dir, module, version)
+ return version
- return read_version(setup_dir, module)
+# Called from calculate_python_sdk_cwl_package_versions() in run-library.sh
+if __name__ == '__main__':
+ print(get_version())
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list