[arvados] created: 2.7.0-655-g50e7dff1be
git repository hosting
git at public.arvados.org
Mon Sep 25 20:55:25 UTC 2023
at 50e7dff1be4d133ce1a599954699bc94bff51c68 (commit)
commit 50e7dff1be4d133ce1a599954699bc94bff51c68
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Sep 25 16:26:46 2023 -0400
20885: Update PySDK docstrings after pdoc migration
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index c51be82b20..94b8013997 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -207,45 +207,41 @@ def api_client(
Arguments:
- version: str
- : A string naming the version of the Arvados API to use.
+ * version: str --- A string naming the version of the Arvados API to use.
- discoveryServiceUrl: str
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`.
+ * discoveryServiceUrl: str --- The URL used to discover APIs passed
+ directly to `googleapiclient.discovery.build`.
- token: str
- : The authentication token to send with each API call.
+ * token: str --- The authentication token to send with each API call.
Keyword-only arguments:
- cache: bool
- : If true, loads the API discovery document from, or saves it to, a cache
- on disk (located at `~/.cache/arvados/discovery`).
+ * cache: bool --- If true, loads the API discovery document from, or
+ saves it to, a cache on disk (located at
+ `~/.cache/arvados/discovery`).
- http: httplib2.Http | None
- : The HTTP client object the API client object will use to make requests.
- If not provided, this function will build its own to use. Either way, the
- object will be patched as part of the build process.
+ * http: httplib2.Http | None --- The HTTP client object the API client
+ object will use to make requests. If not provided, this function will
+ build its own to use. Either way, the object will be patched as part
+ of the build process.
- insecure: bool
- : If true, ignore SSL certificate validation errors. Default `False`.
+ * insecure: bool --- If true, ignore SSL certificate validation
+ errors. Default `False`.
- num_retries: int
- : The number of times to retry each API request if it encounters a
- temporary failure. Default 10.
+ * num_retries: int --- The number of times to retry each API request if
+ it encounters a temporary failure. Default 10.
- request_id: str | None
- : Default `X-Request-Id` header value for outgoing requests that
- don't already provide one. If `None` or omitted, generate a random
- ID. When retrying failed requests, the same ID is used on all
- attempts.
+ * request_id: str | None --- Default `X-Request-Id` header value for
+ outgoing requests that don't already provide one. If `None` or
+ omitted, generate a random ID. When retrying failed requests, the same
+ ID is used on all attempts.
- timeout: int
- : A timeout value for HTTP requests in seconds. Default 300 (5 minutes).
+ * timeout: int --- A timeout value for HTTP requests in seconds. Default
+ 300 (5 minutes).
Additional keyword arguments will be passed directly to
`googleapiclient.discovery.build`.
+
"""
if http is None:
http = httplib2.Http(
@@ -322,22 +318,19 @@ def normalize_api_kwargs(
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- discoveryServiceUrl: str | None
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`. It is an error to pass both
- `discoveryServiceUrl` and `host`.
+ * discoveryServiceUrl: str | None --- The URL used to discover APIs
+ passed directly to `googleapiclient.discovery.build`. It is an error
+ to pass both `discoveryServiceUrl` and `host`.
- host: str | None
- : The hostname and optional port number of the Arvados API server. Used to
- build `discoveryServiceUrl`. It is an error to pass both
- `discoveryServiceUrl` and `host`.
+ * host: str | None --- The hostname and optional port number of the
+ Arvados API server. Used to build `discoveryServiceUrl`. It is an
+ error to pass both `discoveryServiceUrl` and `host`.
- token: str
- : The authentication token to send with each API call.
+ * token: str --- The authentication token to send with each API call.
Additional keyword arguments will be included in the return value.
"""
@@ -378,14 +371,15 @@ def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and
- optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls
- `arvados.config.settings` to get these parameters from user configuration.
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, calls
+ `arvados.config.settings` to get these parameters from user
+ configuration.
Additional keyword arguments will be included in the return value.
"""
@@ -428,19 +422,18 @@ def api(version=None, cache=True, host=None, token=None, insecure=False,
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- host: str | None
- : The hostname and optional port number of the Arvados API server.
+ * host: str | None --- The hostname and optional port number of the
+ Arvados API server.
- token: str | None
- : The authentication token to send with each API call.
+ * token: str | None --- The authentication token to send with each API
+ call.
- discoveryServiceUrl: str | None
- : The URL used to discover APIs passed directly to
- `googleapiclient.discovery.build`.
+ * discoveryServiceUrl: str | None --- The URL used to discover APIs
+ passed directly to `googleapiclient.discovery.build`.
If `host`, `token`, and `discoveryServiceUrl` are all omitted, `host` and
`token` will be loaded from the user's configuration. Otherwise, you must
@@ -479,14 +472,15 @@ def api_from_config(version=None, apiconfig=None, **kwargs):
Arguments:
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ 'v1'.
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and
- optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls
- `arvados.config.settings` to get these parameters from user configuration.
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, calls
+ `arvados.config.settings` to get these parameters from user
+ configuration.
Other arguments are passed directly to `api_client`. See that function's
docstring for more information about their meaning.
diff --git a/sdk/python/arvados/retry.py b/sdk/python/arvados/retry.py
index 2168146a4b..ea8a6f65af 100644
--- a/sdk/python/arvados/retry.py
+++ b/sdk/python/arvados/retry.py
@@ -49,34 +49,29 @@ class RetryLoop(object):
Arguments:
- num_retries: int
- : The maximum number of times to retry the loop if it
- doesn't succeed. This means the loop body could run at most
+ * num_retries: int --- The maximum number of times to retry the loop if
+ it doesn't succeed. This means the loop body could run at most
`num_retries + 1` times.
- success_check: Callable
- : This is a function that will be called each
- time the loop saves a result. The function should return
- `True` if the result indicates the code succeeded, `False` if it
- represents a permanent failure, and `None` if it represents a
- temporary failure. If no function is provided, the loop will
- end after any result is saved.
-
- backoff_start: float
- : The number of seconds that must pass before the loop's second
- iteration. Default 0, which disables all waiting.
-
- backoff_growth: float
- : The wait time multiplier after each iteration.
- Default 2 (i.e., double the wait time each time).
-
- save_results: int
- : Specify a number to store that many saved results from the loop.
- These are available through the `results` attribute, oldest first.
- Default 1.
-
- max_wait: float
- : Maximum number of seconds to wait between retries. Default 60.
+ * success_check: Callable --- This is a function that will be called
+ each time the loop saves a result. The function should return `True`
+ if the result indicates the code succeeded, `False` if it represents a
+ permanent failure, and `None` if it represents a temporary failure.
+ If no function is provided, the loop will end after any result is
+ saved.
+
+ * backoff_start: float --- The number of seconds that must pass before
+ the loop's second iteration. Default 0, which disables all waiting.
+
+ * backoff_growth: float --- The wait time multiplier after each
+ iteration. Default 2 (i.e., double the wait time each time).
+
+ * save_results: int --- Specify a number to store that many saved
+ results from the loop. These are available through the `results`
+ attribute, oldest first. Default 1.
+
+ * max_wait: float --- Maximum number of seconds to wait between
+ retries. Default 60.
"""
def __init__(self, num_retries, success_check=lambda r: True,
backoff_start=0, backoff_growth=2, save_results=1,
@@ -138,8 +133,8 @@ class RetryLoop(object):
Arguments:
- result: Any
- : The result from this loop attempt to check and save.
+ * result: Any --- The result from this loop attempt to check and
+ save.
"""
if not self.running():
raise arvados.errors.AssertionError(
@@ -207,8 +202,7 @@ def check_http_response_success(status_code):
Arguments:
- status_code: int
- : A numeric HTTP response code
+ * status_code: int --- A numeric HTTP response code
"""
if status_code in _HTTP_SUCCESSES:
return True
@@ -229,8 +223,8 @@ def retry_method(orig_func):
Arguments:
- orig_func: Callable
- : A class or instance method that accepts a `num_retries` keyword argument
+ * orig_func: Callable --- A class or instance method that accepts a
+ `num_retries` keyword argument
"""
@functools.wraps(orig_func)
def num_retries_setter(self, *args, **kwargs):
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 16f2999ca8..1ee5f6355a 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -74,16 +74,17 @@ def _deprecated(version=None, preferred=None):
func_doc = re.sub(r'\n\s*$', '', func.__doc__ or '')
match = re.search(r'\n([ \t]+)\S', func_doc)
indent = '' if match is None else match.group(1)
+ warning_doc = f'\n\n{indent}.. WARNING:: Deprecated\n{indent} {warning_msg}'
# Make the deprecation notice the second "paragraph" of the
# docstring if possible. Otherwise append it.
docstring, count = re.subn(
rf'\n[ \t]*\n{indent}',
- f'\n\n{indent}DEPRECATED: {warning_msg}\n\n{indent}',
+ f'{warning_doc}\n\n{indent}',
func_doc,
count=1,
)
if not count:
- docstring = f'{func_doc}\n\n{indent}DEPRECATED: {warning_msg}'.lstrip()
+ docstring = f'{func_doc.lstrip()}{warning_doc}'
deprecated_wrapper.__doc__ = docstring
return deprecated_wrapper
return deprecated_decorator
diff --git a/sdk/python/discovery2pydoc.py b/sdk/python/discovery2pydoc.py
index 9f7f87d988..c827de5e86 100755
--- a/sdk/python/discovery2pydoc.py
+++ b/sdk/python/discovery2pydoc.py
@@ -49,8 +49,8 @@ _ALIASED_METHODS = frozenset([
])
_DEPRECATED_NOTICE = '''
-!!! deprecated
- This resource is deprecated in the Arvados API.
+.. WARNING:: Deprecated
+ This resource is deprecated in the Arvados API.
'''
_DEPRECATED_RESOURCES = frozenset([
'Humans',
commit 95918ee904309ae9685bb95d07f6cdd491ae5af4
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Sep 25 16:54:42 2023 -0400
20885: Add our own pdoc wrapper
Mainly to support admonitions. See the documentation and comments in
pysdk_pdoc.py for details.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7e7be9ea21..fb347ef445 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -655,9 +655,9 @@ install_env() {
# PyYAML is a test requirement used by run_test_server.py and needed for
# other, non-Python tests.
- # pdoc is needed to build PySDK documentation.
+ # pdoc and markdown2 are needed to build PySDK documentation.
env -C "$WORKSPACE/sdk/python" python3 -m pip install . \
- && python3 -m pip install PyYAML pdoc \
+ && python3 -m pip install PyYAML pdoc markdown2 \
|| fatal "installing Python SDK and related dependencies failed"
}
diff --git a/doc/README.textile b/doc/README.textile
index 9799ac564b..d462d83459 100644
--- a/doc/README.textile
+++ b/doc/README.textile
@@ -22,10 +22,10 @@ To generate the Python SDK documentation, these additional dependencies are need
<pre>
arvados/doc$ sudo apt install python3-pip python3-venv
arvados/doc$ python3 -m venv .venv
-arvados/doc$ .venv/bin/pip install pdoc
+arvados/doc$ .venv/bin/pip install pdoc markdown2
</pre>
-Then the generation process will need to be able to find @arvados/doc/.venv/bin/pdoc at . Either add that full directory to your @$PATH@, or make a @pdoc@ symlink in another directory that's already in your @$PATH at .
+Then you must activate the virtualenv (e.g., run @. .venv/bin/activate@) before you run the @bundle exec rake@ commands below.
h2. Generate HTML pages
diff --git a/doc/Rakefile b/doc/Rakefile
index c9322bb6ae..13e87167b6 100644
--- a/doc/Rakefile
+++ b/doc/Rakefile
@@ -54,11 +54,13 @@ file "sdk/python/arvados.html" do |t|
if ENV['NO_SDK'] || File.exist?("no-sdk")
next
end
+ # pysdk_pdoc.py is a wrapper around the pdoc CLI. `which pdoc` is an easy
+ # and good-enough test to check whether it's installed at all.
`which pdoc`
if $? == 0
raise unless system("python3", "setup.py", "build",
chdir: "../sdk/python", out: :err)
- raise unless system("pdoc", "-o", "sdk/python", "../sdk/python/build/lib/arvados/",
+ raise unless system("python3", "pysdk_pdoc.py",
out: :err)
else
puts "Warning: pdoc not found, Python documentation will not be generated".colorize(:light_red)
diff --git a/doc/pysdk_pdoc.py b/doc/pysdk_pdoc.py
new file mode 100755
index 0000000000..3f1a914b72
--- /dev/null
+++ b/doc/pysdk_pdoc.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+"""pysdk_pdoc.py - Run pdoc with extra rendering options
+
+This script is a wrapper around the standard `pdoc` tool that:
+
+* uses a more modern version of the `markdown2` library if installed
+
+* enables the `admonitions` and `smarty-pants` extras for nicer rendering
+
+If run without arguments, it uses arguments to build the Arvados Python SDK
+documentation.
+"""
+# This script was written when the current version of pdoc was 8.3.0, it
+# included a vendored markdown2 2.4.1, and upstream markdown2 was 2.4.10.
+# This script may become unnecessary with later releases.
+
+import collections
+import functools
+import os
+import sys
+
+import pdoc
+import pdoc.__main__
+import pdoc.markdown2
+import pdoc.render
+import pdoc.render_helpers
+
+DEFAULT_ARGLIST = [
+ '--output-directory=sdk/python',
+ '../sdk/python/build/lib/arvados/',
+]
+MD_EXTENSIONS = {
+ 'admonitions': None,
+ 'smarty-pants': None,
+}
+
+def main(arglist=None):
+ # Configure pdoc to use extras we want.
+ pdoc.render_helpers.markdown_extensions = collections.ChainMap(
+ pdoc.render_helpers.markdown_extensions,
+ MD_EXTENSIONS,
+ )
+
+ # If a newer version of markdown2 is installed, use it for rendering.
+ md2_version = pdoc.markdown2.__version_info__
+ try:
+ import markdown2
+ except ImportError:
+ pass
+ else:
+ if markdown2.__version_info__ > md2_version:
+ md2_version = markdown2.__version_info__
+ pdoc.render.env.filters['to_html'] = functools.partial(
+ markdown2.markdown,
+ extras=pdoc.render_helpers.markdown_extensions,
+ )
+
+ # Ensure markdown2 is new enough to support our desired extras.
+ if md2_version < (2, 4, 3):
+ print("error: need markdown2>=2.4.3 to render admonitions", file=sys.stderr)
+ return os.EX_SOFTWARE
+
+ pdoc.__main__.cli(arglist)
+ return os.EX_OK
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv[1:] or DEFAULT_ARGLIST))
commit 337604d3b396de36cb9524e3e857fdf2334560c8
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Sep 25 16:54:11 2023 -0400
20885: Clean up run-tests Python install env
Most of this code duplicates sdk/python/setup.py and should no longer be
necessary with modern pip. Invoking pip so often also makes it slower
than it needs to be.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 0682675753..7e7be9ea21 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -656,7 +656,7 @@ install_env() {
# PyYAML is a test requirement used by run_test_server.py and needed for
# other, non-Python tests.
# pdoc is needed to build PySDK documentation.
- python3 -m pip install "$WORKSPACE/sdk/python" \
+ env -C "$WORKSPACE/sdk/python" python3 -m pip install . \
&& python3 -m pip install PyYAML pdoc \
|| fatal "installing Python SDK and related dependencies failed"
}
commit 84a8cea0b42ea28ab0abbd8dcb180d365600e111
Author: Brett Smith <brett.smith at curii.com>
Date: Mon Sep 25 16:40:56 2023 -0400
20885: Clean up run-tests Python install env
Most of this code duplicates sdk/python/setup.py and should no longer be
necessary with modern pip. Invoking pip so often also makes it slower
than it needs to be.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 481f0c33ab..0682675753 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -653,22 +653,12 @@ install_env() {
setup_virtualenv "$VENV3DIR"
. "$VENV3DIR/bin/activate"
- # Needed for run_test_server.py which is used by certain (non-Python) tests.
- # pdoc needed to generate the Python SDK documentation.
- (
- set -e
- "${VENV3DIR}/bin/pip3" install wheel
- "${VENV3DIR}/bin/pip3" install PyYAML
- "${VENV3DIR}/bin/pip3" install httplib2
- "${VENV3DIR}/bin/pip3" install future
- "${VENV3DIR}/bin/pip3" install google-api-python-client
- "${VENV3DIR}/bin/pip3" install ciso8601
- "${VENV3DIR}/bin/pip3" install pycurl
- "${VENV3DIR}/bin/pip3" install ws4py
- "${VENV3DIR}/bin/pip3" install pdoc
- cd "$WORKSPACE/sdk/python"
- python3 setup.py install
- ) || fatal "installing PyYAML and sdk/python failed"
+ # PyYAML is a test requirement used by run_test_server.py and needed for
+ # other, non-Python tests.
+ # pdoc is needed to build PySDK documentation.
+ python3 -m pip install "$WORKSPACE/sdk/python" \
+ && python3 -m pip install PyYAML pdoc \
+ || fatal "installing Python SDK and related dependencies failed"
}
retry() {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list