[arvados] created: 2.7.0-655-g50e7dff1be

git repository hosting git at public.arvados.org
Mon Sep 25 20:55:25 UTC 2023


        at  50e7dff1be4d133ce1a599954699bc94bff51c68 (commit)


commit 50e7dff1be4d133ce1a599954699bc94bff51c68
Author: Brett Smith <brett.smith at curii.com>
Date:   Mon Sep 25 16:26:46 2023 -0400

    20885: Update PySDK docstrings after pdoc migration
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index c51be82b20..94b8013997 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -207,45 +207,41 @@ def api_client(
 
     Arguments:
 
-    version: str
-    : A string naming the version of the Arvados API to use.
+    * version: str --- A string naming the version of the Arvados API to use.
 
-    discoveryServiceUrl: str
-    : The URL used to discover APIs passed directly to
-      `googleapiclient.discovery.build`.
+    * discoveryServiceUrl: str --- The URL used to discover APIs passed
+      directly to `googleapiclient.discovery.build`.
 
-    token: str
-    : The authentication token to send with each API call.
+    * token: str --- The authentication token to send with each API call.
 
     Keyword-only arguments:
 
-    cache: bool
-    : If true, loads the API discovery document from, or saves it to, a cache
-      on disk (located at `~/.cache/arvados/discovery`).
+    * cache: bool --- If true, loads the API discovery document from, or
+      saves it to, a cache on disk (located at
+      `~/.cache/arvados/discovery`).
 
-    http: httplib2.Http | None
-    : The HTTP client object the API client object will use to make requests.
-      If not provided, this function will build its own to use. Either way, the
-      object will be patched as part of the build process.
+    * http: httplib2.Http | None --- The HTTP client object the API client
+      object will use to make requests.  If not provided, this function will
+      build its own to use. Either way, the object will be patched as part
+      of the build process.
 
-    insecure: bool
-    : If true, ignore SSL certificate validation errors. Default `False`.
+    * insecure: bool --- If true, ignore SSL certificate validation
+      errors. Default `False`.
 
-    num_retries: int
-    : The number of times to retry each API request if it encounters a
-      temporary failure. Default 10.
+    * num_retries: int --- The number of times to retry each API request if
+      it encounters a temporary failure. Default 10.
 
-    request_id: str | None
-    : Default `X-Request-Id` header value for outgoing requests that
-      don't already provide one. If `None` or omitted, generate a random
-      ID. When retrying failed requests, the same ID is used on all
-      attempts.
+    * request_id: str | None --- Default `X-Request-Id` header value for
+      outgoing requests that don't already provide one. If `None` or
+      omitted, generate a random ID. When retrying failed requests, the same
+      ID is used on all attempts.
 
-    timeout: int
-    : A timeout value for HTTP requests in seconds. Default 300 (5 minutes).
+    * timeout: int --- A timeout value for HTTP requests in seconds. Default
+      300 (5 minutes).
 
     Additional keyword arguments will be passed directly to
     `googleapiclient.discovery.build`.
+
     """
     if http is None:
         http = httplib2.Http(
@@ -322,22 +318,19 @@ def normalize_api_kwargs(
 
     Arguments:
 
-    version: str | None
-    : A string naming the version of the Arvados API to use. If not specified,
-      the code will log a warning and fall back to 'v1'.
+    * version: str | None --- A string naming the version of the Arvados API
+      to use. If not specified, the code will log a warning and fall back to
+      'v1'.
 
-    discoveryServiceUrl: str | None
-    : The URL used to discover APIs passed directly to
-      `googleapiclient.discovery.build`. It is an error to pass both
-      `discoveryServiceUrl` and `host`.
+    * discoveryServiceUrl: str | None --- The URL used to discover APIs
+      passed directly to `googleapiclient.discovery.build`. It is an error
+      to pass both `discoveryServiceUrl` and `host`.
 
-    host: str | None
-    : The hostname and optional port number of the Arvados API server. Used to
-      build `discoveryServiceUrl`. It is an error to pass both
-      `discoveryServiceUrl` and `host`.
+    * host: str | None --- The hostname and optional port number of the
+      Arvados API server. Used to build `discoveryServiceUrl`. It is an
+      error to pass both `discoveryServiceUrl` and `host`.
 
-    token: str
-    : The authentication token to send with each API call.
+    * token: str --- The authentication token to send with each API call.
 
     Additional keyword arguments will be included in the return value.
     """
@@ -378,14 +371,15 @@ def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
 
     Arguments:
 
-    version: str | None
-    : A string naming the version of the Arvados API to use. If not specified,
-      the code will log a warning and fall back to 'v1'.
+    * version: str | None --- A string naming the version of the Arvados API
+      to use. If not specified, the code will log a warning and fall back to
+      'v1'.
 
-    apiconfig: Mapping[str, str] | None
-    : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and
-      optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls
-      `arvados.config.settings` to get these parameters from user configuration.
+    * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+      `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+      `ARVADOS_API_HOST_INSECURE`. If not provided, calls
+      `arvados.config.settings` to get these parameters from user
+      configuration.
 
     Additional keyword arguments will be included in the return value.
     """
@@ -428,19 +422,18 @@ def api(version=None, cache=True, host=None, token=None, insecure=False,
 
     Arguments:
 
-    version: str | None
-    : A string naming the version of the Arvados API to use. If not specified,
-      the code will log a warning and fall back to 'v1'.
+    * version: str | None --- A string naming the version of the Arvados API
+      to use. If not specified, the code will log a warning and fall back to
+      'v1'.
 
-    host: str | None
-    : The hostname and optional port number of the Arvados API server.
+    * host: str | None --- The hostname and optional port number of the
+      Arvados API server.
 
-    token: str | None
-    : The authentication token to send with each API call.
+    * token: str | None --- The authentication token to send with each API
+      call.
 
-    discoveryServiceUrl: str | None
-    : The URL used to discover APIs passed directly to
-      `googleapiclient.discovery.build`.
+    * discoveryServiceUrl: str | None --- The URL used to discover APIs
+      passed directly to `googleapiclient.discovery.build`.
 
     If `host`, `token`, and `discoveryServiceUrl` are all omitted, `host` and
     `token` will be loaded from the user's configuration. Otherwise, you must
@@ -479,14 +472,15 @@ def api_from_config(version=None, apiconfig=None, **kwargs):
 
     Arguments:
 
-    version: str | None
-    : A string naming the version of the Arvados API to use. If not specified,
-      the code will log a warning and fall back to 'v1'.
+    * version: str | None --- A string naming the version of the Arvados API
+      to use. If not specified, the code will log a warning and fall back to
+      'v1'.
 
-    apiconfig: Mapping[str, str] | None
-    : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and
-      optionally `ARVADOS_API_HOST_INSECURE`. If not provided, calls
-      `arvados.config.settings` to get these parameters from user configuration.
+    * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+      `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+      `ARVADOS_API_HOST_INSECURE`. If not provided, calls
+      `arvados.config.settings` to get these parameters from user
+      configuration.
 
     Other arguments are passed directly to `api_client`. See that function's
     docstring for more information about their meaning.
diff --git a/sdk/python/arvados/retry.py b/sdk/python/arvados/retry.py
index 2168146a4b..ea8a6f65af 100644
--- a/sdk/python/arvados/retry.py
+++ b/sdk/python/arvados/retry.py
@@ -49,34 +49,29 @@ class RetryLoop(object):
 
     Arguments:
 
-    num_retries: int
-    : The maximum number of times to retry the loop if it
-      doesn't succeed.  This means the loop body could run at most
+    * num_retries: int --- The maximum number of times to retry the loop if
+      it doesn't succeed.  This means the loop body could run at most
       `num_retries + 1` times.
 
-    success_check: Callable
-    : This is a function that will be called each
-      time the loop saves a result.  The function should return
-      `True` if the result indicates the code succeeded, `False` if it
-      represents a permanent failure, and `None` if it represents a
-      temporary failure.  If no function is provided, the loop will
-      end after any result is saved.
-
-    backoff_start: float
-    : The number of seconds that must pass before the loop's second
-      iteration.  Default 0, which disables all waiting.
-
-    backoff_growth: float
-    : The wait time multiplier after each iteration.
-      Default 2 (i.e., double the wait time each time).
-
-    save_results: int
-    : Specify a number to store that many saved results from the loop.
-      These are available through the `results` attribute, oldest first.
-      Default 1.
-
-    max_wait: float
-    : Maximum number of seconds to wait between retries. Default 60.
+    * success_check: Callable --- This is a function that will be called
+      each time the loop saves a result.  The function should return `True`
+      if the result indicates the code succeeded, `False` if it represents a
+      permanent failure, and `None` if it represents a temporary failure.
+      If no function is provided, the loop will end after any result is
+      saved.
+
+    * backoff_start: float --- The number of seconds that must pass before
+      the loop's second iteration.  Default 0, which disables all waiting.
+
+    * backoff_growth: float --- The wait time multiplier after each
+      iteration.  Default 2 (i.e., double the wait time each time).
+
+    * save_results: int --- Specify a number to store that many saved
+      results from the loop.  These are available through the `results`
+      attribute, oldest first.  Default 1.
+
+    * max_wait: float --- Maximum number of seconds to wait between
+      retries. Default 60.
     """
     def __init__(self, num_retries, success_check=lambda r: True,
                  backoff_start=0, backoff_growth=2, save_results=1,
@@ -138,8 +133,8 @@ class RetryLoop(object):
 
         Arguments:
 
-        result: Any
-        : The result from this loop attempt to check and save.
+        * result: Any --- The result from this loop attempt to check and
+        save.
         """
         if not self.running():
             raise arvados.errors.AssertionError(
@@ -207,8 +202,7 @@ def check_http_response_success(status_code):
 
     Arguments:
 
-    status_code: int
-    : A numeric HTTP response code
+    * status_code: int --- A numeric HTTP response code
     """
     if status_code in _HTTP_SUCCESSES:
         return True
@@ -229,8 +223,8 @@ def retry_method(orig_func):
 
     Arguments:
 
-    orig_func: Callable
-    : A class or instance method that accepts a `num_retries` keyword argument
+    * orig_func: Callable --- A class or instance method that accepts a
+    `num_retries` keyword argument
     """
     @functools.wraps(orig_func)
     def num_retries_setter(self, *args, **kwargs):
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 16f2999ca8..1ee5f6355a 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -74,16 +74,17 @@ def _deprecated(version=None, preferred=None):
         func_doc = re.sub(r'\n\s*$', '', func.__doc__ or '')
         match = re.search(r'\n([ \t]+)\S', func_doc)
         indent = '' if match is None else match.group(1)
+        warning_doc = f'\n\n{indent}.. WARNING:: Deprecated\n{indent}   {warning_msg}'
         # Make the deprecation notice the second "paragraph" of the
         # docstring if possible. Otherwise append it.
         docstring, count = re.subn(
             rf'\n[ \t]*\n{indent}',
-            f'\n\n{indent}DEPRECATED: {warning_msg}\n\n{indent}',
+            f'{warning_doc}\n\n{indent}',
             func_doc,
             count=1,
         )
         if not count:
-            docstring = f'{func_doc}\n\n{indent}DEPRECATED: {warning_msg}'.lstrip()
+            docstring = f'{func_doc.lstrip()}{warning_doc}'
         deprecated_wrapper.__doc__ = docstring
         return deprecated_wrapper
     return deprecated_decorator
diff --git a/sdk/python/discovery2pydoc.py b/sdk/python/discovery2pydoc.py
index 9f7f87d988..c827de5e86 100755
--- a/sdk/python/discovery2pydoc.py
+++ b/sdk/python/discovery2pydoc.py
@@ -49,8 +49,8 @@ _ALIASED_METHODS = frozenset([
 ])
 _DEPRECATED_NOTICE = '''
 
-!!! deprecated
-    This resource is deprecated in the Arvados API.
+.. WARNING:: Deprecated
+   This resource is deprecated in the Arvados API.
 '''
 _DEPRECATED_RESOURCES = frozenset([
     'Humans',

commit 95918ee904309ae9685bb95d07f6cdd491ae5af4
Author: Brett Smith <brett.smith at curii.com>
Date:   Mon Sep 25 16:54:42 2023 -0400

    20885: Add our own pdoc wrapper
    
    Mainly to support admonitions. See the documentation and comments in
    pysdk_pdoc.py for details.
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 7e7be9ea21..fb347ef445 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -655,9 +655,9 @@ install_env() {
 
     # PyYAML is a test requirement used by run_test_server.py and needed for
     # other, non-Python tests.
-    # pdoc is needed to build PySDK documentation.
+    # pdoc and markdown2 are needed to build PySDK documentation.
     env -C "$WORKSPACE/sdk/python" python3 -m pip install . \
-        && python3 -m pip install PyYAML pdoc \
+        && python3 -m pip install PyYAML pdoc markdown2 \
         || fatal "installing Python SDK and related dependencies failed"
 }
 
diff --git a/doc/README.textile b/doc/README.textile
index 9799ac564b..d462d83459 100644
--- a/doc/README.textile
+++ b/doc/README.textile
@@ -22,10 +22,10 @@ To generate the Python SDK documentation, these additional dependencies are need
 <pre>
 arvados/doc$ sudo apt install python3-pip python3-venv
 arvados/doc$ python3 -m venv .venv
-arvados/doc$ .venv/bin/pip install pdoc
+arvados/doc$ .venv/bin/pip install pdoc markdown2
 </pre>
 
-Then the generation process will need to be able to find @arvados/doc/.venv/bin/pdoc at . Either add that full directory to your @$PATH@, or make a @pdoc@ symlink in another directory that's already in your @$PATH at .
+Then you must activate the virtualenv (e.g., run @. .venv/bin/activate@) before you run the @bundle exec rake@ commands below.
 
 h2. Generate HTML pages
 
diff --git a/doc/Rakefile b/doc/Rakefile
index c9322bb6ae..13e87167b6 100644
--- a/doc/Rakefile
+++ b/doc/Rakefile
@@ -54,11 +54,13 @@ file "sdk/python/arvados.html" do |t|
   if ENV['NO_SDK'] || File.exist?("no-sdk")
     next
   end
+  # pysdk_pdoc.py is a wrapper around the pdoc CLI. `which pdoc` is an easy
+  # and good-enough test to check whether it's installed at all.
   `which pdoc`
   if $? == 0
     raise unless system("python3", "setup.py", "build",
                         chdir: "../sdk/python", out: :err)
-    raise unless system("pdoc", "-o", "sdk/python", "../sdk/python/build/lib/arvados/",
+    raise unless system("python3", "pysdk_pdoc.py",
                         out: :err)
   else
     puts "Warning: pdoc not found, Python documentation will not be generated".colorize(:light_red)
diff --git a/doc/pysdk_pdoc.py b/doc/pysdk_pdoc.py
new file mode 100755
index 0000000000..3f1a914b72
--- /dev/null
+++ b/doc/pysdk_pdoc.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+"""pysdk_pdoc.py - Run pdoc with extra rendering options
+
+This script is a wrapper around the standard `pdoc` tool that:
+
+* uses a more modern version of the `markdown2` library if installed
+
+* enables the `admonitions` and `smarty-pants` extras for nicer rendering
+
+If run without arguments, it uses arguments to build the Arvados Python SDK
+documentation.
+"""
+# This script was written when the current version of pdoc was 8.3.0, it
+# included a vendored markdown2 2.4.1, and upstream markdown2 was 2.4.10.
+# This script may become unnecessary with later releases.
+
+import collections
+import functools
+import os
+import sys
+
+import pdoc
+import pdoc.__main__
+import pdoc.markdown2
+import pdoc.render
+import pdoc.render_helpers
+
+DEFAULT_ARGLIST = [
+    '--output-directory=sdk/python',
+    '../sdk/python/build/lib/arvados/',
+]
+MD_EXTENSIONS = {
+    'admonitions': None,
+    'smarty-pants': None,
+}
+
+def main(arglist=None):
+    # Configure pdoc to use extras we want.
+    pdoc.render_helpers.markdown_extensions = collections.ChainMap(
+        pdoc.render_helpers.markdown_extensions,
+        MD_EXTENSIONS,
+    )
+
+    # If a newer version of markdown2 is installed, use it for rendering.
+    md2_version = pdoc.markdown2.__version_info__
+    try:
+        import markdown2
+    except ImportError:
+        pass
+    else:
+        if markdown2.__version_info__ > md2_version:
+            md2_version = markdown2.__version_info__
+            pdoc.render.env.filters['to_html'] = functools.partial(
+                markdown2.markdown,
+                extras=pdoc.render_helpers.markdown_extensions,
+            )
+
+    # Ensure markdown2 is new enough to support our desired extras.
+    if md2_version < (2, 4, 3):
+        print("error: need markdown2>=2.4.3 to render admonitions", file=sys.stderr)
+        return os.EX_SOFTWARE
+
+    pdoc.__main__.cli(arglist)
+    return os.EX_OK
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:] or DEFAULT_ARGLIST))

commit 337604d3b396de36cb9524e3e857fdf2334560c8
Author: Brett Smith <brett.smith at curii.com>
Date:   Mon Sep 25 16:54:11 2023 -0400

    20885: Clean up run-tests Python install env
    
    Most of this code duplicates sdk/python/setup.py and should no longer be
    necessary with modern pip. Invoking pip so often also makes it slower
    than it needs to be.
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 0682675753..7e7be9ea21 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -656,7 +656,7 @@ install_env() {
     # PyYAML is a test requirement used by run_test_server.py and needed for
     # other, non-Python tests.
     # pdoc is needed to build PySDK documentation.
-    python3 -m pip install "$WORKSPACE/sdk/python" \
+    env -C "$WORKSPACE/sdk/python" python3 -m pip install . \
         && python3 -m pip install PyYAML pdoc \
         || fatal "installing Python SDK and related dependencies failed"
 }

commit 84a8cea0b42ea28ab0abbd8dcb180d365600e111
Author: Brett Smith <brett.smith at curii.com>
Date:   Mon Sep 25 16:40:56 2023 -0400

    20885: Clean up run-tests Python install env
    
    Most of this code duplicates sdk/python/setup.py and should no longer be
    necessary with modern pip. Invoking pip so often also makes it slower
    than it needs to be.
    
    Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 481f0c33ab..0682675753 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -653,22 +653,12 @@ install_env() {
     setup_virtualenv "$VENV3DIR"
     . "$VENV3DIR/bin/activate"
 
-    # Needed for run_test_server.py which is used by certain (non-Python) tests.
-    # pdoc needed to generate the Python SDK documentation.
-    (
-        set -e
-        "${VENV3DIR}/bin/pip3" install wheel
-        "${VENV3DIR}/bin/pip3" install PyYAML
-        "${VENV3DIR}/bin/pip3" install httplib2
-        "${VENV3DIR}/bin/pip3" install future
-        "${VENV3DIR}/bin/pip3" install google-api-python-client
-        "${VENV3DIR}/bin/pip3" install ciso8601
-        "${VENV3DIR}/bin/pip3" install pycurl
-        "${VENV3DIR}/bin/pip3" install ws4py
-        "${VENV3DIR}/bin/pip3" install pdoc
-        cd "$WORKSPACE/sdk/python"
-        python3 setup.py install
-    ) || fatal "installing PyYAML and sdk/python failed"
+    # PyYAML is a test requirement used by run_test_server.py and needed for
+    # other, non-Python tests.
+    # pdoc is needed to build PySDK documentation.
+    python3 -m pip install "$WORKSPACE/sdk/python" \
+        && python3 -m pip install PyYAML pdoc \
+        || fatal "installing Python SDK and related dependencies failed"
 }
 
 retry() {

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list