[arvados] updated: 2.7.0-6648-gdf989fd015
git repository hosting
git at public.arvados.org
Thu May 30 18:21:19 UTC 2024
Summary of changes:
doc/sdk/fuse/options.html.textile.liquid | 27 +++++++-----
doc/sdk/python/api-client.html.textile.liquid | 9 +++-
doc/user/reference/api-tokens.html.textile.liquid | 15 ++++---
doc/user/topics/arv-copy.html.textile.liquid | 54 +++++++++++++----------
sdk/python/arvados/commands/_util.py | 22 ---------
sdk/python/arvados/util.py | 52 +++++++++++++++++++---
sdk/python/tests/test_util.py | 17 +++++++
7 files changed, 129 insertions(+), 67 deletions(-)
via df989fd01534296d1c147987ab86b61c4ab7dc17 (commit)
via e699544a4017eb1266ee4b10931a5b8bca2ba0f8 (commit)
via 8a34694d1844164d1fd93404d2c7a2c62838f863 (commit)
via 5e763515911f345fc318b28a461cdfefc6d121a3 (commit)
via fe86ecd376c0c436a1fe17c00f6c0f4884570485 (commit)
via dd87fcfe5d6478b7c44384022b10f450eb8181b7 (commit)
via f596238118c6a50addf9ce01b431b3e66adccf04 (commit)
via 63e0ed21b48bd9bc54f4ea04d50261b2518647dd (commit)
from 21beff7a6713d5a2ea0fc7d9b88bd4c51119d737 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit df989fd01534296d1c147987ab86b61c4ab7dc17
Author: Brett Smith <brett.smith at curii.com>
Date: Thu May 30 14:18:34 2024 -0400
21020: Update arv-mount service definition
Now that the Python SDK supports configuration from systemd's
environment variables, we can use higher-level directives for
customization where needed.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/doc/sdk/fuse/options.html.textile.liquid b/doc/sdk/fuse/options.html.textile.liquid
index 3b02d058ce..9cae2c5a5e 100644
--- a/doc/sdk/fuse/options.html.textile.liquid
+++ b/doc/sdk/fuse/options.html.textile.liquid
@@ -164,8 +164,21 @@ Documentation={{ site.baseurl }}/sdk/fuse/options.html
[Service]
Type=simple
-CacheDirectory=arvados/keep
-CacheDirectoryMode=0700
+
+# arv-mount will cache data under a `keep` subdirectory of CacheDirectory.
+# If this is a system service installed under /etc/systemd/system,
+# the cache will be at /var/cache/arvados/keep.
+# The default value of `arvados` lets arv-mount share the cache with other
+# tools.
+CacheDirectory=arvados
+
+# arv-mount will get Arvados API credentials from the `settings.conf` file
+# under ConfigurationDirectory.
+# If this is a system service installed under /etc/systemd/system,
+# the configuration will be read from /etc/arvados/settings.conf.
+# The default value of `arvados` lets arv-mount read configuration from the
+# same location as other tools.
+ConfigurationDirectory=arvados
# This unit makes the mount available as `Arvados` under the runtime directory root.
# If this is a system service installed under /etc/systemd/system,
@@ -175,19 +188,13 @@ CacheDirectoryMode=0700
# If you want to mount at another location on the filesystem, remove RuntimeDirectory
# and replace both instances of %t/Arvados with your desired path.
RuntimeDirectory=Arvados
+
# The arv-mount path must be the absolute path where you installed the command.
# If you installed from a distribution package, make this /usr/bin/arv-mount.
# If you installed from pip, replace ... with the path to your virtualenv.
# You can add options to select what gets mounted, access permissions,
# cache size, log level, etc.
-ExecStart=<span class="userinput">...</span>/bin/arv-mount --foreground --disk-cache-dir %C/arvados/keep %t/Arvados
+ExecStart=<span class="userinput">...</span>/bin/arv-mount --foreground %t/Arvados
ExecStop=/usr/bin/fusermount -u %t/Arvados
-
-# This unit assumes the running user has a ~/.config/arvados/settings.conf
-# with ARVADOS_API_HOST and ARVADOS_API_TOKEN defined.
-# If not, you can write those in a separate file
-# and set its path as EnvironmentFile.
-# Make sure that file is owned and only readable by the running user (mode 0600).
-#EnvironmentFile=...
</code></pre>
</notextile>
commit e699544a4017eb1266ee4b10931a5b8bca2ba0f8
Author: Brett Smith <brett.smith at curii.com>
Date: Thu May 30 14:17:01 2024 -0400
21020: Note configuration differences when $XDG_CONFIG_HOME is set
Customizing $XDG_CONFIG_HOME is expected to be uncommon. Users who do it
are expected to know what they're doing. Therefore we do not provide
complete instructions for how to adapt, we just note it and expect them
to be able to follow high-level instruction.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/doc/user/reference/api-tokens.html.textile.liquid b/doc/user/reference/api-tokens.html.textile.liquid
index 73da17ccd2..f079940237 100644
--- a/doc/user/reference/api-tokens.html.textile.liquid
+++ b/doc/user/reference/api-tokens.html.textile.liquid
@@ -42,6 +42,8 @@ EOF
{% include 'notebox_begin' %}
This will overwrite the file @~/.config/arvados/settings.conf at .
+
+Arvados tools written in Python (most notably the @arv keep@ commands, @arv copy@, and @arv-mount@) search for configuration files following the XDG Base Directory Specification. This is uncommon, but if you have customized the @XDG_CONFIG_HOME@ environment variable, you may need to add @$HOME/.config@ to the @XDG_CONFIG_DIRS@ envirnoment variable to have all the tools find the same configuration.
{% include 'notebox_end' %}
h2. .bashrc
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index 7cdab3f726..703f25425f 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -25,6 +25,10 @@ For example, let's copy from the <a href="https://playground.arvados.org/">Arvad
In order to communicate with both clusters, you must create custom configuration files for each cluster. The "Getting an API token":{{site.baseurl}}/user/reference/api-tokens.html page describes how to get a token and create a configuration file. However, instead of creating the default @~/.config/arvados/settings.conf@ you need two configuration files, one for each cluster, with filenames in the format of <notextile><code><strong>ClusterID</strong>.conf</code></notextile>. For this example, follow these steps:
+{% include 'notebox_begin' %}
+ at arv-copy@ searches for configuration files following the XDG Base Directory Specification. This is uncommon, but if you have customized the @XDG_CONFIG_HOME@ environment variable, save both configuration files under @$XDG_CONFIG_HOME/arvados/@ instead of the default @~/.config/arvados/@ shown below.
+{% include 'notebox_end' %}
+
# Open the "Arvados Playground Workbench":https://playground.arvados.org.
# On the system where you'll run @arv-copy@, start a new file named @~/.config/arvados/pirca.conf@ in your editor.
# In Workbench, open the user menu in the upper right, and select "Get API token."
commit 8a34694d1844164d1fd93404d2c7a2c62838f863
Author: Brett Smith <brett.smith at curii.com>
Date: Thu May 30 14:16:32 2024 -0400
21020: Expand Python SDK configuration documentation
Update to describe the recent addition of searching systemd and XDG
directories.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/doc/sdk/python/api-client.html.textile.liquid b/doc/sdk/python/api-client.html.textile.liquid
index dabd2d37f8..62275aa476 100644
--- a/doc/sdk/python/api-client.html.textile.liquid
+++ b/doc/sdk/python/api-client.html.textile.liquid
@@ -26,7 +26,14 @@ import arvados
arv_client = arvados.api('v1')
{% endcodeblock %}
-This will connect to the Arvados API server using the @ARVADOS_API_HOST@, @ARVADOS_API_TOKEN@, and @ARVADOS_API_HOST_INSECURE@ settings from environment variables or @~/.config/arvados/settings.conf at . You can alternatively pass these settings as arguments:
+When called this way, the SDK gets Arvados API credentials from the first source it finds in this list:
+
+# The environment variables @ARVADOS_API_HOST@, @ARVADOS_API_TOKEN@, and @ARVADOS_API_HOST_INSECURE at .
+# The @settings.conf@ file under the directories listed in systemd's @CONFIGURATION_DIRECTORY@ environment variable.
+# The @arvados/settings.conf@ file under the directory in the @XDG_CONFIG_HOME@ environment variable. This defaults to @~/.config/arvados/settings.conf@ if @XDG_CONFIG_HOME@ is not set.
+# The @arvados/settings.conf@ file under the directories in the @XDG_CONFIG_DIRS@ environment variable.
+
+You can alternatively pass these settings as arguments:
{% codeblock as python %}
import arvados
commit 5e763515911f345fc318b28a461cdfefc6d121a3
Author: Brett Smith <brett.smith at curii.com>
Date: Thu May 30 12:33:01 2024 -0400
21020: Streamline settings.conf reference instructions
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/doc/user/reference/api-tokens.html.textile.liquid b/doc/user/reference/api-tokens.html.textile.liquid
index 4c35530e60..73da17ccd2 100644
--- a/doc/user/reference/api-tokens.html.textile.liquid
+++ b/doc/user/reference/api-tokens.html.textile.liquid
@@ -33,13 +33,16 @@ h2. settings.conf
Arvados tools will also look for the authentication information in @~/.config/arvados/settings.conf at . If you have already put the variables into the environment following the instructions above, you can use these commands to create an Arvados configuration file:
<notextile>
-<pre><code>$ <span class="userinput">echo "ARVADOS_API_HOST=$ARVADOS_API_HOST" > ~/.config/arvados/settings.conf</span>
-$ <span class="userinput">echo "ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN" >> ~/.config/arvados/settings.conf</span>
-</code></pre>
+<pre><code>$ <span class="userinput">cat >~/.config/arvados/settings.conf <<EOF
+ARVADOS_API_HOST=$ARVADOS_API_HOST
+ARVADOS_API_TOKEN=$ARVADOS_API_TOKEN
+EOF
+</span></code></pre>
</notextile>
-* The output-redirection operator @>@ in the first command will cause the target file @~/.config/arvados/settings.conf@ to be created anew, wiping out the content of any existing file at that path.
-* The @>>@ operator in the second command appends to the target file.
+{% include 'notebox_begin' %}
+This will overwrite the file @~/.config/arvados/settings.conf at .
+{% include 'notebox_end' %}
h2. .bashrc
commit fe86ecd376c0c436a1fe17c00f6c0f4884570485
Author: Brett Smith <brett.smith at curii.com>
Date: Thu May 30 12:19:57 2024 -0400
21020: Expound documentation for setting up arv-copy
Explain the process in more detail. Update it for Workbench 2.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index d6664abc6d..7cdab3f726 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -23,20 +23,22 @@ For workflow definitions, @arv-copy@ will recursively go through the workflow an
For example, let's copy from the <a href="https://playground.arvados.org/">Arvados Playground</a>, also known as *pirca*, to *dstcl*. The names *pirca* and *dstcl* are interchangable with any cluster ID. You can find the cluster ID from the prefix of the UUID of the object you want to copy. For example, in <notextile><code><strong>zzzzz</strong>-4zz18-tci4vn4fa95w0zx</code></notextile>, the cluster name is *zzzzz*.
-In order to communicate with both clusters, you must create custom configuration files for each cluster. The "Getting an API token":{{site.baseurl}}/user/reference/api-tokens.html page describes how to get a token and create a configuration file. However, instead of creating the default @~/.config/arvados/settings.conf@ you need two configuration files, one for each cluster, with filenames in the format of <notextile><code><strong>ClusterID</strong>.conf</code></notextile>.
-
-In this example, navigate to the *Current token* page on each of *pirca* and *dstcl* to get the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN at .
-
-The config file consists of two lines, one for @ARVADOS_API_HOST@ and one for @ARVADOS_API_TOKEN@:
-
-<pre>
-ARVADOS_API_HOST=zzzzz.arvadosapi.com
-ARVADOS_API_TOKEN=v2/zzzzz-gj3su-xxxxxxxxxxxxxxx/123456789abcdefghijkl
-</pre>
-
-Copy your @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN@ into the config files as shown below in the shell account from which you are executing the commands. In our example, you need two files, @~/.config/arvados/pirca.conf@ and @~/.config/arvados/dstcl.conf at .
-
-Now you're ready to copy between *pirca* and *dstcl*!
+In order to communicate with both clusters, you must create custom configuration files for each cluster. The "Getting an API token":{{site.baseurl}}/user/reference/api-tokens.html page describes how to get a token and create a configuration file. However, instead of creating the default @~/.config/arvados/settings.conf@ you need two configuration files, one for each cluster, with filenames in the format of <notextile><code><strong>ClusterID</strong>.conf</code></notextile>. For this example, follow these steps:
+
+# Open the "Arvados Playground Workbench":https://playground.arvados.org.
+# On the system where you'll run @arv-copy@, start a new file named @~/.config/arvados/pirca.conf@ in your editor.
+# In Workbench, open the user menu in the upper right, and select "Get API token."
+# In the Workbench "Get API Token" dialog, under the "API Host" header, copy the value to your clipboard using the button.
+# In your editor, write the text @ARVADOS_API_HOST=@, then paste the "API Host" value you copied in the previous step, and start a new line.
+# In the Workbench "Get API Token" dialog, under the "API Token" header, copy the value to your clipboard using the button.
+# In your editor, write the text @ARVADOS_API_TOKEN=@, then paste the "API Token" value you copied in the previous step, and start a new line.
+# Review your work. In your editor, @pirca.conf@ should look like this, with a different value for @ARVADOS_API_TOKEN@:
+ <pre><code>ARVADOS_API_HOST=pirca.arvadosapi.com
+ARVADOS_API_TOKEN=v2/jutro-gj3su-12345abcde67890/abcdefghijklmnopqrstuvwxyz1234567890
+</code></pre> If it looks right, save and close the file.
+# Open Workbench for your destination cluster *dstcl*.
+# On the system where you'll run @arv-copy@, start a new file named <notextile><code>~/.config/arvados/<b>dstcl</b>.conf</code></notextile> in your editor. Replace *@dstcl@* in the filename with the actual cluster ID of your destination cluster.
+# Repeat steps 3-8 to create a settings file with credentials for *dsctl*.
h3. How to copy a collection
commit dd87fcfe5d6478b7c44384022b10f450eb8181b7
Author: Brett Smith <brett.smith at curii.com>
Date: Thu May 30 11:43:15 2024 -0400
21020: Copyedit arv-copy documentation
For more consistent typography and capitalization.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/doc/user/topics/arv-copy.html.textile.liquid b/doc/user/topics/arv-copy.html.textile.liquid
index a05620d62d..d6664abc6d 100644
--- a/doc/user/topics/arv-copy.html.textile.liquid
+++ b/doc/user/topics/arv-copy.html.textile.liquid
@@ -21,13 +21,13 @@ For projects, @arv-copy@ will copy all the collections workflow definitions owne
For workflow definitions, @arv-copy@ will recursively go through the workflow and copy all associated dependencies (input collections and Docker images).
-For example, let's copy from the <a href="https://playground.arvados.org/">Arvados playground</a>, also known as *pirca*, to *dstcl*. The names *pirca* and *dstcl* are interchangable with any cluster id. You can find the cluster name from the prefix of the uuid of the object you want to copy. For example, in *zzzzz*-4zz18-tci4vn4fa95w0zx, the cluster name is *zzzzz* .
+For example, let's copy from the <a href="https://playground.arvados.org/">Arvados Playground</a>, also known as *pirca*, to *dstcl*. The names *pirca* and *dstcl* are interchangable with any cluster ID. You can find the cluster ID from the prefix of the UUID of the object you want to copy. For example, in <notextile><code><strong>zzzzz</strong>-4zz18-tci4vn4fa95w0zx</code></notextile>, the cluster name is *zzzzz*.
-In order to communicate with both clusters, you must create custom configuration files for each cluster. The "Getting an API token":{{site.baseurl}}/user/reference/api-tokens.html page describes how to get a token and create a configuration file. However, instead of "settings.conf" in @~/.config/arvados@ you need two configuration files, one for each cluster, with filenames in the format of *ClusterID.conf*.
+In order to communicate with both clusters, you must create custom configuration files for each cluster. The "Getting an API token":{{site.baseurl}}/user/reference/api-tokens.html page describes how to get a token and create a configuration file. However, instead of creating the default @~/.config/arvados/settings.conf@ you need two configuration files, one for each cluster, with filenames in the format of <notextile><code><strong>ClusterID</strong>.conf</code></notextile>.
In this example, navigate to the *Current token* page on each of *pirca* and *dstcl* to get the @ARVADOS_API_HOST@ and @ARVADOS_API_TOKEN at .
-The config file consists of two lines, one for ARVADOS_API_HOST and one for ARVADOS_API_TOKEN:
+The config file consists of two lines, one for @ARVADOS_API_HOST@ and one for @ARVADOS_API_TOKEN@:
<pre>
ARVADOS_API_HOST=zzzzz.arvadosapi.com
@@ -40,9 +40,9 @@ Now you're ready to copy between *pirca* and *dstcl*!
h3. How to copy a collection
-First, determine the uuid or portable data hash of the collection you want to copy from the source cluster. The uuid can be found in the collection display page in the collection summary area (top left box), or from the URL bar (the part after @collections/...@)
+First, determine the UUID or portable data hash of the collection you want to copy from the source cluster. The UUID can be found in the collection display page in the collection summary area (top left box), or from the URL bar (the part after @collections/...@)
-Now copy the collection from *pirca* to *dstcl*. We will use the uuid @jutro-4zz18-tv416l321i4r01e@ as an example. You can find this collection on <a href="https://playground.arvados.org/collections/jutro-4zz18-tv416l321i4r01e">playground.arvados.org</a>.
+Now copy the collection from *pirca* to *dstcl*. We will use the UUID @jutro-4zz18-tv416l321i4r01e@ as an example. You can find this collection on <a href="https://playground.arvados.org/collections/jutro-4zz18-tv416l321i4r01e">playground.arvados.org</a>.
<notextile>
<pre><code>~$ <span class="userinput">arv-copy --src pirca --dst dstcl jutro-4zz18-tv416l321i4r01e</span>
jutro-4zz18-tv416l321i4r01e: 6.1M / 6.1M 100.0%
@@ -59,7 +59,7 @@ arvados.arv-copy[1234] INFO: Success: created copy with uuid dstcl-4zz18-xxxxxxx
</code></pre>
</notextile>
-The output of arv-copy displays the uuid of the collection generated in the destination cluster. By default, the output is placed in your home project in the destination cluster. If you want to place your collection in an existing project, you can specify the project you want it to be in using the tag @--project-uuid@ followed by the project uuid.
+The output of arv-copy displays the UUID of the collection generated in the destination cluster. By default, the output is placed in your home project in the destination cluster. If you want to place your collection in an existing project, you can specify the project you want it to be in using the tag @--project-uuid@ followed by the project UUID.
For example, this will copy the collection to project @dstcl-j7d0g-a894213ukjhal12@ in the destination cluster.
@@ -73,9 +73,9 @@ h3. How to copy a workflow
Copying workflows requires @arvados-cwl-runner@ to be available in your @$PATH at .
-We will use the uuid @jutro-7fd4e-mkmmq53m1ze6apx@ as an example workflow.
+We will use the UUID @jutro-7fd4e-mkmmq53m1ze6apx@ as an example workflow.
-Arv-copy will infer the source cluster is @jutro@ from the object uuid, and destination cluster is @pirca@ from @--project-uuid at .
+Arv-copy will infer the source cluster is @jutro@ from the object UUID, and destination cluster is @pirca@ from @--project-uuid at .
<notextile>
<pre><code>~$ <span class="userinput">arv-copy --project-uuid pirca-j7d0g-ecak8knpefz8ere jutro-7fd4e-mkmmq53m1ze6apx</span>
@@ -87,15 +87,15 @@ jutro-4zz18-vvvqlops0a0kpdl: 94M / 94M 100.0%
</code></pre>
</notextile>
-The name, description, and workflow definition from the original workflow will be used for the destination copy. In addition, any *collections* and *docker images* referenced in the source workflow definition will also be copied to the destination.
+The name, description, and workflow definition from the original workflow will be used for the destination copy. In addition, any *collections* and *Docker images* referenced in the source workflow definition will also be copied to the destination.
If you would like to copy the object without dependencies, you can use the @--no-recursive@ flag.
h3. How to copy a project
-We will use the uuid @jutro-j7d0g-xj19djofle3aryq@ as an example project.
+We will use the UUID @jutro-j7d0g-xj19djofle3aryq@ as an example project.
-Arv-copy will infer the source cluster is @jutro@ from the source project uuid, and destination cluster is @pirca@ from @--project-uuid at .
+Arv-copy will infer the source cluster is @jutro@ from the source project UUID, and destination cluster is @pirca@ from @--project-uuid at .
<notextile>
<pre><code>~$ <span class="userinput">arv-copy --project-uuid pirca-j7d0g-lr8sq3tx3ovn68k jutro-j7d0g-xj19djofle3aryq</span>
commit f596238118c6a50addf9ce01b431b3e66adccf04
Author: Brett Smith <brett.smith at curii.com>
Date: Thu May 30 10:55:46 2024 -0400
21020: Warn user if a file can't be found under a custom $XDG_*_HOME
This is primarily meant to be a transitional aid on the off-chance that
someone had a program working under the 2.7 SDK with, e.g.,
~/.config/arvados/settings.conf but a custom $XDG_CONFIG_HOME.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index a9d69c324a..4f0c165e18 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -15,9 +15,11 @@ import functools
import hashlib
import httplib2
import itertools
+import logging
import os
import random
import re
+import shlex
import stat
import subprocess
import sys
@@ -81,6 +83,8 @@ job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
release. Prefer the containers API instead.
"""
+logger = logging.getLogger('arvados')
+
def _deprecated(version=None, preferred=None):
"""Mark a callable as deprecated in the SDK
@@ -174,11 +178,17 @@ class _BaseDirectorySpec:
yield path / subdir
def xdg_home(self, env: Mapping[str, str], subdir: PurePath) -> Path:
- home_path = self._abspath_from_env(env, self.xdg_home_key)
- if home_path is None:
- home_path = self._abspath_from_env(env, 'HOME') or Path.home()
- home_path /= self.xdg_home_default
- return home_path / subdir
+ return (
+ self._abspath_from_env(env, self.xdg_home_key)
+ or self.xdg_home_default_path(env)
+ ) / subdir
+
+ def xdg_home_default_path(self, env: Mapping[str, str]) -> Path:
+ return (self._abspath_from_env(env, 'HOME') or Path.home()) / self.xdg_home_default
+
+ def xdg_home_is_customized(self, env: Mapping[str, str]) -> bool:
+ xdg_home = self._abspath_from_env(env, self.xdg_home_key)
+ return xdg_home is not None and xdg_home != self.xdg_home_default_path(env)
class _BaseDirectorySpecs(enum.Enum):
@@ -228,6 +238,7 @@ class _BaseDirectories:
self._xdg_subdir = PurePath(xdg_subdir)
def search(self, name: str) -> Iterator[Path]:
+ any_found = False
for search_path in itertools.chain(
self._spec.iter_systemd(self._env),
self._spec.iter_xdg(self._env, self._xdg_subdir),
@@ -235,6 +246,37 @@ class _BaseDirectories:
path = search_path / name
if path.exists():
yield path
+ any_found = True
+ # The rest of this function is dedicated to warning the user if they
+ # have a custom XDG_*_HOME value that prevented the search from
+ # succeeding. This should be rare.
+ if any_found or not self._spec.xdg_home_is_customized(self._env):
+ return
+ default_home = self._spec.xdg_home_default_path(self._env)
+ default_path = Path(self._xdg_subdir / name)
+ if not (default_home / default_path).exists():
+ return
+ if self._spec.xdg_dirs_key is None:
+ suggest_key = self._spec.xdg_home_key
+ suggest_value = default_home
+ else:
+ suggest_key = self._spec.xdg_dirs_key
+ cur_value = self._env.get(suggest_key, '')
+ value_sep = ':' if cur_value else ''
+ suggest_value = f'{cur_value}{value_sep}{default_home}'
+ logger.warning(
+ "\
+%s was not found under your configured $%s (%s), \
+but does exist at the default location (%s) - \
+consider running this program with the environment setting %s=%s\
+",
+ default_path,
+ self._spec.xdg_home_key,
+ self._spec.xdg_home(self._env, ''),
+ default_home,
+ suggest_key,
+ shlex.quote(suggest_value),
+ )
def storage_path(
self,
diff --git a/sdk/python/tests/test_util.py b/sdk/python/tests/test_util.py
index 9f74d0205f..b31a7a8d6e 100644
--- a/sdk/python/tests/test_util.py
+++ b/sdk/python/tests/test_util.py
@@ -310,6 +310,23 @@ class TestBaseDirectories:
dirs = arvados.util._BaseDirectories(dir_spec, env, parent_path)
assert not list(dirs.search(test_path.name))
+ def test_search_warns_nondefault_home(self, dir_spec, env, tmp_path, caplog):
+ search_path = tmp_path / dir_spec.xdg_home_default / 'Search' / 'SearchConfig'
+ search_path.parent.mkdir(parents=True)
+ search_path.touch()
+ env[dir_spec.xdg_home_key] = str(tmp_path / '.nonexistent')
+ dirs = arvados.util._BaseDirectories(dir_spec, env, search_path.parent.name)
+ results = list(dirs.search(search_path.name))
+ expect_msg = "{} was not found under your configured ${} ({}), but does exist at the default location ({})".format(
+ Path(*search_path.parts[-2:]),
+ dir_spec.xdg_home_key,
+ env[dir_spec.xdg_home_key],
+ Path(*search_path.parts[:-2]),
+ )
+ assert caplog.messages
+ assert any(msg.startswith(expect_msg) for msg in caplog.messages)
+ assert not results
+
def test_storage_path_systemd(self, dir_spec, env, tmp_path):
expected = tmp_path / 'rwsystemd'
expected.mkdir(0o700)
commit 63e0ed21b48bd9bc54f4ea04d50261b2518647dd
Author: Brett Smith <brett.smith at curii.com>
Date: Thu May 30 09:34:48 2024 -0400
21020: Remove arvados.commands._util.make_home_conf_dir
Obsoleted by, and unused after, arvados.util._BaseDirectories.
Arvados-DCO-1.1-Signed-off-by: Brett Smith <brett.smith at curii.com>
diff --git a/sdk/python/arvados/commands/_util.py b/sdk/python/arvados/commands/_util.py
index 6c792b2e0d..0bb41a2598 100644
--- a/sdk/python/arvados/commands/_util.py
+++ b/sdk/python/arvados/commands/_util.py
@@ -36,28 +36,6 @@ def _ignore_error(error):
def _raise_error(error):
raise error
-def make_home_conf_dir(path, mode=None, errors='ignore'):
- # Make the directory path under the user's home directory, making parent
- # directories as needed.
- # If the directory is newly created, and a mode is specified, chmod it
- # with those permissions.
- # If there's an error, return None if errors is 'ignore', else raise an
- # exception.
- error_handler = _ignore_error if (errors == 'ignore') else _raise_error
- tilde_path = os.path.join('~', path)
- abs_path = os.path.expanduser(tilde_path)
- if abs_path == tilde_path:
- return error_handler(ValueError("no home directory available"))
- try:
- os.makedirs(abs_path)
- except OSError as error:
- if error.errno != errno.EEXIST:
- return error_handler(error)
- else:
- if mode is not None:
- os.chmod(abs_path, mode)
- return abs_path
-
CAUGHT_SIGNALS = [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]
def exit_signal_handler(sigcode, frame):
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list