[ARVADOS] updated: 2.1.0-501-gf61be590b
Git user
git at public.arvados.org
Mon Mar 15 21:13:07 UTC 2021
Summary of changes:
doc/api/methods.html.textile.liquid | 2 --
doc/api/methods/groups.html.textile.liquid | 4 +++-
doc/user/topics/projects.html.textile.liquid | 35 +++++++++++++++++++++++++++-
services/api/app/models/group.rb | 8 +++----
services/fuse/arvados_fuse/fusedir.py | 14 +++++------
5 files changed, 47 insertions(+), 16 deletions(-)
discards 6db15b85a9ddb534f6628ce6195772f27cedaa44 (commit)
discards 949e4651a74241a184c1c63c59d7533b86f2dc52 (commit)
discards 79c45742e058f5ecd7248bd4252313e8af62dff2 (commit)
discards 937767712cbe0f6c9d33866c1ef0d9f2619cf71d (commit)
discards 52c1ef5a2637052ff4b3e88e1e05623d64df5a11 (commit)
discards f737eafb0f51879f5f27dbb23b19dd050709e05c (commit)
discards f877728c9876fb61fcf3ef91136e7c9eb7cd266c (commit)
discards cac9d1111f86a7ff6da2176e3069dec4484154d4 (commit)
discards d8d6bca4b5db4851a29473f08dc600816c977a21 (commit)
discards 38104975556f7a0a59c1a21a97aa37cd0e178d69 (commit)
discards 7183ca1596f6509fe2ef1960e8ca948424294781 (commit)
discards edd97da398232aa1161b781c956806b5e6e40d70 (commit)
discards d0196e27f0072b32a6c17448f1a76e05da4ed84c (commit)
discards c869babecb193f02a24f071b8fd101e16aeec680 (commit)
discards 8347fca787700f142014a6571c181d90c04e0251 (commit)
discards 8480cc77dbb55340b07946eae5130c3db96b8136 (commit)
discards 54d8c4e41a276ac82c79506f63907a108ebd9bfd (commit)
discards c39fcfbee2de3df0dc3229175316be9e2e647e1f (commit)
discards 2031e72047634a3f5944f36d175c1ae351a3bd3e (commit)
discards a76b52ff503ae14df608904349670151a5b15e47 (commit)
discards e5838a144b230f828bf189b41acb45c1cf9b5202 (commit)
discards 06fac7e369df875da835e8bab1eff72c5188c9d8 (commit)
discards 99524c2ef7404a20f3933293cb8050bae3e4d4dd (commit)
discards ed6110546d621c8d3924aa813fba1209d47fb1be (commit)
discards 30bfb516f3f5bc88c3d0c07e380496f31b65945f (commit)
discards 2fe25e2b32042098106acead136fd3064bab30e3 (commit)
discards 7db9eeb4cb9a972d79de6daf4441d036cbd3dc4c (commit)
discards b6131783bf7f0ca8035b1461688af09c292b8e7f (commit)
discards 614f8f17c491dd2d2e9563ae9b339f89bb8e7fce (commit)
discards 31a756a09ad8d98154bd57706b3d36381eedcb81 (commit)
discards 2e370ca1df7e1f99df5a08de40b8826d8cbc6349 (commit)
discards 7d3775f7dcd87bb5c210e33ff099460074080749 (commit)
discards 43af5f57d2de2cb5657e252c2f327e1213057f67 (commit)
discards affd80401d1ce578de5216a5328949d759ab495c (commit)
discards 8c2bbecb1c09fbc3818dc1a2d73b3fda2ba68e02 (commit)
discards 820fc945c069d237e515dcc1608a5661dbf7700e (commit)
discards 89a3090417e1f8241497a1d26425df9c5d2cb3ba (commit)
discards b3e4886cbbe195347179d0664621da9bc34e6170 (commit)
discards a4ca6916de801f811bd7f97e94e6a08b0d617d53 (commit)
discards 0b1d8b20daee92e306ebed05006e6c0ad45a8bc8 (commit)
discards 1eed3583ea85b3384a97eeef4971a4385e6d7c96 (commit)
discards 752ab2f077fe4c0789f5fdf78417ead5e5db518f (commit)
discards 3193023d7335f793d5cc015aa185f7a450e650f7 (commit)
discards 491ff06465da8b5dc3678dfb44622a824e583488 (commit)
discards ccb5293993df4d535b3ca1e3224a5a146d8f90c2 (commit)
discards b5884b515a7fe6255761020cdd39d450db6d603b (commit)
discards a999ea55a6fdfabeca12c8d8db24214698ae2908 (commit)
discards 8685bdc41012f1623cc02b573e27439fdf314799 (commit)
discards 5dc622e37805f511b04eb66557441b28fba13b80 (commit)
discards 4dc6c2c37fba1c59e3e57ad47a1ee55d0593ed04 (commit)
discards 65219ea552b17e3501f933e0b5a40506a5837709 (commit)
discards 71fd7a7a8bc1d00ef0ed3a9e6e5240c13b7967a0 (commit)
discards 46a5e3958ce98fd1bca4f035dab83ea4c39e6666 (commit)
discards 1dc205aca4d9df2880083022b33216209d414052 (commit)
discards 4aa26076d2c7ed1f2a84ee0c5e9c63ab30ace530 (commit)
discards e8d73e8066b61f7704dc0f6cf200953cdf9a5e60 (commit)
discards 1118101f84c013e4a9f8d33d1f2f9c072c6ff4aa (commit)
discards 9af58889fa57b1d81065aa3f202dac00767afb3a (commit)
discards 20bd61710abe56bb63df4b7a906d04c7c27f765a (commit)
discards 046cc7b98a96f76cbc1e150bd0468f4238f8dfe9 (commit)
discards 8b1aca5c3415bfee3b4bc242596e1ee68ddef354 (commit)
discards 5ccc1a94d1621e319990a4362c131d5ab776c478 (commit)
discards ea194f11512bbe272fd9a2767c43870b25216de7 (commit)
discards 3639e67a0d23f327be3032fcdf57dff555e7a927 (commit)
discards 279dd72bcd41f38e4bcd9ee87d7e8803903f4bca (commit)
discards 92935cb2cea3a5bae6da115b0119a8da5952b7e4 (commit)
discards 984703527b7205372d28a9c27e8356986d3ba278 (commit)
discards 41a052d1faf57249eeb86674256372225ff9b7ed (commit)
discards ba6b58b83c0c3a7a5f31b551e430a7b983fb998d (commit)
discards 6a64c9760267bd33f6058eac9c303205c1c21be8 (commit)
via f61be590b1e2be2b287c3f6df37dd52ad58e6327 (commit)
via 1aa78fa26ac0d9fa12e66a3dc6f26db205d51890 (commit)
via e50e07e6e51f32b1fda75409cdecb0729ba1bdfb (commit)
via ea78f40b05f12b1e874403c69aeef955208ac503 (commit)
via db96032fe610578d94dc415a5e2eea7c5b595e3f (commit)
via b368ac71f1f0da01474cb06bdf6be10415e1460e (commit)
via c47f71160f6d20aadf4078497abb6d4d0dcba6ee (commit)
via 987070282743f20d0b677b04fe3dd0660d3e9b35 (commit)
via 841623d90e7edb5f8e9d78e449951aba8683e51d (commit)
via b6d7d686e54b1ae03b8f7cffd56e6bf48fbc2e43 (commit)
via 64af0c168829ffb4076a97b21b1844a14476c3a0 (commit)
via ea5d09596c6de5808bdd8e7b3cecd4960727471c (commit)
via 9e327d8380a0ce95920128c533940b3cdd222a8e (commit)
via 63143617792648585a4c072390cbb58bcfdcfdce (commit)
via 168deb4dd5428964f857fa6de8d856aec3953532 (commit)
via f6b78bb3d0769199c811da40367afd4d9bded914 (commit)
via 9bae12f6e831f2803bc6f4d83e3b239e588313cc (commit)
via 80f95de92d14143faa9dc9605b52e098e75cfc71 (commit)
via 7e8549dcb3788e87a23661b6bae91a181c0e99d6 (commit)
via e7c4e3977ba54f91568e1c6274b4b4dcef71c8a4 (commit)
via 9d9f7df00a092de9c2813cb99907092a20e34fd5 (commit)
via cc088a7b64d8bf0fc827a5df619f875367b9fffa (commit)
via 9b9c01638a9b4bd9ff7f32fd186daf77d222eec6 (commit)
via e5f1a98ef29dca94fb5742ae2fec8524bd093fb5 (commit)
via c38f4a309ef8dc034e0a04e13d258206b02b2cdf (commit)
via 0e4fcd09203ca889efd938203bdf23aaf3083dc6 (commit)
via 3ff8939a3d9dab549ff8c250e2338dc515ef2c86 (commit)
via 2c02c23493558b7a0b52b60ab843d5ac0db0cd2a (commit)
via 53d3ee8c8b0566162661fbc571deff0106b85020 (commit)
via 92920a2a882edfe3d7b02c80bd694b9b61b97c85 (commit)
via d5d9c40b758dfaf48043b58177c96168925068aa (commit)
via 766582be2d323b6fc12bea181f1239b4649590f6 (commit)
via 61de965bf3fc966fce4d70f1d85a0c26016000e3 (commit)
via 7fc7398eeb37f2d76da55561eff5a4f77ee73d50 (commit)
via 2f8ee11fc4f756f6d2774cf6c6dae96ec4a16f8b (commit)
via 0e68845158573e94e5a6e2ed595b5a370420a543 (commit)
via 6a9bbdcb1ebfd8a67b7a0154c2576c79ace5088b (commit)
via 3c924b22e3705a35adbe12a82bc6ffdfe1c55835 (commit)
via 1919d3b07380831d8dc485bbf22857c5caf1da0f (commit)
via 87df20ebc9a695df24e92c0c6c44ee7d801733b9 (commit)
via c1f0890ca1539f6b71946491827dd35d7d214354 (commit)
via f7ae654bf33819bfae67d4be200e692394a507c5 (commit)
via be34fa5f0ae5c616798fdc9accae9677c7e53508 (commit)
via d26fce5afa74c38a094409867c18f6973512e439 (commit)
via 97502961518e9ac24dd9e6278dce6097d47b2bd6 (commit)
via e2d819b5216b056137fcc7863f788440a37a452d (commit)
via 165f300730a1c69b71a0b4008191baeed074f606 (commit)
via 7df49ca80b5ed10c9efc644de5b07ff3eb1829c8 (commit)
via 6543a0758850a52c8c1d3d9879c8937a40fc1640 (commit)
via 3474a8e3567f8119a7149dfa6b8beee16549e89b (commit)
via 198a6dd8ceb61dd2f6fca7b28d3b074f9b01f67d (commit)
via fe8a7a83e41dbc236621d5cd90128b0cf6721203 (commit)
via 1a427d62dd9ff3fc9294879b0ae5fe2b9b6195c5 (commit)
via 66fa80a47802b23a75b217e7e9a92e0bf6781b3e (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (6db15b85a9ddb534f6628ce6195772f27cedaa44)
\
N -- N -- N (f61be590b1e2be2b287c3f6df37dd52ad58e6327)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f61be590b1e2be2b287c3f6df37dd52ad58e6327
Author: Ward Vandewege <ward at curii.com>
Date: Mon Mar 15 17:10:57 2021 -0400
17119: further documentation updates.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/api/methods.html.textile.liquid b/doc/api/methods.html.textile.liquid
index d6c34f4d3..c6e4ba00a 100644
--- a/doc/api/methods.html.textile.liquid
+++ b/doc/api/methods.html.textile.liquid
@@ -107,8 +107,6 @@ table(table table-bordered table-condensed).
|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
-Note:
-
h4(#substringsearchfilter). Filtering using substring search
Resources can also be filtered by searching for a substring in attributes of type @string@, @array of strings@, @text@, and @hash@, which are indexed in the database specifically for search. To use substring search, the filter must:
diff --git a/doc/api/methods/groups.html.textile.liquid b/doc/api/methods/groups.html.textile.liquid
index 73df7a066..4b63de5de 100644
--- a/doc/api/methods/groups.html.textile.liquid
+++ b/doc/api/methods/groups.html.textile.liquid
@@ -35,7 +35,9 @@ table(table table-bordered table-condensed).
|delete_at|datetime|If @delete_at@ is non-null and in the past, the group and all objects directly or indirectly owned by the group may be permanently deleted.||
|is_trashed|datetime|True if @trash_at@ is in the past, false if not.||
- at filter@ groups have a special @properties@ field named @filters@, which must be an array of arrays with 3 elements, each describing a filter. @filter@ groups are virtual groups; they can not own other objects. Filter attributes must include the object type (@collections@, @container_requests@, @groups@, @workflows@), separated with a dot from the field to be filtered on. Filters are applied with an implied *and* between them, but each filter only applies to the object type specified. The results are subject to the usual access controls - they are a subset of all objects the user can see. Here is an example:
+ at filter@ groups are virtual groups; they can not own other objects. Filter groups have a special @properties@ field named @filters@, which must be an array of filter conditions. See "list method filters":{{site.baseurl}}/api/methods.html#filters for details on the syntax of valid filters, but keep in mind that the attributes must include the object type (@collections@, @container_requests@, @groups@, @workflows@), separated with a dot from the field to be filtered on.
+
+Filters are applied with an implied *and* between them, but each filter only applies to the object type specified. The results are subject to the usual access controls - they are a subset of all objects the user can see. Here is an example:
<pre>
"properties":{
diff --git a/doc/user/topics/projects.html.textile.liquid b/doc/user/topics/projects.html.textile.liquid
index f4b236c4c..a23ff528a 100644
--- a/doc/user/topics/projects.html.textile.liquid
+++ b/doc/user/topics/projects.html.textile.liquid
@@ -31,6 +31,39 @@ h2. Filter groups
Filter groups are another type of virtual project. They are implemented as an Arvados @group@ object with @group_class@ set to the value "filter".
-Filter groups define one or more filters which are applied to all objects that the current user can see, and returned as the contents of the @group at . @filter@ groups are described in more detail in the "groups API reference":/api/methods/groups.html.
+Filter groups define one or more filters which are applied to all objects that the current user can see, and returned as the contents of the @group at . Filter groups are described in more detail in the "groups API reference":{{site.baseurl}}/api/methods/groups.html, and the rules for creating valid filters are the same as for "list method filters":{{site.baseurl}}/api/methods.html#filters.
Filter groups are accessible (read-only) via Workbench and the Arvados FUSE mount, WebDAV and S3 interface. Filter groups must currently be defined via the API, SDK or cli, there is no Workbench support yet.
+
+As an example, create a filter group with the @arv@ cli:
+
+<notextile>
+<pre><code>~$ <span class="userinput"> FILTER_GROUP_UUID=`arv -s group create --group '{
+ "group_class":"filter",
+ "name":"my filter group",
+ "properties":{
+ "filters":
+ [
+ ["collections.name","ilike","%test%"],
+ ["uuid","is_a","arvados#collection"]
+ ]
+ }
+ }'`
+</code>
+</pre>
+</notextile>
+This filter group will contain all collections visible to the current user whose name matches the word @test@ (case insensitive).
+
+To see how this works via the keep FUSE mount, create a few matching (and non-matching) collections:
+
+<notextile>
+<pre><code>~$ <span class="userinput">arv collection create --collection '{"name":"empty test collection 1"}'</span>
+~$ <span class="userinput">arv collection create --collection '{"name":"another empty collection"}'</span>
+~$ <span class="userinput">arv collection create --collection '{"name":"empty Test collection 2"}'</span>
+~$ <span class="userinput">mkdir -p keep</span>
+~$ <span class="userinput">arv-mount keep</span>
+~$ <span class="userinput">ls keep/by_id/$FILTER_GROUP_UUID/ -C1</span>
+'empty test collection 1'
+'empty Test collection 2'</code>
+</pre>
+</notextile>
commit 1aa78fa26ac0d9fa12e66a3dc6f26db205d51890
Author: Ward Vandewege <ward at curii.com>
Date: Mon Mar 15 13:16:32 2021 -0400
17119: Make arv-mount work with the new filter groups code. Also in
arv-mount, lower the default project cache TTL from 60s to 3s,
and enable its use by default anywhere in arv-mount (it was only enabled
under home/).
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index cfef43fdb..e8da789fa 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -683,11 +683,9 @@ and the directory will appear if it exists.
if group_uuid_pattern.match(k):
project = self.api.groups().list(
- filters=[["uuid", "=", k]]).execute(num_retries=self.num_retries)
+ filters=[['group_class', 'in', ['project','filter']], ["uuid", "=", k]]).execute(num_retries=self.num_retries)
if project[u'items_available'] == 0:
return False
- if project[u'items'][0][u'group_class'] != u'project' and project[u'items'][0][u'group_class'] != u'filter':
- return False
e = self.inodes.add_entry(ProjectDirectory(
self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
else:
@@ -813,7 +811,7 @@ class ProjectDirectory(Directory):
"""A special directory that contains the contents of a project."""
def __init__(self, parent_inode, inodes, api, num_retries, project_object,
- poll=False, poll_time=60):
+ poll=True, poll_time=3):
super(ProjectDirectory, self).__init__(parent_inode, inodes, api.config)
self.api = api
self.num_retries = num_retries
@@ -901,7 +899,7 @@ class ProjectDirectory(Directory):
self.num_retries,
uuid=self.project_uuid,
filters=[["uuid", "is_a", "arvados#group"],
- ["group_class", "=", "project"]])
+ ["groups.group_class", "in", ["project","filter"]]])
contents.extend(arvados.util.list_all(self.api.groups().contents,
self.num_retries,
uuid=self.project_uuid,
@@ -936,7 +934,7 @@ class ProjectDirectory(Directory):
else:
namefilter = ["name", "in", [k, k2]]
contents = self.api.groups().list(filters=[["owner_uuid", "=", self.project_uuid],
- ["group_class", "=", "project"],
+ ["group_class", "in", ["project","filter"]],
namefilter],
limit=2).execute(num_retries=self.num_retries)["items"]
if not contents:
@@ -1105,7 +1103,7 @@ class SharedDirectory(Directory):
if 'httpMethod' in methods.get('shared', {}):
page = []
while True:
- resp = self.api.groups().shared(filters=[['group_class', '=', 'project']]+page,
+ resp = self.api.groups().shared(filters=[['group_class', 'in', ['project','filter']]]+page,
order="uuid",
limit=10000,
count="none",
@@ -1122,7 +1120,7 @@ class SharedDirectory(Directory):
else:
all_projects = arvados.util.list_all(
self.api.groups().list, self.num_retries,
- filters=[['group_class','=','project']],
+ filters=[['group_class','in',['project','filter']]],
select=["uuid", "owner_uuid"])
for ob in all_projects:
objects[ob['uuid']] = ob
commit e50e07e6e51f32b1fda75409cdecb0729ba1bdfb
Author: Ward Vandewege <ward at curii.com>
Date: Thu Mar 11 10:49:42 2021 -0500
17119: controller should export the include_old_versions flag on group
contents.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index bcb51d6b2..694b61d69 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -143,6 +143,7 @@ type GroupContentsOptions struct {
Order []string `json:"order"`
Include string `json:"include"`
Recursive bool `json:"recursive"`
+ IncludeOldVersions bool `json:"include_old_versions"`
ExcludeHomeProject bool `json:"exclude_home_project"`
}
commit ea78f40b05f12b1e874403c69aeef955208ac503
Author: Ward Vandewege <ward at curii.com>
Date: Wed Mar 10 17:12:33 2021 -0500
Documentation: fix copy/paste error in keep-web installation
instructions. Add missing word in keepproxy installation instructions.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/install/install-keep-web.html.textile.liquid b/doc/install/install-keep-web.html.textile.liquid
index 5dd229b31..033efe63f 100644
--- a/doc/install/install-keep-web.html.textile.liquid
+++ b/doc/install/install-keep-web.html.textile.liquid
@@ -122,7 +122,7 @@ Set @Users.AnonymousUserToken: ""@ (empty string) or leave it out if you do not
h3. Update nginx configuration
-Put a reverse proxy with SSL support in front of keep-web. Keep-web itself runs on the port 25107 (or whatever is specified in @Services.Keepproxy.InternalURL@) the reverse proxy runs on port 443 and forwards requests to Keepproxy.
+Put a reverse proxy with SSL support in front of keep-web. Keep-web itself runs on the port 9002 (or whatever is specified in @Services.WebDAV.InternalURL@) while the reverse proxy runs on port 443 and forwards requests to Keep-web.
Use a text editor to create a new file @/etc/nginx/conf.d/keep-web.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
diff --git a/doc/install/install-keepproxy.html.textile.liquid b/doc/install/install-keepproxy.html.textile.liquid
index b4edd4f57..2d32a2455 100644
--- a/doc/install/install-keepproxy.html.textile.liquid
+++ b/doc/install/install-keepproxy.html.textile.liquid
@@ -49,7 +49,7 @@ Edit the cluster config at @config.yml@ and set @Services.Keepproxy.ExternalURL@
h2(#update-nginx). Update Nginx configuration
-Put a reverse proxy with SSL support in front of Keepproxy. Keepproxy itself runs on the port 25107 (or whatever is specified in @Services.Keepproxy.InternalURL@) the reverse proxy runs on port 443 and forwards requests to Keepproxy.
+Put a reverse proxy with SSL support in front of Keepproxy. Keepproxy itself runs on the port 25107 (or whatever is specified in @Services.Keepproxy.InternalURL@) while the reverse proxy runs on port 443 and forwards requests to Keepproxy.
Use a text editor to create a new file @/etc/nginx/conf.d/keepproxy.conf@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
commit db96032fe610578d94dc415a5e2eea7c5b595e3f
Author: Tom Clegg <tom at curii.com>
Date: Thu Mar 4 16:51:12 2021 -0500
16669: Accept OIDC access token in federated requests.
...provided both local and remote clusters use the same login cluster.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/federation.go b/lib/controller/federation.go
index cab5e4c4c..419d8b010 100644
--- a/lib/controller/federation.go
+++ b/lib/controller/federation.go
@@ -263,10 +263,10 @@ func (h *Handler) saltAuthToken(req *http.Request, remote string) (updatedReq *h
return updatedReq, nil
}
- ctxlog.FromContext(req.Context()).Infof("saltAuthToken: cluster %s token %s remote %s", h.Cluster.ClusterID, creds.Tokens[0], remote)
+ ctxlog.FromContext(req.Context()).Debugf("saltAuthToken: cluster %s token %s remote %s", h.Cluster.ClusterID, creds.Tokens[0], remote)
token, err := auth.SaltToken(creds.Tokens[0], remote)
- if err == auth.ErrObsoleteToken {
+ if err == auth.ErrObsoleteToken || err == auth.ErrTokenFormat {
// If the token exists in our own database for our own
// user, salt it for the remote. Otherwise, assume it
// was issued by the remote, and pass it through
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index a9352098d..b2e928b82 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -69,6 +69,9 @@ func saltedTokenProvider(local backend, remoteID string) rpc.TokenProvider {
tokens = append(tokens, salted)
case auth.ErrSalted:
tokens = append(tokens, token)
+ case auth.ErrTokenFormat:
+ // pass through unmodified (assume it's an OIDC access token)
+ tokens = append(tokens, token)
case auth.ErrObsoleteToken:
ctx := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{token}})
aca, err := local.APIClientAuthorizationCurrent(ctx, arvados.GetOptions{})
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index 3d0639f6c..db1f7f0d0 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -683,15 +683,16 @@ func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
accesstoken := s.oidcprovider.ValidAccessToken()
for _, clusterID := range []string{"z1111", "z2222"} {
- c.Logf("trying clusterid %s", clusterID)
-
- conn := s.testClusters[clusterID].Conn()
- ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken)
var coll arvados.Collection
// Write some file data and create a collection
{
+ c.Logf("save collection to %s", clusterID)
+
+ conn := s.testClusters[clusterID].Conn()
+ ctx, ac, kc := s.testClusters[clusterID].ClientsWithToken(accesstoken)
+
fs, err := coll.FileSystem(ac, kc)
c.Assert(err, check.IsNil)
f, err := fs.OpenFile("test.txt", os.O_CREATE|os.O_RDWR, 0777)
@@ -708,15 +709,22 @@ func (s *IntegrationSuite) TestOIDCAccessTokenAuth(c *check.C) {
c.Assert(err, check.IsNil)
}
- // Read the collection & file data
- {
+ // Read the collection & file data -- both from the
+ // cluster where it was created, and from the other
+ // cluster.
+ for _, readClusterID := range []string{"z1111", "z2222", "z3333"} {
+ c.Logf("retrieve %s from %s", coll.UUID, readClusterID)
+
+ conn := s.testClusters[readClusterID].Conn()
+ ctx, ac, kc := s.testClusters[readClusterID].ClientsWithToken(accesstoken)
+
user, err := conn.UserGetCurrent(ctx, arvados.GetOptions{})
c.Assert(err, check.IsNil)
c.Check(user.FullName, check.Equals, "Example User")
- coll, err = conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
+ readcoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: coll.UUID})
c.Assert(err, check.IsNil)
- c.Check(coll.ManifestText, check.Not(check.Equals), "")
- fs, err := coll.FileSystem(ac, kc)
+ c.Check(readcoll.ManifestText, check.Not(check.Equals), "")
+ fs, err := readcoll.FileSystem(ac, kc)
c.Assert(err, check.IsNil)
f, err := fs.Open("test.txt")
c.Assert(err, check.IsNil)
diff --git a/lib/controller/localdb/login_oidc.go b/lib/controller/localdb/login_oidc.go
index 74b8929a2..73b557723 100644
--- a/lib/controller/localdb/login_oidc.go
+++ b/lib/controller/localdb/login_oidc.go
@@ -129,6 +129,7 @@ func (ctrl *oidcLoginController) Login(ctx context.Context, opts arvados.LoginOp
if err != nil {
return loginError(fmt.Errorf("error in OAuth2 exchange: %s", err))
}
+ ctxlog.FromContext(ctx).WithField("oauth2Token", oauth2Token).Debug("oauth2 exchange succeeded")
rawIDToken, ok := oauth2Token.Extra("id_token").(string)
if !ok {
return loginError(errors.New("error in OAuth2 exchange: no ID token in OAuth2 token"))
commit b368ac71f1f0da01474cb06bdf6be10415e1460e
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Mar 4 14:45:15 2021 -0500
Update cwltool for bug fixes. refs #17213
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index a2fba730c..4bccadb1b 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -39,8 +39,8 @@ setup(name='arvados-cwl-runner',
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20201121085451',
- 'schema-salad==7.0.20200612160654',
+ 'cwltool==3.0.20210124104916',
+ 'schema-salad==7.0.20210124093443',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0'
commit c47f71160f6d20aadf4078497abb6d4d0dcba6ee
Author: Javier Bértoli <jbertoli at curii.com>
Date: Wed Mar 3 11:34:14 2021 -0300
fix(compute-images): fix sed invokation
refs #17438
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index be67ffb56..5ec67b92c 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -83,7 +83,10 @@ if [ "x$RESOLVER" != "x" ]; then
SET_RESOLVER="--dns ${RESOLVER}"
fi
-$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" > /etc/systemd/system/docker.service
+$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
+ /lib/systemd/system/docker.service \
+ > /etc/systemd/system/docker.service
+
$SUDO systemctl daemon-reload
# Make sure user_allow_other is set in fuse.conf
commit 987070282743f20d0b677b04fe3dd0660d3e9b35
Author: Javier Bértoli <jbertoli at curii.com>
Date: Wed Mar 3 08:59:22 2021 -0300
fix(compute-images): add resolver variable to json files
refs #17438
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/compute-images/arvados-images-aws.json b/tools/compute-images/arvados-images-aws.json
index 6a1c45da2..4d757abfd 100644
--- a/tools/compute-images/arvados-images-aws.json
+++ b/tools/compute-images/arvados-images-aws.json
@@ -1,16 +1,18 @@
{
"variables": {
+ "arvados_cluster": "",
+ "associate_public_ip_address": "true",
"aws_access_key": "",
- "aws_secret_key": "",
"aws_profile": "",
- "build_environment": "aws",
- "arvados_cluster": "",
+ "aws_secret_key": "",
"aws_source_ami": "ami-04d70e069399af2e9",
+ "build_environment": "aws",
+ "public_key_file": "",
+ "reposuffix": "",
+ "resolver": "",
"ssh_user": "admin",
- "vpc_id": "",
"subnet_id": "",
- "public_key_file": "",
- "associate_public_ip_address": "true"
+ "vpc_id": ""
},
"builders": [{
"type": "amazon-ebs",
diff --git a/tools/compute-images/arvados-images-azure.json b/tools/compute-images/arvados-images-azure.json
index a0278d515..ec1d9b6a6 100644
--- a/tools/compute-images/arvados-images-azure.json
+++ b/tools/compute-images/arvados-images-azure.json
@@ -1,22 +1,22 @@
{
"variables": {
- "resource_group": null,
+ "account_file": "",
+ "arvados_cluster": "",
+ "build_environment": "azure-arm",
"client_id": "{{env `ARM_CLIENT_ID`}}",
"client_secret": "{{env `ARM_CLIENT_SECRET`}}",
- "subscription_id": "{{env `ARM_SUBSCRIPTION_ID`}}",
- "tenant_id": "{{env `ARM_TENANT_ID`}}",
- "build_environment": "azure-arm",
"cloud_environment_name": "Public",
- "location": "centralus",
- "ssh_user": "packer",
- "ssh_private_key_file": "{{env `PACKERPRIVKEY`}}",
"image_sku": "",
- "arvados_cluster": "",
+ "location": "centralus",
"project_id": "",
- "account_file": "",
- "resolver": "",
+ "public_key_file": "",
"reposuffix": "",
- "public_key_file": ""
+ "resolver": "",
+ "resource_group": null,
+ "ssh_private_key_file": "{{env `PACKERPRIVKEY`}}",
+ "ssh_user": "packer",
+ "subscription_id": "{{env `ARM_SUBSCRIPTION_ID`}}",
+ "tenant_id": "{{env `ARM_TENANT_ID`}}"
},
"builders": [
{
commit 841623d90e7edb5f8e9d78e449951aba8683e51d
Author: Javier Bértoli <jbertoli at curii.com>
Date: Wed Mar 3 08:57:03 2021 -0300
fix(compute-images): make resolver setting optional
Also, change docker's systemd service path to /etc instead of /lib
refs #17438
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/compute-images/build.sh b/tools/compute-images/build.sh
index fb02ce944..36f0e18a3 100755
--- a/tools/compute-images/build.sh
+++ b/tools/compute-images/build.sh
@@ -49,7 +49,7 @@ Options:
Azure SKU image to use
--ssh_user (default: packer)
The user packer will use to log into the image
- --resolver (default: 8.8.8.8)
+ --resolver (default: host's network provided)
The dns resolver for the machine
--reposuffix (default: unset)
Set this to "-dev" to track the unstable/dev Arvados repositories
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index eeda57388..be67ffb56 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -78,8 +78,12 @@ $SUDO echo -e "{\n \"Quota\": \"10G\",\n \"RemoveStoppedContainers\": \"always
$SUDO sed -i 's/GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"/g' /etc/default/grub
$SUDO update-grub
-# Set a higher ulimit for docker
-$SUDO sed -i "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 --dns ${RESOLVER}/g" /lib/systemd/system/docker.service
+# Set a higher ulimit and the resolver (if set) for docker
+if [ "x$RESOLVER" != "x" ]; then
+ SET_RESOLVER="--dns ${RESOLVER}"
+fi
+
+$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" > /etc/systemd/system/docker.service
$SUDO systemctl daemon-reload
# Make sure user_allow_other is set in fuse.conf
@@ -97,10 +101,11 @@ $SUDO chown -R crunch:crunch /home/crunch/.ssh
$SUDO chmod 600 /home/crunch/.ssh/authorized_keys
$SUDO chmod 700 /home/crunch/.ssh/
-# Make sure we resolve via the provided resolver IP. Prepending is good enough because
+# Make sure we resolve via the provided resolver IP if set. Prepending is good enough because
# unless 'rotate' is set, the nameservers are queried in order (cf. man resolv.conf)
-$SUDO sed -i "s/#prepend domain-name-servers 127.0.0.1;/prepend domain-name-servers ${RESOLVER};/" /etc/dhcp/dhclient.conf
-
+if [ "x$RESOLVER" != "x" ]; then
+ $SUDO sed -i "s/#prepend domain-name-servers 127.0.0.1;/prepend domain-name-servers ${RESOLVER};/" /etc/dhcp/dhclient.conf
+fi
# Set up the cloud-init script that will ensure encrypted disks
$SUDO mv /tmp/usr-local-bin-ensure-encrypted-partitions.sh /usr/local/bin/ensure-encrypted-partitions.sh
$SUDO chmod 755 /usr/local/bin/ensure-encrypted-partitions.sh
commit b6d7d686e54b1ae03b8f7cffd56e6bf48fbc2e43
Author: Tom Clegg <tom at curii.com>
Date: Wed Mar 3 10:05:02 2021 -0500
16745: Update test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 0a1c7d1b3..5c68eb424 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -395,7 +395,7 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
// FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
- c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
+ c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51))
// If the Host header indicates a collection, /metrics.json
// refers to a file in the collection -- the metrics handler
commit 64af0c168829ffb4076a97b21b1844a14476c3a0
Author: Tom Clegg <tom at curii.com>
Date: Tue Mar 2 22:25:11 2021 -0500
16745: Rename session cache size metric.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index ec48be609..07db7a016 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -81,8 +81,8 @@ func (m *cacheMetrics) setup(reg *prometheus.Registry) {
reg.MustRegister(m.apiCalls)
m.collectionBytes = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "arvados",
- Subsystem: "keepweb_collectioncache",
- Name: "cached_manifest_bytes",
+ Subsystem: "keepweb_sessions",
+ Name: "cached_collection_bytes",
Help: "Total size of all cached manifests and sessions.",
})
reg.MustRegister(m.collectionBytes)
commit ea5d09596c6de5808bdd8e7b3cecd4960727471c
Author: Tom Clegg <tom at curii.com>
Date: Tue Mar 2 16:27:18 2021 -0500
16745: Reject unsupported APIs instead of mishandling.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 9479b5886..620a21b88 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -102,6 +102,7 @@ func s3stringToSign(alg, scope, signedHeaders string, r *http.Request) (string,
normalizedURL := *r.URL
normalizedURL.RawPath = ""
normalizedURL.Path = reMultipleSlashChars.ReplaceAllString(normalizedURL.Path, "/")
+ ctxlog.FromContext(r.Context()).Infof("escapedPath %s", normalizedURL.EscapedPath())
canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", r.Method, normalizedURL.EscapedPath(), s3querystring(r.URL), canonicalHeaders, signedHeaders, r.Header.Get("X-Amz-Content-Sha256"))
ctxlog.FromContext(r.Context()).Debugf("s3stringToSign: canonicalRequest %s", canonicalRequest)
return fmt.Sprintf("%s\n%s\n%s\n%s", alg, r.Header.Get("X-Amz-Date"), scope, hashdigest(sha256.New(), canonicalRequest)), nil
@@ -221,6 +222,8 @@ var UnauthorizedAccess = "UnauthorizedAccess"
var InvalidRequest = "InvalidRequest"
var SignatureDoesNotMatch = "SignatureDoesNotMatch"
+var reRawQueryIndicatesAPI = regexp.MustCompile(`^[a-z]+(&|$)`)
+
// serveS3 handles r and returns true if r is a request from an S3
// client, otherwise it returns false.
func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
@@ -292,13 +295,23 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
// GetBucketLocation
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
- fmt.Fprintln(w, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+h.Config.cluster.ClusterID+`</LocationConstraint>`)
+ fmt.Fprintln(w, `<LocationConstraint><LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+
+ h.Config.cluster.ClusterID+
+ `</LocationConstraint></LocationConstraint>`)
+ } else if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
+ // GetBucketWebsite ("GET /bucketid/?website"), GetBucketTagging, etc.
+ s3ErrorResponse(w, InvalidRequest, "API not supported", r.URL.Path+"?"+r.URL.RawQuery, http.StatusBadRequest)
} else {
// ListObjects
h.s3list(bucketName, w, r, fs)
}
return true
case r.Method == http.MethodGet || r.Method == http.MethodHead:
+ if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
+ // GetObjectRetention ("GET /bucketid/objectid?retention&versionID=..."), etc.
+ s3ErrorResponse(w, InvalidRequest, "API not supported", r.URL.Path+"?"+r.URL.RawQuery, http.StatusBadRequest)
+ return true
+ }
fi, err := fs.Stat(fspath)
if r.Method == "HEAD" && !objectNameGiven {
// HeadBucket
@@ -328,6 +341,11 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
http.FileServer(fs).ServeHTTP(w, &r)
return true
case r.Method == http.MethodPut:
+ if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
+ // PutObjectAcl ("PUT /bucketid/objectid?acl&versionID=..."), etc.
+ s3ErrorResponse(w, InvalidRequest, "API not supported", r.URL.Path+"?"+r.URL.RawQuery, http.StatusBadRequest)
+ return true
+ }
if !objectNameGiven {
s3ErrorResponse(w, InvalidArgument, "Missing object name in PUT request.", r.URL.Path, http.StatusBadRequest)
return true
@@ -424,6 +442,11 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
w.WriteHeader(http.StatusOK)
return true
case r.Method == http.MethodDelete:
+ if reRawQueryIndicatesAPI.MatchString(r.URL.RawQuery) {
+ // DeleteObjectTagging ("DELETE /bucketid/objectid?tagging&versionID=..."), etc.
+ s3ErrorResponse(w, InvalidRequest, "API not supported", r.URL.Path+"?"+r.URL.RawQuery, http.StatusBadRequest)
+ return true
+ }
if !objectNameGiven || r.URL.Path == "/" {
s3ErrorResponse(w, InvalidArgument, "missing object name in DELETE request", r.URL.Path, http.StatusBadRequest)
return true
diff --git a/services/keep-web/s3_test.go b/services/keep-web/s3_test.go
index 4b92d4dad..e60b55c93 100644
--- a/services/keep-web/s3_test.go
+++ b/services/keep-web/s3_test.go
@@ -76,7 +76,7 @@ func (s *IntegrationSuite) s3setup(c *check.C) s3stage {
auth := aws.NewAuth(arvadostest.ActiveTokenUUID, arvadostest.ActiveToken, "", time.Now().Add(time.Hour))
region := aws.Region{
- Name: s.testServer.Addr,
+ Name: "zzzzz",
S3Endpoint: "http://" + s.testServer.Addr,
}
client := s3.New(*auth, region)
@@ -455,7 +455,7 @@ func (stage *s3stage) writeBigDirs(c *check.C, dirs int, filesPerDir int) {
}
func (s *IntegrationSuite) sign(c *check.C, req *http.Request, key, secret string) {
- scope := "20200202/region/service/aws4_request"
+ scope := "20200202/zzzzz/service/aws4_request"
signedHeaders := "date"
req.Header.Set("Date", time.Now().UTC().Format(time.RFC1123))
stringToSign, err := s3stringToSign(s3SignAlgorithm, scope, signedHeaders, req)
@@ -560,7 +560,7 @@ func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
{"/foo%5bbar", "/foo%5Bbar"}, // %XX must be uppercase
} {
date := time.Now().UTC().Format("20060102T150405Z")
- scope := "20200202/fakeregion/S3/aws4_request"
+ scope := "20200202/zzzzz/S3/aws4_request"
canonicalRequest := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", "GET", trial.normalizedPath, "", "host:host.example.com\n", "host", "")
c.Logf("canonicalRequest %q", canonicalRequest)
expect := fmt.Sprintf("%s\n%s\n%s\n%s", s3SignAlgorithm, date, scope, hashdigest(sha256.New(), canonicalRequest))
@@ -579,6 +579,23 @@ func (s *IntegrationSuite) TestS3NormalizeURIForSignature(c *check.C) {
}
}
+func (s *IntegrationSuite) TestS3GetBucketLocation(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
+ req, err := http.NewRequest("GET", bucket.URL("/"), nil)
+ c.Check(err, check.IsNil)
+ req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ req.URL.RawQuery = "location"
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ c.Check(string(buf), check.Equals, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<LocationConstraint><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">zzzzz</LocationConstraint></LocationConstraint>\n")
+ }
+}
+
func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
stage := s.s3setup(c)
defer stage.teardown(c)
@@ -596,6 +613,37 @@ func (s *IntegrationSuite) TestS3GetBucketVersioning(c *check.C) {
}
}
+func (s *IntegrationSuite) TestS3UnsupportedAPIs(c *check.C) {
+ stage := s.s3setup(c)
+ defer stage.teardown(c)
+ for _, trial := range []struct {
+ method string
+ path string
+ rawquery string
+ }{
+ {"GET", "/", "acl&versionId=1234"}, // GetBucketAcl
+ {"GET", "/foo", "acl&versionId=1234"}, // GetObjectAcl
+ {"PUT", "/", "acl"}, // PutBucketAcl
+ {"PUT", "/foo", "acl"}, // PutObjectAcl
+ {"DELETE", "/", "tagging"}, // DeleteBucketTagging
+ {"DELETE", "/foo", "tagging"}, // DeleteObjectTagging
+ } {
+ for _, bucket := range []*s3.Bucket{stage.collbucket, stage.projbucket} {
+ c.Logf("trial %v bucket %v", trial, bucket)
+ req, err := http.NewRequest(trial.method, bucket.URL(trial.path), nil)
+ c.Check(err, check.IsNil)
+ req.Header.Set("Authorization", "AWS "+arvadostest.ActiveTokenV2+":none")
+ req.URL.RawQuery = trial.rawquery
+ resp, err := http.DefaultClient.Do(req)
+ c.Assert(err, check.IsNil)
+ c.Check(resp.Header.Get("Content-Type"), check.Equals, "application/xml")
+ buf, err := ioutil.ReadAll(resp.Body)
+ c.Assert(err, check.IsNil)
+ c.Check(string(buf), check.Matches, "(?ms).*InvalidRequest.*API not supported.*")
+ }
+ }
+}
+
// If there are no CommonPrefixes entries, the CommonPrefixes XML tag
// should not appear at all.
func (s *IntegrationSuite) TestS3ListNoCommonPrefixes(c *check.C) {
commit 9e327d8380a0ce95920128c533940b3cdd222a8e
Author: Tom Clegg <tom at curii.com>
Date: Tue Mar 2 11:12:11 2021 -0500
16745: Handle GetBucketLocation API.
Previously misinterpreted as ListObjects with no delimiter, which is
very slow.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index d500f1e65..9479b5886 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -288,6 +288,11 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
w.Header().Set("Content-Type", "application/xml")
io.WriteString(w, xml.Header)
fmt.Fprintln(w, `<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"/>`)
+ } else if _, ok = r.URL.Query()["location"]; ok {
+ // GetBucketLocation
+ w.Header().Set("Content-Type", "application/xml")
+ io.WriteString(w, xml.Header)
+ fmt.Fprintln(w, `<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">`+h.Config.cluster.ClusterID+`</LocationConstraint>`)
} else {
// ListObjects
h.s3list(bucketName, w, r, fs)
commit 63143617792648585a4c072390cbb58bcfdcfdce
Author: Tom Clegg <tom at curii.com>
Date: Tue Mar 2 09:55:16 2021 -0500
16745: Don't store nil in an atomic.Value (panic).
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index af143c77e..ec48be609 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -218,6 +218,7 @@ func (c *cache) GetSession(token string) (arvados.CustomFileSystem, error) {
now := time.Now()
ent, _ := c.sessions.Get(token)
sess, _ := ent.(*cachedSession)
+ expired := false
if sess == nil {
c.metrics.sessionMisses.Inc()
sess = &cachedSession{
@@ -226,13 +227,13 @@ func (c *cache) GetSession(token string) (arvados.CustomFileSystem, error) {
c.sessions.Add(token, sess)
} else if sess.expire.Before(now) {
c.metrics.sessionMisses.Inc()
- sess.fs.Store(nil)
+ expired = true
} else {
c.metrics.sessionHits.Inc()
}
go c.pruneSessions()
fs, _ := sess.fs.Load().(arvados.CustomFileSystem)
- if fs != nil {
+ if fs != nil && !expired {
return fs, nil
}
ac, err := arvados.NewClientFromConfig(c.cluster)
commit 168deb4dd5428964f857fa6de8d856aec3953532
Author: Tom Clegg <tom at curii.com>
Date: Mon Feb 22 21:46:27 2021 -0500
16745: Prune enough sessions to reach size limit, not all.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index 71e995330..af143c77e 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -83,7 +83,7 @@ func (m *cacheMetrics) setup(reg *prometheus.Registry) {
Namespace: "arvados",
Subsystem: "keepweb_collectioncache",
Name: "cached_manifest_bytes",
- Help: "Total size of all manifests in cache.",
+ Help: "Total size of all cached manifests and sessions.",
})
reg.MustRegister(m.collectionBytes)
m.collectionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
@@ -251,10 +251,13 @@ func (c *cache) GetSession(token string) (arvados.CustomFileSystem, error) {
return fs, nil
}
+// Remove all expired session cache entries, then remove more entries
+// until approximate remaining size <= maxsize/2
func (c *cache) pruneSessions() {
now := time.Now()
var size int64
- for _, token := range c.sessions.Keys() {
+ keys := c.sessions.Keys()
+ for _, token := range keys {
ent, ok := c.sessions.Peek(token)
if !ok {
continue
@@ -264,16 +267,28 @@ func (c *cache) pruneSessions() {
c.sessions.Remove(token)
continue
}
+ if fs, ok := s.fs.Load().(arvados.CustomFileSystem); ok {
+ size += fs.MemorySize()
+ }
+ }
+ // Remove tokens until reaching size limit, starting with the
+ // least frequently used entries (which Keys() returns last).
+ for i := len(keys) - 1; i >= 0; i-- {
+ token := keys[i]
+ if size <= c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
+ break
+ }
+ ent, ok := c.sessions.Peek(token)
+ if !ok {
+ continue
+ }
+ s := ent.(*cachedSession)
fs, _ := s.fs.Load().(arvados.CustomFileSystem)
if fs == nil {
continue
}
- size += fs.MemorySize()
- }
- if size > c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
- for _, token := range c.sessions.Keys() {
- c.sessions.Remove(token)
- }
+ c.sessions.Remove(token)
+ size -= fs.MemorySize()
}
}
commit f6b78bb3d0769199c811da40367afd4d9bded914
Author: Tom Clegg <tom at curii.com>
Date: Mon Feb 22 11:10:10 2021 -0500
16745: Keep a SiteFileSystem alive for multiple read requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 6f72c02c1..bcaa692ff 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -530,21 +530,30 @@ Clusters:
TrustAllContent: false
# Cache parameters for WebDAV content serving:
- # * TTL: Maximum time to cache manifests and permission checks.
- # * UUIDTTL: Maximum time to cache collection state.
- # * MaxBlockEntries: Maximum number of block cache entries.
- # * MaxCollectionEntries: Maximum number of collection cache entries.
- # * MaxCollectionBytes: Approximate memory limit for collection cache.
- # * MaxPermissionEntries: Maximum number of permission cache entries.
- # * MaxUUIDEntries: Maximum number of UUID cache entries.
WebDAVCache:
+ # Time to cache manifests, permission checks, and sessions.
TTL: 300s
+
+ # Time to cache collection state.
UUIDTTL: 5s
- MaxBlockEntries: 4
+
+ # Block cache entries. Each block consumes up to 64 MiB RAM.
+ MaxBlockEntries: 4
+
+ # Collection cache entries.
MaxCollectionEntries: 1000
- MaxCollectionBytes: 100000000
+
+ # Approximate memory limit (in bytes) for collection cache.
+ MaxCollectionBytes: 100000000
+
+ # Permission cache entries.
MaxPermissionEntries: 1000
- MaxUUIDEntries: 1000
+
+ # UUID cache entries.
+ MaxUUIDEntries: 1000
+
+ # Persistent sessions.
+ MaxSessions: 100
Login:
# One of the following mechanisms (SSO, Google, PAM, LDAP, or
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 50869bf61..4787f4fab 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -536,21 +536,30 @@ Clusters:
TrustAllContent: false
# Cache parameters for WebDAV content serving:
- # * TTL: Maximum time to cache manifests and permission checks.
- # * UUIDTTL: Maximum time to cache collection state.
- # * MaxBlockEntries: Maximum number of block cache entries.
- # * MaxCollectionEntries: Maximum number of collection cache entries.
- # * MaxCollectionBytes: Approximate memory limit for collection cache.
- # * MaxPermissionEntries: Maximum number of permission cache entries.
- # * MaxUUIDEntries: Maximum number of UUID cache entries.
WebDAVCache:
+ # Time to cache manifests, permission checks, and sessions.
TTL: 300s
+
+ # Time to cache collection state.
UUIDTTL: 5s
- MaxBlockEntries: 4
+
+ # Block cache entries. Each block consumes up to 64 MiB RAM.
+ MaxBlockEntries: 4
+
+ # Collection cache entries.
MaxCollectionEntries: 1000
- MaxCollectionBytes: 100000000
+
+ # Approximate memory limit (in bytes) for collection cache.
+ MaxCollectionBytes: 100000000
+
+ # Permission cache entries.
MaxPermissionEntries: 1000
- MaxUUIDEntries: 1000
+
+ # UUID cache entries.
+ MaxUUIDEntries: 1000
+
+ # Persistent sessions.
+ MaxSessions: 100
Login:
# One of the following mechanisms (SSO, Google, PAM, LDAP, or
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 4ccb1ef5d..2fda7febe 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -65,6 +65,7 @@ type WebDAVCacheConfig struct {
MaxCollectionBytes int64
MaxPermissionEntries int
MaxUUIDEntries int
+ MaxSessions int
}
type Cluster struct {
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index aa75fee7c..2478641df 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -106,6 +106,9 @@ type FileSystem interface {
// path is "", flush all dirs/streams; otherwise, flush only
// the specified dir/stream.
Flush(path string, shortBlocks bool) error
+
+ // Estimate current memory usage.
+ MemorySize() int64
}
type inode interface {
@@ -156,6 +159,7 @@ type inode interface {
sync.Locker
RLock()
RUnlock()
+ MemorySize() int64
}
type fileinfo struct {
@@ -229,6 +233,13 @@ func (*nullnode) Child(name string, replace func(inode) (inode, error)) (inode,
return nil, ErrNotADirectory
}
+func (*nullnode) MemorySize() int64 {
+ // Types that embed nullnode should report their own size, but
+ // if they don't, we at least report a non-zero size to ensure
+ // a large tree doesn't get reported as 0 bytes.
+ return 64
+}
+
type treenode struct {
fs FileSystem
parent inode
@@ -319,6 +330,15 @@ func (n *treenode) Sync() error {
return nil
}
+func (n *treenode) MemorySize() (size int64) {
+ n.RLock()
+ defer n.RUnlock()
+ for _, inode := range n.inodes {
+ size += inode.MemorySize()
+ }
+ return
+}
+
type fileSystem struct {
root inode
fsBackend
@@ -607,6 +627,10 @@ func (fs *fileSystem) Flush(string, bool) error {
return ErrInvalidOperation
}
+func (fs *fileSystem) MemorySize() int64 {
+ return fs.root.MemorySize()
+}
+
// rlookup (recursive lookup) returns the inode for the file/directory
// with the given name (which may contain "/" separators). If no such
// file/directory exists, the returned node is nil.
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 1de558a1b..0233826a7 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -38,9 +38,6 @@ type CollectionFileSystem interface {
// Total data bytes in all files.
Size() int64
-
- // Memory consumed by buffered file data.
- memorySize() int64
}
type collectionFileSystem struct {
@@ -232,10 +229,10 @@ func (fs *collectionFileSystem) Flush(path string, shortBlocks bool) error {
return dn.flush(context.TODO(), names, flushOpts{sync: false, shortBlocks: shortBlocks})
}
-func (fs *collectionFileSystem) memorySize() int64 {
+func (fs *collectionFileSystem) MemorySize() int64 {
fs.fileSystem.root.Lock()
defer fs.fileSystem.root.Unlock()
- return fs.fileSystem.root.(*dirnode).memorySize()
+ return fs.fileSystem.root.(*dirnode).MemorySize()
}
func (fs *collectionFileSystem) MarshalManifest(prefix string) (string, error) {
@@ -879,14 +876,14 @@ func (dn *dirnode) flush(ctx context.Context, names []string, opts flushOpts) er
}
// caller must have write lock.
-func (dn *dirnode) memorySize() (size int64) {
+func (dn *dirnode) MemorySize() (size int64) {
for _, name := range dn.sortedNames() {
node := dn.inodes[name]
node.Lock()
defer node.Unlock()
switch node := node.(type) {
case *dirnode:
- size += node.memorySize()
+ size += node.MemorySize()
case *filenode:
for _, seg := range node.segments {
switch seg := seg.(type) {
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index 59a6a6ba8..05c8ea61a 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -1153,9 +1153,9 @@ func (s *CollectionFSSuite) TestFlushAll(c *check.C) {
fs.Flush("", true)
}
- size := fs.memorySize()
+ size := fs.MemorySize()
if !c.Check(size <= 1<<24, check.Equals, true) {
- c.Logf("at dir%d fs.memorySize()=%d", i, size)
+ c.Logf("at dir%d fs.MemorySize()=%d", i, size)
return
}
}
@@ -1188,13 +1188,13 @@ func (s *CollectionFSSuite) TestFlushFullBlocksOnly(c *check.C) {
c.Assert(err, check.IsNil)
}
}
- c.Check(fs.memorySize(), check.Equals, int64(nDirs*67<<20))
+ c.Check(fs.MemorySize(), check.Equals, int64(nDirs*67<<20))
c.Check(flushed, check.Equals, int64(0))
waitForFlush := func(expectUnflushed, expectFlushed int64) {
- for deadline := time.Now().Add(5 * time.Second); fs.memorySize() > expectUnflushed && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
+ for deadline := time.Now().Add(5 * time.Second); fs.MemorySize() > expectUnflushed && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) {
}
- c.Check(fs.memorySize(), check.Equals, expectUnflushed)
+ c.Check(fs.MemorySize(), check.Equals, expectUnflushed)
c.Check(flushed, check.Equals, expectFlushed)
}
diff --git a/sdk/go/arvados/fs_deferred.go b/sdk/go/arvados/fs_deferred.go
index 254b90c81..bb6c7a262 100644
--- a/sdk/go/arvados/fs_deferred.go
+++ b/sdk/go/arvados/fs_deferred.go
@@ -112,3 +112,4 @@ func (dn *deferrednode) RLock() { dn.realinode().RLock(
func (dn *deferrednode) RUnlock() { dn.realinode().RUnlock() }
func (dn *deferrednode) FS() FileSystem { return dn.currentinode().FS() }
func (dn *deferrednode) Parent() inode { return dn.currentinode().Parent() }
+func (dn *deferrednode) MemorySize() int64 { return dn.currentinode().MemorySize() }
diff --git a/services/keep-web/cache.go b/services/keep-web/cache.go
index eeb78ad90..71e995330 100644
--- a/services/keep-web/cache.go
+++ b/services/keep-web/cache.go
@@ -6,23 +6,27 @@ package main
import (
"sync"
+ "sync/atomic"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
- "github.com/hashicorp/golang-lru"
+ "git.arvados.org/arvados.git/sdk/go/keepclient"
+ lru "github.com/hashicorp/golang-lru"
"github.com/prometheus/client_golang/prometheus"
)
const metricsUpdateInterval = time.Second / 10
type cache struct {
- config *arvados.WebDAVCacheConfig
+ cluster *arvados.Cluster
+ config *arvados.WebDAVCacheConfig // TODO: use cluster.Collections.WebDAV instead
registry *prometheus.Registry
metrics cacheMetrics
pdhs *lru.TwoQueueCache
collections *lru.TwoQueueCache
permissions *lru.TwoQueueCache
+ sessions *lru.TwoQueueCache
setupOnce sync.Once
}
@@ -30,9 +34,12 @@ type cacheMetrics struct {
requests prometheus.Counter
collectionBytes prometheus.Gauge
collectionEntries prometheus.Gauge
+ sessionEntries prometheus.Gauge
collectionHits prometheus.Counter
pdhHits prometheus.Counter
permissionHits prometheus.Counter
+ sessionHits prometheus.Counter
+ sessionMisses prometheus.Counter
apiCalls prometheus.Counter
}
@@ -86,6 +93,27 @@ func (m *cacheMetrics) setup(reg *prometheus.Registry) {
Help: "Number of manifests in cache.",
})
reg.MustRegister(m.collectionEntries)
+ m.sessionEntries = prometheus.NewGauge(prometheus.GaugeOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_sessions",
+ Name: "active",
+ Help: "Number of active token sessions.",
+ })
+ reg.MustRegister(m.sessionEntries)
+ m.sessionHits = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_sessions",
+ Name: "hits",
+ Help: "Number of token session cache hits.",
+ })
+ reg.MustRegister(m.sessionHits)
+ m.sessionMisses = prometheus.NewCounter(prometheus.CounterOpts{
+ Namespace: "arvados",
+ Subsystem: "keepweb_sessions",
+ Name: "misses",
+ Help: "Number of token session cache misses.",
+ })
+ reg.MustRegister(m.sessionMisses)
}
type cachedPDH struct {
@@ -102,6 +130,11 @@ type cachedPermission struct {
expire time.Time
}
+type cachedSession struct {
+ expire time.Time
+ fs atomic.Value
+}
+
func (c *cache) setup() {
var err error
c.pdhs, err = lru.New2Q(c.config.MaxUUIDEntries)
@@ -116,6 +149,10 @@ func (c *cache) setup() {
if err != nil {
panic(err)
}
+ c.sessions, err = lru.New2Q(c.config.MaxSessions)
+ if err != nil {
+ panic(err)
+ }
reg := c.registry
if reg == nil {
@@ -132,6 +169,7 @@ func (c *cache) setup() {
func (c *cache) updateGauges() {
c.metrics.collectionBytes.Set(float64(c.collectionBytes()))
c.metrics.collectionEntries.Set(float64(c.collections.Len()))
+ c.metrics.sessionEntries.Set(float64(c.sessions.Len()))
}
var selectPDH = map[string]interface{}{
@@ -165,6 +203,80 @@ func (c *cache) Update(client *arvados.Client, coll arvados.Collection, fs arvad
return err
}
+// ResetSession unloads any potentially stale state. Should be called
+// after write operations, so subsequent reads don't return stale
+// data.
+func (c *cache) ResetSession(token string) {
+ c.setupOnce.Do(c.setup)
+ c.sessions.Remove(token)
+}
+
+// Get a long-lived CustomFileSystem suitable for doing a read operation
+// with the given token.
+func (c *cache) GetSession(token string) (arvados.CustomFileSystem, error) {
+ c.setupOnce.Do(c.setup)
+ now := time.Now()
+ ent, _ := c.sessions.Get(token)
+ sess, _ := ent.(*cachedSession)
+ if sess == nil {
+ c.metrics.sessionMisses.Inc()
+ sess = &cachedSession{
+ expire: now.Add(c.config.TTL.Duration()),
+ }
+ c.sessions.Add(token, sess)
+ } else if sess.expire.Before(now) {
+ c.metrics.sessionMisses.Inc()
+ sess.fs.Store(nil)
+ } else {
+ c.metrics.sessionHits.Inc()
+ }
+ go c.pruneSessions()
+ fs, _ := sess.fs.Load().(arvados.CustomFileSystem)
+ if fs != nil {
+ return fs, nil
+ }
+ ac, err := arvados.NewClientFromConfig(c.cluster)
+ if err != nil {
+ return nil, err
+ }
+ ac.AuthToken = token
+ arv, err := arvadosclient.New(ac)
+ if err != nil {
+ return nil, err
+ }
+ kc := keepclient.New(arv)
+ fs = ac.SiteFileSystem(kc)
+ fs.ForwardSlashNameSubstitution(c.cluster.Collections.ForwardSlashNameSubstitution)
+ sess.fs.Store(fs)
+ return fs, nil
+}
+
+func (c *cache) pruneSessions() {
+ now := time.Now()
+ var size int64
+ for _, token := range c.sessions.Keys() {
+ ent, ok := c.sessions.Peek(token)
+ if !ok {
+ continue
+ }
+ s := ent.(*cachedSession)
+ if s.expire.Before(now) {
+ c.sessions.Remove(token)
+ continue
+ }
+ fs, _ := s.fs.Load().(arvados.CustomFileSystem)
+ if fs == nil {
+ continue
+ }
+ size += fs.MemorySize()
+ }
+ if size > c.cluster.Collections.WebDAVCache.MaxCollectionBytes/2 {
+ for _, token := range c.sessions.Keys() {
+ c.sessions.Remove(token)
+ }
+ }
+}
+
func (c *cache) Get(arv *arvadosclient.ArvadosClient, targetID string, forceReload bool) (*arvados.Collection, error) {
c.setupOnce.Do(c.setup)
c.metrics.requests.Inc()
@@ -288,7 +400,7 @@ func (c *cache) pruneCollections() {
}
}
for i, k := range keys {
- if size <= c.config.MaxCollectionBytes {
+ if size <= c.config.MaxCollectionBytes/2 {
break
}
if expired[i] {
@@ -300,8 +412,8 @@ func (c *cache) pruneCollections() {
}
}
-// collectionBytes returns the approximate memory size of the
-// collection cache.
+// collectionBytes returns the approximate combined memory size of the
+// collection cache and session filesystem cache.
func (c *cache) collectionBytes() uint64 {
var size uint64
for _, k := range c.collections.Keys() {
@@ -311,6 +423,15 @@ func (c *cache) collectionBytes() uint64 {
}
size += uint64(len(v.(*cachedCollection).collection.ManifestText))
}
+ for _, token := range c.sessions.Keys() {
+ ent, ok := c.sessions.Peek(token)
+ if !ok {
+ continue
+ }
+ if fs, ok := ent.(*cachedSession).fs.Load().(arvados.CustomFileSystem); ok {
+ size += uint64(fs.MemorySize())
+ }
+ }
return size
}
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 2d6fb78f8..4ea2fa2f6 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -520,7 +520,8 @@ func (h *handler) ServeHTTP(wOrig http.ResponseWriter, r *http.Request) {
func (h *handler) getClients(reqID, token string) (arv *arvadosclient.ArvadosClient, kc *keepclient.KeepClient, client *arvados.Client, release func(), err error) {
arv = h.clientPool.Get()
if arv == nil {
- return nil, nil, nil, nil, err
+ err = h.clientPool.Err()
+ return
}
release = func() { h.clientPool.Put(arv) }
arv.ApiToken = token
@@ -548,14 +549,11 @@ func (h *handler) serveSiteFS(w http.ResponseWriter, r *http.Request, tokens []s
http.Error(w, errReadOnly.Error(), http.StatusMethodNotAllowed)
return
}
- _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), tokens[0])
+ fs, err := h.Config.Cache.GetSession(tokens[0])
if err != nil {
- http.Error(w, "Pool failed: "+h.clientPool.Err().Error(), http.StatusInternalServerError)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
- defer release()
-
- fs := client.SiteFileSystem(kc)
fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
f, err := fs.Open(r.URL.Path)
if os.IsNotExist(err) {
diff --git a/services/keep-web/main.go b/services/keep-web/main.go
index 647eab165..a62e0abb6 100644
--- a/services/keep-web/main.go
+++ b/services/keep-web/main.go
@@ -38,6 +38,7 @@ func newConfig(arvCfg *arvados.Config) *Config {
}
cfg.cluster = cls
cfg.Cache.config = &cfg.cluster.Collections.WebDAVCache
+ cfg.Cache.cluster = cls
return &cfg
}
diff --git a/services/keep-web/s3.go b/services/keep-web/s3.go
index 1c84976d2..d500f1e65 100644
--- a/services/keep-web/s3.go
+++ b/services/keep-web/s3.go
@@ -243,15 +243,29 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
return false
}
- _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
- if err != nil {
- s3ErrorResponse(w, InternalError, "Pool failed: "+h.clientPool.Err().Error(), r.URL.Path, http.StatusInternalServerError)
- return true
+ var err error
+ var fs arvados.CustomFileSystem
+ if r.Method == http.MethodGet || r.Method == http.MethodHead {
+ // Use a single session (cached FileSystem) across
+ // multiple read requests.
+ fs, err = h.Config.Cache.GetSession(token)
+ if err != nil {
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
+ return true
+ }
+ } else {
+ // Create a FileSystem for this request, to avoid
+ // exposing incomplete write operations to concurrent
+ // requests.
+ _, kc, client, release, err := h.getClients(r.Header.Get("X-Request-Id"), token)
+ if err != nil {
+ s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
+ return true
+ }
+ defer release()
+ fs = client.SiteFileSystem(kc)
+ fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
}
- defer release()
-
- fs := client.SiteFileSystem(kc)
- fs.ForwardSlashNameSubstitution(h.Config.cluster.Collections.ForwardSlashNameSubstitution)
var objectNameGiven bool
var bucketName string
@@ -400,6 +414,8 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
+ // Ensure a subsequent read operation will see the changes.
+ h.Config.Cache.ResetSession(token)
w.WriteHeader(http.StatusOK)
return true
case r.Method == http.MethodDelete:
@@ -447,11 +463,12 @@ func (h *handler) serveS3(w http.ResponseWriter, r *http.Request) bool {
s3ErrorResponse(w, InternalError, err.Error(), r.URL.Path, http.StatusInternalServerError)
return true
}
+ // Ensure a subsequent read operation will see the changes.
+ h.Config.Cache.ResetSession(token)
w.WriteHeader(http.StatusNoContent)
return true
default:
s3ErrorResponse(w, InvalidRequest, "method not allowed", r.URL.Path, http.StatusMethodNotAllowed)
-
return true
}
}
commit 9bae12f6e831f2803bc6f4d83e3b239e588313cc
Author: Javier Bértoli <jbertoli at curii.com>
Date: Tue Mar 2 12:22:00 2021 -0300
fix(compute-images): remove unneeded code
refs #17435 & #16611
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index b6245f963..eeda57388 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -60,14 +60,6 @@ wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes ins
cryptsetup \
xfsprogs
-# See if python3-distutils is installable, and if so install it. This is a
-# temporary workaround for an Arvados packaging bug and should be removed once
-# Arvados 2.0.4 or 2.1.0 is released, whichever comes first.
-# See https://dev.arvados.org/issues/16611 for more information
-if apt-cache -qq show python3-distutils >/dev/null 2>&1; then
- wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install python3-distutils
-fi
-
# Install the Arvados packages we need
wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
python3-arvados-fuse \
commit 80f95de92d14143faa9dc9605b52e098e75cfc71
Author: Javier Bértoli <jbertoli at curii.com>
Date: Tue Mar 2 12:20:37 2021 -0300
fix(compute-images): upgrade python-arvados-fuse to py3
refs #17435
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index 1c381447c..b6245f963 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -70,7 +70,7 @@ fi
# Install the Arvados packages we need
wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
- python-arvados-fuse \
+ python3-arvados-fuse \
crunch-run \
arvados-docker-cleaner \
docker.io
commit 7e8549dcb3788e87a23661b6bae91a181c0e99d6
Author: Javier Bértoli <jbertoli at curii.com>
Date: Tue Mar 2 12:04:48 2021 -0300
fix(compute-images): wait for apt/dpkg locks to be released
refs #17435
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli at curii.com>
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index 78cbccdb9..1c381447c 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -6,20 +6,27 @@
SUDO=sudo
+wait_for_apt_locks() {
+ while $SUDO fuser /var/{lib/{dpkg,apt/lists},cache/apt/archives}/lock >/dev/null 2>&1; do
+ echo "APT: Waiting for apt/dpkg locks to be released..."
+ sleep 1
+ done
+}
+
# Run apt-get update
$SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
# Install gnupg and dirmgr or gpg key checks will fail
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
gnupg \
dirmngr \
lsb-release
# For good measure, apt-get upgrade
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes upgrade
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes upgrade
# Make sure cloud-init is installed
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install cloud-init
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install cloud-init
if [[ ! -d /var/lib/cloud/scripts/per-boot ]]; then
mkdir -p /var/lib/cloud/scripts/per-boot
fi
@@ -34,15 +41,15 @@ echo "deb http://apt.arvados.org/$LSB_RELEASE_CODENAME $LSB_RELEASE_CODENAME${RE
# Add the arvados signing key
cat /tmp/1078ECD7.asc | $SUDO apt-key add -
# Add the debian keys
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get install --yes debian-keyring debian-archive-keyring
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get install --yes debian-keyring debian-archive-keyring
# Fix locale
$SUDO /bin/sed -ri 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
$SUDO /usr/sbin/locale-gen
# Install some packages we always need
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
openssh-server \
apt-utils \
git \
@@ -58,18 +65,18 @@ $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
# Arvados 2.0.4 or 2.1.0 is released, whichever comes first.
# See https://dev.arvados.org/issues/16611 for more information
if apt-cache -qq show python3-distutils >/dev/null 2>&1; then
- $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install python3-distutils
+ wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install python3-distutils
fi
# Install the Arvados packages we need
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
python-arvados-fuse \
crunch-run \
arvados-docker-cleaner \
docker.io
# Remove unattended-upgrades if it is installed
-$SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes remove unattended-upgrades --purge
+wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes remove unattended-upgrades --purge
# Configure arvados-docker-cleaner
$SUDO mkdir -p /etc/arvados/docker-cleaner
commit e7c4e3977ba54f91568e1c6274b4b4dcef71c8a4
Author: Ward Vandewege <ward at curii.com>
Date: Sun Feb 28 17:13:03 2021 -0500
Fix a few more golint warnings.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/boot/cert.go b/lib/boot/cert.go
index b2b8c896c..2b38dab05 100644
--- a/lib/boot/cert.go
+++ b/lib/boot/cert.go
@@ -32,14 +32,14 @@ func (createCertificates) Run(ctx context.Context, fail func(error), super *Supe
} else {
san += fmt.Sprintf(",DNS:%s", super.ListenHost)
}
- if hostname, err := os.Hostname(); err != nil {
+ hostname, err := os.Hostname()
+ if err != nil {
return fmt.Errorf("hostname: %w", err)
- } else {
- san += ",DNS:" + hostname
}
+ san += ",DNS:" + hostname
// Generate root key
- err := super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
+ err = super.RunProgram(ctx, super.tempdir, runOptions{}, "openssl", "genrsa", "-out", "rootCA.key", "4096")
if err != nil {
return err
}
diff --git a/lib/crunchrun/container_gateway.go b/lib/crunchrun/container_gateway.go
index 1116c4bb1..19feac169 100644
--- a/lib/crunchrun/container_gateway.go
+++ b/lib/crunchrun/container_gateway.go
@@ -49,9 +49,8 @@ func (gw *Gateway) Start() error {
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
if c.User() == "_" {
return nil, nil
- } else {
- return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
}
+ return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
},
PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
if c.User() == "_" {
@@ -60,9 +59,8 @@ func (gw *Gateway) Start() error {
"pubkey-fp": ssh.FingerprintSHA256(pubKey),
},
}, nil
- } else {
- return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
}
+ return nil, fmt.Errorf("cannot specify user %q via ssh client", c.User())
},
}
pvt, err := rsa.GenerateKey(rand.Reader, 2048)
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index ea3cb6899..13bb3bf80 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -321,9 +321,8 @@ func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, m
if c.APIHost == "" {
if c.loadedFromEnv {
return errors.New("ARVADOS_API_HOST and/or ARVADOS_API_TOKEN environment variables are not set")
- } else {
- return errors.New("arvados.Client cannot perform request: APIHost is not set")
}
+ return errors.New("arvados.Client cannot perform request: APIHost is not set")
}
urlString := c.apiURL(path)
urlValues, err := anythingToValues(params)
commit 9d9f7df00a092de9c2813cb99907092a20e34fd5
Author: Ward Vandewege <ward at curii.com>
Date: Thu Feb 25 17:52:24 2021 -0500
When a compute node comes up, we run `mkfs.xfs` on a newly encrypted
partition. Sometimes there are ghost filesystem signatures on those
partitions, make mkfs.xfs ignore partition signatures.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/tools/compute-images/scripts/usr-local-bin-ensure-encrypted-partitions.sh b/tools/compute-images/scripts/usr-local-bin-ensure-encrypted-partitions.sh
index 08579bf19..462158e04 100644
--- a/tools/compute-images/scripts/usr-local-bin-ensure-encrypted-partitions.sh
+++ b/tools/compute-images/scripts/usr-local-bin-ensure-encrypted-partitions.sh
@@ -114,7 +114,7 @@ head -c321 /dev/urandom >"$KEYPATH"
echo YES | cryptsetup luksFormat "$LVPATH" "$KEYPATH"
cryptsetup --key-file "$KEYPATH" luksOpen "$LVPATH" "$(basename "$CRYPTPATH")"
shred -u "$KEYPATH"
-mkfs.xfs "$CRYPTPATH"
+mkfs.xfs -f "$CRYPTPATH"
# First make sure docker is not using /tmp, then unmount everything under it.
if [ -d /etc/sv/docker.io ]
commit cc088a7b64d8bf0fc827a5df619f875367b9fffa
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Fri Feb 19 12:48:32 2021 -0300
17295: Adds cluster ID validation on the config file.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/lib/config/export_test.go b/lib/config/export_test.go
index 7af117e38..f11b65f45 100644
--- a/lib/config/export_test.go
+++ b/lib/config/export_test.go
@@ -17,7 +17,7 @@ var _ = check.Suite(&ExportSuite{})
type ExportSuite struct{}
func (s *ExportSuite) TestExport(c *check.C) {
- confdata := strings.Replace(string(DefaultYAML), "SAMPLE", "testkey", -1)
+ confdata := strings.Replace(string(DefaultYAML), "SAMPLE", "12345", -1)
cfg, err := testLoader(c, confdata, nil).Load()
c.Assert(err, check.IsNil)
cluster, err := cfg.GetCluster("xxxxx")
diff --git a/lib/config/load.go b/lib/config/load.go
index 7eb403910..f68235937 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -270,7 +270,18 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
// Check for known mistakes
for id, cc := range cfg.Clusters {
+ for remote, _ := range cc.RemoteClusters {
+ if remote == "*" || remote == "SAMPLE" {
+ continue
+ }
+ err = ldr.checkClusterID(fmt.Sprintf("Clusters.%s.RemoteClusters.%s", id, remote), remote, true)
+ if err != nil {
+ return nil, err
+ }
+ }
for _, err = range []error{
+ ldr.checkClusterID(fmt.Sprintf("Clusters.%s", id), id, false),
+ ldr.checkClusterID(fmt.Sprintf("Clusters.%s.Login.LoginCluster", id), cc.Login.LoginCluster, true),
ldr.checkToken(fmt.Sprintf("Clusters.%s.ManagementToken", id), cc.ManagementToken),
ldr.checkToken(fmt.Sprintf("Clusters.%s.SystemRootToken", id), cc.SystemRootToken),
ldr.checkToken(fmt.Sprintf("Clusters.%s.Collections.BlobSigningKey", id), cc.Collections.BlobSigningKey),
@@ -286,6 +297,17 @@ func (ldr *Loader) Load() (*arvados.Config, error) {
return &cfg, nil
}
+var acceptableClusterIDRe = regexp.MustCompile(`^[a-z0-9]{5}$`)
+
+func (ldr *Loader) checkClusterID(label, clusterID string, emptyStringOk bool) error {
+ if emptyStringOk && clusterID == "" {
+ return nil
+ } else if !acceptableClusterIDRe.MatchString(clusterID) {
+ return fmt.Errorf("%s: cluster ID should be 5 alphanumeric characters", label)
+ }
+ return nil
+}
+
var acceptableTokenRe = regexp.MustCompile(`^[a-zA-Z0-9]+$`)
var acceptableTokenLength = 32
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index 4cdebf62b..91bd6a743 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -351,7 +351,7 @@ Clusters:
Proxy: true
`, `
Clusters:
- abcdef:
+ abcde:
ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
Collections:
commit 9b9c01638a9b4bd9ff7f32fd186daf77d222eec6
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Fri Feb 19 12:45:11 2021 -0300
17295: Adds tests exposing the bug.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/lib/config/load_test.go b/lib/config/load_test.go
index c9ed37b83..4cdebf62b 100644
--- a/lib/config/load_test.go
+++ b/lib/config/load_test.go
@@ -330,6 +330,45 @@ Clusters:
c.Check(err, check.ErrorMatches, `Clusters.zzzzz.PostgreSQL.Connection: multiple entries for "(dbname|host)".*`)
}
+func (s *LoadSuite) TestBadClusterIDs(c *check.C) {
+ for _, data := range []string{`
+Clusters:
+ 123456:
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+`, `
+Clusters:
+ 12345:
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ RemoteClusters:
+ Zzzzz:
+ Host: Zzzzz.arvadosapi.com
+ Proxy: true
+`, `
+Clusters:
+ abcdef:
+ ManagementToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ SystemRootToken: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Collections:
+ BlobSigningKey: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+ Login:
+ LoginCluster: zz-zz
+`,
+ } {
+ c.Log(data)
+ v, err := testLoader(c, data, nil).Load()
+ if v != nil {
+ c.Logf("%#v", v.Clusters)
+ }
+ c.Check(err, check.ErrorMatches, `.*cluster ID should be 5 alphanumeric characters.*`)
+ }
+}
+
func (s *LoadSuite) TestBadType(c *check.C) {
for _, data := range []string{`
Clusters:
commit e5f1a98ef29dca94fb5742ae2fec8524bd093fb5
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Tue Feb 9 19:27:23 2021 -0300
16736: Adds tests to confirm expires_at gets properly set on runtime tokens.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index a92fc7105..e3b2291bc 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -695,6 +695,8 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
arvadostest.SetServiceURL(&s.testHandler.Cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
s.testHandler.Cluster.ClusterID = "zzzzz"
s.testHandler.Cluster.SystemRootToken = arvadostest.SystemRootToken
+ s.testHandler.Cluster.API.MaxTokenLifetime = arvados.Duration(time.Hour)
+ s.testHandler.Cluster.Collections.BlobSigningTTL = arvados.Duration(336 * time.Hour) // For some reason, this was set to 0h
resp := s.testRequest(req).Result()
c.Check(resp.StatusCode, check.Equals, http.StatusOK)
@@ -703,8 +705,22 @@ func (s *FederationSuite) TestCreateRemoteContainerRequestCheckRuntimeToken(c *c
// Runtime token must match zzzzz cluster
c.Check(cr.RuntimeToken, check.Matches, "v2/zzzzz-gj3su-.*")
+
// RuntimeToken must be different than the Original Token we originally did the request with.
c.Check(cr.RuntimeToken, check.Not(check.Equals), arvadostest.ActiveTokenV2)
+
+ // Runtime token should not have an expiration based on API.MaxTokenLifetime
+ req2 := httptest.NewRequest("GET", "/arvados/v1/api_client_authorizations/current", nil)
+ req2.Header.Set("Authorization", "Bearer "+cr.RuntimeToken)
+ req2.Header.Set("Content-type", "application/json")
+ resp = s.testRequest(req2).Result()
+ c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+ var aca arvados.APIClientAuthorization
+ c.Check(json.NewDecoder(resp.Body).Decode(&aca), check.IsNil)
+ c.Check(aca.ExpiresAt, check.NotNil) // Time.Now()+BlobSigningTTL
+ t, _ := time.Parse(time.RFC3339Nano, aca.ExpiresAt)
+ c.Check(t.After(time.Now().Add(s.testHandler.Cluster.API.MaxTokenLifetime.Duration())), check.Equals, true)
+ c.Check(t.Before(time.Now().Add(s.testHandler.Cluster.Collections.BlobSigningTTL.Duration())), check.Equals, true)
}
func (s *FederationSuite) TestCreateRemoteContainerRequestCheckSetRuntimeToken(c *check.C) {
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index 8feee77ff..e6d945a00 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -605,7 +605,8 @@ class Container < ArvadosModel
self.runtime_auth_scopes = ["all"]
end
- # generate a new token
+ # Generate a new token. This runs with admin credentials as it's done by a
+ # dispatcher user, so expires_at isn't enforced by API.MaxTokenLifetime.
self.auth = ApiClientAuthorization.
create!(user_id: User.find_by_uuid(self.runtime_user_uuid).id,
api_client_id: 0,
diff --git a/services/api/test/integration/container_dispatch_test.rb b/services/api/test/integration/container_dispatch_test.rb
index 61e01da64..556b889fa 100644
--- a/services/api/test/integration/container_dispatch_test.rb
+++ b/services/api/test/integration/container_dispatch_test.rb
@@ -11,7 +11,6 @@ class ContainerDispatchTest < ActionDispatch::IntegrationTest
get("/arvados/v1/api_client_authorizations/current",
headers: authheaders)
assert_response 200
- #assert_not_empty json_response['uuid']
system_auth_uuid = json_response['uuid']
post("/arvados/v1/containers/#{containers(:queued).uuid}/lock",
diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 35e2b7ed1..375ab5a7b 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -750,6 +750,17 @@ class ContainerTest < ActiveSupport::TestCase
check_no_change_from_cancelled c
end
+ test "Container locked with non-expiring token" do
+ Rails.configuration.API.TokenMaxLifetime = 1.hour
+ set_user_from_auth :active
+ c, _ = minimal_new
+ set_user_from_auth :dispatch1
+ assert c.lock, show_errors(c)
+ refute c.auth.nil?
+ assert c.auth.expires_at.nil?
+ assert c.auth.user_id == User.find_by_uuid(users(:active).uuid).id
+ end
+
test "Container locked cancel with log" do
set_user_from_auth :active
c, _ = minimal_new
commit c38f4a309ef8dc034e0a04e13d258206b02b2cdf
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Mon Feb 8 19:01:41 2021 -0300
16736: Simplifies conditionals.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index 03e1b38fd..ee63c4d93 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -390,9 +390,7 @@ class ApiClientAuthorization < ArvadosModel
def clamp_token_expiration
if !current_user.andand.is_admin && Rails.configuration.API.MaxTokenLifetime > 0
max_token_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime
- if self.new_record? && (self.expires_at.nil? || self.expires_at > max_token_expiration)
- self.expires_at = max_token_expiration
- elsif !self.new_record? && self.expires_at_changed? && (self.expires_at.nil? || self.expires_at > max_token_expiration)
+ if (self.new_record? || self.expires_at_changed?) && (self.expires_at.nil? || self.expires_at > max_token_expiration)
self.expires_at = max_token_expiration
end
end
commit 0e4fcd09203ca889efd938203bdf23aaf3083dc6
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Mon Feb 8 18:16:52 2021 -0300
16736: Replaces Time.now with db_current_time on token expiration handling code.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/services/api/app/controllers/user_sessions_controller.rb b/services/api/app/controllers/user_sessions_controller.rb
index da0523d2b..8e9a26b7a 100644
--- a/services/api/app/controllers/user_sessions_controller.rb
+++ b/services/api/app/controllers/user_sessions_controller.rb
@@ -158,9 +158,9 @@ class UserSessionsController < ApplicationController
end
if Rails.configuration.Login.TokenLifetime > 0
if token_expiration == nil
- token_expiration = Time.now + Rails.configuration.Login.TokenLifetime
+ token_expiration = db_current_time + Rails.configuration.Login.TokenLifetime
else
- token_expiration = [token_expiration, Time.now + Rails.configuration.Login.TokenLifetime].min
+ token_expiration = [token_expiration, db_current_time + Rails.configuration.Login.TokenLifetime].min
end
end
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index 4218645d5..03e1b38fd 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -7,6 +7,7 @@ class ApiClientAuthorization < ArvadosModel
include KindAndEtag
include CommonApiTemplate
extend CurrentApiClient
+ extend DbCurrentTime
belongs_to :api_client
belongs_to :user
@@ -356,7 +357,7 @@ class ApiClientAuthorization < ArvadosModel
auth.update_attributes!(user: user,
api_token: stored_secret,
api_client_id: 0,
- expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
+ expires_at: db_current_time + Rails.configuration.Login.RemoteTokenRefresh)
Rails.logger.debug "cached remote token #{token_uuid} with secret #{stored_secret} in local db"
auth.api_token = secret
return auth
@@ -388,7 +389,7 @@ class ApiClientAuthorization < ArvadosModel
def clamp_token_expiration
if !current_user.andand.is_admin && Rails.configuration.API.MaxTokenLifetime > 0
- max_token_expiration = Time.now + Rails.configuration.API.MaxTokenLifetime
+ max_token_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime
if self.new_record? && (self.expires_at.nil? || self.expires_at > max_token_expiration)
self.expires_at = max_token_expiration
elsif !self.new_record? && self.expires_at_changed? && (self.expires_at.nil? || self.expires_at > max_token_expiration)
diff --git a/services/api/test/integration/api_client_authorizations_api_test.rb b/services/api/test/integration/api_client_authorizations_api_test.rb
index 3a7b20131..ce79fc557 100644
--- a/services/api/test/integration/api_client_authorizations_api_test.rb
+++ b/services/api/test/integration/api_client_authorizations_api_test.rb
@@ -5,6 +5,8 @@
require 'test_helper'
class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
+ include DbCurrentTime
+ extend DbCurrentTime
fixtures :all
test "create system auth" do
@@ -74,12 +76,12 @@ class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
assert_response 403
end
- [nil, Time.now + 2.hours].each do |desired_expiration|
+ [nil, db_current_time + 2.hours].each do |desired_expiration|
test "expires_at gets clamped on non-admins when API.MaxTokenLifetime is set and desired expires_at #{desired_expiration.nil? ? 'is not set' : 'exceeds the limit'}" do
Rails.configuration.API.MaxTokenLifetime = 1.hour
# Test token creation
- start_t = Time.now
+ start_t = db_current_time
post "/arvados/v1/api_client_authorizations",
params: {
:format => :json,
@@ -89,7 +91,7 @@ class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
}
},
headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
- end_t = Time.now
+ end_t = db_current_time
assert_response 200
expiration_t = json_response['expires_at'].to_time
assert_operator expiration_t.to_f, :>, (start_t + Rails.configuration.API.MaxTokenLifetime).to_f
@@ -102,7 +104,7 @@ class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
# Test token update
previous_expiration = expiration_t
token_uuid = json_response["uuid"]
- start_t = Time.now
+ start_t = db_current_time
put "/arvados/v1/api_client_authorizations/#{token_uuid}",
params: {
:api_client_authorization => {
@@ -110,7 +112,7 @@ class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
}
},
headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
- end_t = Time.now
+ end_t = db_current_time
assert_response 200
expiration_t = json_response['expires_at'].to_time
assert_operator previous_expiration.to_f, :<, expiration_t.to_f
@@ -146,7 +148,7 @@ class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
previous_expiration = json_response['expires_at']
token_uuid = json_response['uuid']
if previous_expiration.nil?
- desired_updated_expiration = Time.now + Rails.configuration.API.MaxTokenLifetime + 1.hour
+ desired_updated_expiration = db_current_time + Rails.configuration.API.MaxTokenLifetime + 1.hour
else
desired_updated_expiration = nil
end
commit 3ff8939a3d9dab549ff8c250e2338dc515ef2c86
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Thu Feb 4 14:46:30 2021 -0300
16736: Updates arvados-login-sync to support expiring tokens.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/services/login-sync/bin/arvados-login-sync b/services/login-sync/bin/arvados-login-sync
index a9bff0546..d8836f19b 100755
--- a/services/login-sync/bin/arvados-login-sync
+++ b/services/login-sync/bin/arvados-login-sync
@@ -9,6 +9,7 @@ require 'arvados'
require 'etc'
require 'fileutils'
require 'yaml'
+require 'optparse'
req_envs = %w(ARVADOS_API_HOST ARVADOS_API_TOKEN ARVADOS_VIRTUAL_MACHINE_UUID)
req_envs.each do |k|
@@ -17,16 +18,20 @@ req_envs.each do |k|
end
end
-exclusive_mode = ARGV.index("--exclusive")
+options = {}
+OptionParser.new do |parser|
+ parser.on('--exclusive', 'Manage SSH keys file exclusively.')
+ parser.on('--rotate-tokens', 'Always create new user tokens. Usually needed with --token-lifetime.')
+ parser.on('--skip-missing-users', "Don't try to create any local accounts.")
+ parser.on('--token-lifetime SECONDS', 'Create user tokens that expire after SECONDS.', Integer)
+end.parse!(into: options)
+
exclusive_banner = "#######################################################################################
# THIS FILE IS MANAGED BY #{$0} -- CHANGES WILL BE OVERWRITTEN #
#######################################################################################\n\n"
start_banner = "### BEGIN Arvados-managed keys -- changes between markers will be overwritten\n"
end_banner = "### END Arvados-managed keys -- changes between markers will be overwritten\n"
-# Don't try to create any local accounts
-skip_missing_users = ARGV.index("--skip-missing-users")
-
keys = ''
begin
@@ -64,7 +69,7 @@ begin
begin
pwnam[l[:username]] = Etc.getpwnam(l[:username])
rescue
- if skip_missing_users
+ if options[:"skip-missing-users"]
STDERR.puts "Account #{l[:username]} not found. Skipping"
true
end
@@ -165,7 +170,7 @@ begin
oldkeys = ""
end
- if exclusive_mode
+ if options[:exclusive]
newkeys = exclusive_banner + newkeys
elsif oldkeys.start_with?(exclusive_banner)
newkeys = start_banner + newkeys + end_banner
@@ -192,8 +197,12 @@ begin
tokenfile = File.join(configarvados, "settings.conf")
begin
- if !File.exist?(tokenfile)
- user_token = logincluster_arv.api_client_authorization.create(api_client_authorization: {owner_uuid: l[:user_uuid], api_client_id: 0})
+ if !File.exist?(tokenfile) || options[:"rotate-tokens"]
+ aca_params = {owner_uuid: l[:user_uuid], api_client_id: 0}
+ if options[:"token-lifetime"] && options[:"token-lifetime"] > 0
+ aca_params.merge!(expires_at: (Time.now + options[:"token-lifetime"]))
+ end
+ user_token = logincluster_arv.api_client_authorization.create(api_client_authorization: aca_params)
f = File.new(tokenfile, 'w')
f.write("ARVADOS_API_HOST=#{ENV['ARVADOS_API_HOST']}\n")
f.write("ARVADOS_API_TOKEN=v2/#{user_token[:uuid]}/#{user_token[:api_token]}\n")
commit 2c02c23493558b7a0b52b60ab843d5ac0db0cd2a
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Thu Jan 28 18:30:30 2021 -0300
16736: Fixes typo.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index b7d555d08..6f72c02c1 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -161,7 +161,7 @@ Clusters:
# Limits for how long a client token created by regular users can be valid,
# and also is used as a default expiration policy when no expiration date is
# specified.
- # Default value zero menans token expirations don't get clamped and no
+ # Default value zero means token expirations don't get clamped and no
# default expiration is set.
MaxTokenLifetime: 0s
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 9481d7a5c..50869bf61 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -167,7 +167,7 @@ Clusters:
# Limits for how long a client token created by regular users can be valid,
# and also is used as a default expiration policy when no expiration date is
# specified.
- # Default value zero menans token expirations don't get clamped and no
+ # Default value zero means token expirations don't get clamped and no
# default expiration is set.
MaxTokenLifetime: 0s
commit 53d3ee8c8b0566162661fbc571deff0106b85020
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Thu Jan 28 17:44:59 2021 -0300
16736: Limit token's expires_at depending on the cluster config and user type.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb b/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb
index 59e359232..99446688d 100644
--- a/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb
+++ b/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb
@@ -17,6 +17,7 @@ class Arvados::V1::ApiClientAuthorizationsController < ApplicationController
scopes: {type: 'array', required: false}
}
end
+
def create_system_auth
@object = ApiClientAuthorization.
new(user_id: system_user.id,
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index 9290e01a1..4218645d5 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -13,6 +13,8 @@ class ApiClientAuthorization < ArvadosModel
after_initialize :assign_random_api_token
serialize :scopes, Array
+ before_validation :clamp_token_expiration
+
api_accessible :user, extend: :common do |t|
t.add :owner_uuid
t.add :user_id
@@ -384,6 +386,17 @@ class ApiClientAuthorization < ArvadosModel
protected
+ def clamp_token_expiration
+ if !current_user.andand.is_admin && Rails.configuration.API.MaxTokenLifetime > 0
+ max_token_expiration = Time.now + Rails.configuration.API.MaxTokenLifetime
+ if self.new_record? && (self.expires_at.nil? || self.expires_at > max_token_expiration)
+ self.expires_at = max_token_expiration
+ elsif !self.new_record? && self.expires_at_changed? && (self.expires_at.nil? || self.expires_at > max_token_expiration)
+ self.expires_at = max_token_expiration
+ end
+ end
+ end
+
def permission_to_create
current_user.andand.is_admin or (current_user.andand.id == self.user_id)
end
diff --git a/services/api/config/arvados_config.rb b/services/api/config/arvados_config.rb
index 5327713f6..8f4395dad 100644
--- a/services/api/config/arvados_config.rb
+++ b/services/api/config/arvados_config.rb
@@ -92,6 +92,7 @@ arvcfg.declare_config "API.DisabledAPIs", Hash, :disable_api_methods, ->(cfg, k,
arvcfg.declare_config "API.MaxRequestSize", Integer, :max_request_size
arvcfg.declare_config "API.MaxIndexDatabaseRead", Integer, :max_index_database_read
arvcfg.declare_config "API.MaxItemsPerResponse", Integer, :max_items_per_response
+arvcfg.declare_config "API.MaxTokenLifetime", ActiveSupport::Duration
arvcfg.declare_config "API.AsyncPermissionsUpdateInterval", ActiveSupport::Duration, :async_permissions_update_interval
arvcfg.declare_config "Users.AutoSetupNewUsers", Boolean, :auto_setup_new_users
arvcfg.declare_config "Users.AutoSetupNewUsersWithVmUUID", String, :auto_setup_new_users_with_vm_uuid
diff --git a/services/api/test/integration/api_client_authorizations_api_test.rb b/services/api/test/integration/api_client_authorizations_api_test.rb
index 296ab8a2f..3a7b20131 100644
--- a/services/api/test/integration/api_client_authorizations_api_test.rb
+++ b/services/api/test/integration/api_client_authorizations_api_test.rb
@@ -74,4 +74,95 @@ class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
assert_response 403
end
+ [nil, Time.now + 2.hours].each do |desired_expiration|
+ test "expires_at gets clamped on non-admins when API.MaxTokenLifetime is set and desired expires_at #{desired_expiration.nil? ? 'is not set' : 'exceeds the limit'}" do
+ Rails.configuration.API.MaxTokenLifetime = 1.hour
+
+ # Test token creation
+ start_t = Time.now
+ post "/arvados/v1/api_client_authorizations",
+ params: {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:active).uuid,
+ :expires_at => desired_expiration,
+ }
+ },
+ headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
+ end_t = Time.now
+ assert_response 200
+ expiration_t = json_response['expires_at'].to_time
+ assert_operator expiration_t.to_f, :>, (start_t + Rails.configuration.API.MaxTokenLifetime).to_f
+ if !desired_expiration.nil?
+ assert_operator expiration_t.to_f, :<, desired_expiration.to_f
+ else
+ assert_operator expiration_t.to_f, :<, (end_t + Rails.configuration.API.MaxTokenLifetime).to_f
+ end
+
+ # Test token update
+ previous_expiration = expiration_t
+ token_uuid = json_response["uuid"]
+ start_t = Time.now
+ put "/arvados/v1/api_client_authorizations/#{token_uuid}",
+ params: {
+ :api_client_authorization => {
+ :expires_at => desired_expiration
+ }
+ },
+ headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
+ end_t = Time.now
+ assert_response 200
+ expiration_t = json_response['expires_at'].to_time
+ assert_operator previous_expiration.to_f, :<, expiration_t.to_f
+ assert_operator expiration_t.to_f, :>, (start_t + Rails.configuration.API.MaxTokenLifetime).to_f
+ if !desired_expiration.nil?
+ assert_operator expiration_t.to_f, :<, desired_expiration.to_f
+ else
+ assert_operator expiration_t.to_f, :<, (end_t + Rails.configuration.API.MaxTokenLifetime).to_f
+ end
+ end
+
+ test "expires_at can be set to #{desired_expiration.nil? ? 'nil' : 'exceed the limit'} by admins when API.MaxTokenLifetime is set" do
+ Rails.configuration.API.MaxTokenLifetime = 1.hour
+
+ # Test token creation
+ post "/arvados/v1/api_client_authorizations",
+ params: {
+ :format => :json,
+ :api_client_authorization => {
+ :owner_uuid => users(:admin).uuid,
+ :expires_at => desired_expiration,
+ }
+ },
+ headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
+ assert_response 200
+ if desired_expiration.nil?
+ assert json_response['expires_at'].nil?
+ else
+ assert_equal json_response['expires_at'].to_time.to_i, desired_expiration.to_i
+ end
+
+ # Test token update (reverse the above behavior)
+ previous_expiration = json_response['expires_at']
+ token_uuid = json_response['uuid']
+ if previous_expiration.nil?
+ desired_updated_expiration = Time.now + Rails.configuration.API.MaxTokenLifetime + 1.hour
+ else
+ desired_updated_expiration = nil
+ end
+ put "/arvados/v1/api_client_authorizations/#{token_uuid}",
+ params: {
+ :api_client_authorization => {
+ :expires_at => desired_updated_expiration,
+ }
+ },
+ headers: {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
+ assert_response 200
+ if desired_updated_expiration.nil?
+ assert json_response['expires_at'].nil?
+ else
+ assert_equal json_response['expires_at'].to_time.to_i, desired_updated_expiration.to_i
+ end
+ end
+ end
end
diff --git a/services/api/test/integration/user_sessions_test.rb b/services/api/test/integration/user_sessions_test.rb
index fcc0ce4e5..6e951499a 100644
--- a/services/api/test/integration/user_sessions_test.rb
+++ b/services/api/test/integration/user_sessions_test.rb
@@ -53,19 +53,19 @@ class UserSessionsApiTest < ActionDispatch::IntegrationTest
test 'existing user login' do
mock_auth_with(identity_url: "https://active-user.openid.local")
u = assigns(:user)
- assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ assert_equal users(:active).uuid, u.uuid
end
test 'user redirect_to_user_uuid' do
mock_auth_with(identity_url: "https://redirects-to-active-user.openid.local")
u = assigns(:user)
- assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ assert_equal users(:active).uuid, u.uuid
end
test 'user double redirect_to_user_uuid' do
mock_auth_with(identity_url: "https://double-redirects-to-active-user.openid.local")
u = assigns(:user)
- assert_equal 'zzzzz-tpzed-xurymjxw79nv3jz', u.uuid
+ assert_equal users(:active).uuid, u.uuid
end
test 'create new user during omniauth callback' do
commit 92920a2a882edfe3d7b02c80bd694b9b61b97c85
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Fri Jan 15 12:34:57 2021 -0300
16736: Adds API.MaxTokenLifetime config knob.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index c644de374..b7d555d08 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -158,6 +158,13 @@ Clusters:
dbname: ""
SAMPLE: ""
API:
+ # Limits for how long a client token created by regular users can be valid,
+ # and also is used as a default expiration policy when no expiration date is
+ # specified.
+ # Default value zero menans token expirations don't get clamped and no
+ # default expiration is set.
+ MaxTokenLifetime: 0s
+
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Note: You must separately configure the upstream web server or
diff --git a/lib/config/export.go b/lib/config/export.go
index 3d0e27c72..b6531c59d 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -69,6 +69,7 @@ var whitelist = map[string]bool{
"API.MaxKeepBlobBuffers": false,
"API.MaxRequestAmplification": false,
"API.MaxRequestSize": true,
+ "API.MaxTokenLifetime": false,
"API.RequestTimeout": true,
"API.SendTimeout": true,
"API.WebsocketClientEventQueue": false,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 8354102c2..9481d7a5c 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -164,6 +164,13 @@ Clusters:
dbname: ""
SAMPLE: ""
API:
+ # Limits for how long a client token created by regular users can be valid,
+ # and also is used as a default expiration policy when no expiration date is
+ # specified.
+ # Default value zero menans token expirations don't get clamped and no
+ # default expiration is set.
+ MaxTokenLifetime: 0s
+
# Maximum size (in bytes) allowed for a single API request. This
# limit is published in the discovery document for use by clients.
# Note: You must separately configure the upstream web server or
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 4a56c9302..4ccb1ef5d 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -86,6 +86,7 @@ type Cluster struct {
MaxKeepBlobBuffers int
MaxRequestAmplification int
MaxRequestSize int
+ MaxTokenLifetime Duration
RequestTimeout Duration
SendTimeout Duration
WebsocketClientEventQueue int
commit d5d9c40b758dfaf48043b58177c96168925068aa
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Thu Sep 10 11:11:33 2020 -0300
16736: Enhances tests about login issued tokens.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/services/api/test/functional/user_sessions_controller_test.rb b/services/api/test/functional/user_sessions_controller_test.rb
index 129464cf1..1f9196893 100644
--- a/services/api/test/functional/user_sessions_controller_test.rb
+++ b/services/api/test/functional/user_sessions_controller_test.rb
@@ -30,6 +30,7 @@ class UserSessionsControllerTest < ActionController::TestCase
authorize_with :inactive
api_client_page = 'http://client.example.com/home'
get :login, params: {return_to: api_client_page}
+ assert_response :redirect
assert_not_nil assigns(:api_client)
assert_nil assigns(:api_client_auth).expires_at
end
@@ -40,6 +41,7 @@ class UserSessionsControllerTest < ActionController::TestCase
authorize_with :inactive
api_client_page = 'http://client.example.com/home'
get :login, params: {return_to: api_client_page}
+ assert_response :redirect
assert_not_nil assigns(:api_client)
api_client_auth = assigns(:api_client_auth)
assert_in_delta(api_client_auth.expires_at,
commit 766582be2d323b6fc12bea181f1239b4649590f6
Author: Ward Vandewege <ward at curii.com>
Date: Thu Feb 18 15:57:57 2021 -0500
Documentation: fix typos.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/sdk/python/cookbook.html.textile.liquid b/doc/sdk/python/cookbook.html.textile.liquid
index 3aa01bbb5..ff3bcf90e 100644
--- a/doc/sdk/python/cookbook.html.textile.liquid
+++ b/doc/sdk/python/cookbook.html.textile.liquid
@@ -237,7 +237,7 @@ with c.open(filename, "rb") as reader:
print("Finished downloading %s" % filename)
{% endcodeblock %}
-h2. Copy files from a collection a new collection
+h2. Copy files from a collection to a new collection
{% codeblock as python %}
import arvados.collection
@@ -258,7 +258,7 @@ target.save_new(name=target_name, owner_uuid=target_project)
print("Created collection %s" % target.manifest_locator())
{% endcodeblock %}
-h2. Copy files from a collection another collection
+h2. Copy files from a collection to another collection
{% codeblock as python %}
import arvados.collection
commit 61de965bf3fc966fce4d70f1d85a0c26016000e3
Author: Tom Clegg <tom at curii.com>
Date: Thu Feb 18 11:43:27 2021 -0500
17398: Skip gateway server if dispatcher does not send desired addr.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 7d6fb4ed4..969682f46 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -1887,10 +1887,12 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
Log: cr.CrunchLog,
}
os.Unsetenv("GatewayAuthSecret")
- err = cr.gateway.Start()
- if err != nil {
- log.Printf("error starting gateway server: %s", err)
- return 1
+ if cr.gateway.Address != "" {
+ err = cr.gateway.Start()
+ if err != nil {
+ log.Printf("error starting gateway server: %s", err)
+ return 1
+ }
}
parentTemp, tmperr := cr.MkTempDir("", "crunch-run."+containerID+".")
commit 7fc7398eeb37f2d76da55561eff5a4f77ee73d50
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Feb 15 16:37:37 2021 -0500
Update vscode training page no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/cwl/arvados-vscode-training.html.md.liquid b/doc/user/cwl/arvados-vscode-training.html.md.liquid
index 80477e9d7..1858f5b57 100644
--- a/doc/user/cwl/arvados-vscode-training.html.md.liquid
+++ b/doc/user/cwl/arvados-vscode-training.html.md.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
Imported from https://github.com/arvados/arvados-vscode-cwl-training
-git hash: e21c963c05fb35bd8be15e24f72045e9c1cec668
+git hash: f39d44c1bdb2f82ec8f22ade874ca70544531289
{% endcomment %}
@@ -79,16 +79,16 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Vscode: On the left sidebar, choose `Explorer` 
1. Select `Clone Repository` and enter [https://github.com/arvados/arvados-vscode-cwl-training](https://github.com/arvados/arvados-vscode-cwl-training), then click `Open`
1. If asked `Would you like to open the cloned repository?` choose `Open`
-1. Vscode: Click on the `Terminal` menu
+1. Go to Arvados Workbench
+ 1. Workbench: In the user menu, select `Current token`
+ 1. Workbench: Click on `Copy to Clipboard`.
+ 1. Workbench: You should see a notification `Token copied to clipboard`.
+ 1. Go to Vscode
+ 1. Vscode: Click on the `Terminal` menu
1. Vscode: Click `Run Task…`
1. Vscode: Select `Configure Arvados`
- 1. Go to Arvados Workbench
- 1. Workbench: In the user menu, select `Current token`
- 1. Workbench: Copy the text following `ARVADOS_API_HOST=` to the end of the line
- 1. Vscode: Paste the string at the `Value for ARVADOS_API_HOST` prompt
- 1. Workbench: Copy the text following `ARVADOS_API_TOKEN=` to the end of the line
- 1. Vscode: Paste the string at the `Value for ARVADOS_API_TOKEN` prompt
- 1. This will create files called `API_HOST` and `API_TOKEN`
+ 1. Vscode: Paste text into the `Current API_TOKEN and API_HOST from Workbench` prompt
+ 1. Vscode: This will create files called `API_HOST` and `API_TOKEN`
## 3. Register & run a workflow
commit 2f8ee11fc4f756f6d2774cf6c6dae96ec4a16f8b
Author: Nico Cesar <nico at nicocesar.com>
Date: Mon Feb 15 15:44:50 2021 -0500
Added note about /bin/false as UNIX login
Arvados-DCO-1.1-Signed-off-by: Nico Cesar <nico at curii.com>
diff --git a/doc/install/setup-login.html.textile.liquid b/doc/install/setup-login.html.textile.liquid
index aec82cfe2..d11fec9e1 100644
--- a/doc/install/setup-login.html.textile.liquid
+++ b/doc/install/setup-login.html.textile.liquid
@@ -98,7 +98,7 @@ Enable PAM authentication in @config.yml@:
Check the "default config file":{{site.baseurl}}/admin/config.html for more PAM configuration options.
-The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a shell account and password on the controller host itself. This can be convenient for a single-user or test cluster.
+The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a UNIX account and password on the controller host itself. This can be convenient for a single-user or test cluster. User accounts can have @/dev/false@ as the shell in order to allow the user to log into Arvados but not log into a shell on the controller host.
PAM can also be configured to use different backends like LDAP. In a production environment, PAM configuration should use the service name ("arvados" by default) to set a separate policy for Arvados logins: generally, Arvados users should not have shell accounts on the controller node.
commit 0e68845158573e94e5a6e2ed595b5a370420a543
Author: Tom Clegg <tom at curii.com>
Date: Mon Feb 15 11:06:35 2021 -0500
17384: Respect CrunchRunCommand and CrunchRunArgumentsList in a-d-c.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 68e518732..c644de374 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -831,7 +831,11 @@ Clusters:
# stale locks from a previous dispatch process.
StaleLockTimeout: 1m
- # The crunch-run command to manage the container on a node
+ # The crunch-run command used to start a container on a worker node.
+ #
+ # When dispatching to cloud VMs, this is used only if
+ # DeployRunnerBinary in the CloudVMs section is set to the empty
+ # string.
CrunchRunCommand: "crunch-run"
# Extra arguments to add to crunch-run invocation
@@ -1052,7 +1056,7 @@ Clusters:
#
# Use the empty string to disable this step: nothing will be
# copied, and cloud instances are assumed to have a suitable
- # version of crunch-run installed.
+ # version of crunch-run installed; see CrunchRunCommand above.
DeployRunnerBinary: "/proc/self/exe"
# Tags to add on all resources (VMs, NICs, disks) created by
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 8ef787771..8354102c2 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -837,7 +837,11 @@ Clusters:
# stale locks from a previous dispatch process.
StaleLockTimeout: 1m
- # The crunch-run command to manage the container on a node
+ # The crunch-run command used to start a container on a worker node.
+ #
+ # When dispatching to cloud VMs, this is used only if
+ # DeployRunnerBinary in the CloudVMs section is set to the empty
+ # string.
CrunchRunCommand: "crunch-run"
# Extra arguments to add to crunch-run invocation
@@ -1058,7 +1062,7 @@ Clusters:
#
# Use the empty string to disable this step: nothing will be
# copied, and cloud instances are assumed to have a suitable
- # version of crunch-run installed.
+ # version of crunch-run installed; see CrunchRunCommand above.
DeployRunnerBinary: "/proc/self/exe"
# Tags to add on all resources (VMs, NICs, disks) created by
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index d5d90bf35..8752ee054 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -52,8 +52,10 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
s.cluster = &arvados.Cluster{
ManagementToken: "test-management-token",
Containers: arvados.ContainersConfig{
- DispatchPrivateKey: string(dispatchprivraw),
- StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
+ CrunchRunCommand: "crunch-run",
+ CrunchRunArgumentsList: []string{"--foo", "--extra='args'"},
+ DispatchPrivateKey: string(dispatchprivraw),
+ StaleLockTimeout: arvados.Duration(5 * time.Millisecond),
CloudVMs: arvados.CloudVMsConfig{
Driver: "test",
SyncInterval: arvados.Duration(10 * time.Millisecond),
@@ -161,6 +163,7 @@ func (s *DispatcherSuite) TestDispatchToStubDriver(c *check.C) {
stubvm.CrunchRunDetachDelay = time.Duration(rand.Int63n(int64(10 * time.Millisecond)))
stubvm.ExecuteContainer = executeContainer
stubvm.CrashRunningContainer = finishContainer
+ stubvm.ExtraCrunchRunArgs = "'--foo' '--extra='\\''args'\\'''"
switch n % 7 {
case 0:
stubvm.Broken = time.Now().Add(time.Duration(rand.Int63n(90)) * time.Millisecond)
diff --git a/lib/dispatchcloud/test/stub_driver.go b/lib/dispatchcloud/test/stub_driver.go
index 4d32cf221..1b31a71a2 100644
--- a/lib/dispatchcloud/test/stub_driver.go
+++ b/lib/dispatchcloud/test/stub_driver.go
@@ -193,6 +193,7 @@ type StubVM struct {
ArvMountDeadlockRate float64
ExecuteContainer func(arvados.Container) int
CrashRunningContainer func(arvados.Container)
+ ExtraCrunchRunArgs string // extra args expected after "crunch-run --detach --stdin-env "
sis *StubInstanceSet
id cloud.InstanceID
@@ -251,7 +252,7 @@ func (svm *StubVM) Exec(env map[string]string, command string, stdin io.Reader,
fmt.Fprint(stderr, "crunch-run: command not found\n")
return 1
}
- if strings.HasPrefix(command, "crunch-run --detach --stdin-env ") {
+ if strings.HasPrefix(command, "crunch-run --detach --stdin-env "+svm.ExtraCrunchRunArgs) {
var stdinKV map[string]string
err := json.Unmarshal(stdinData, &stdinKV)
if err != nil {
diff --git a/lib/dispatchcloud/worker/pool.go b/lib/dispatchcloud/worker/pool.go
index 6a74280ca..7289179fd 100644
--- a/lib/dispatchcloud/worker/pool.go
+++ b/lib/dispatchcloud/worker/pool.go
@@ -121,6 +121,8 @@ func NewPool(logger logrus.FieldLogger, arvClient *arvados.Client, reg *promethe
systemRootToken: cluster.SystemRootToken,
installPublicKey: installPublicKey,
tagKeyPrefix: cluster.Containers.CloudVMs.TagKeyPrefix,
+ runnerCmdDefault: cluster.Containers.CrunchRunCommand,
+ runnerArgs: cluster.Containers.CrunchRunArgumentsList,
stop: make(chan bool),
}
wp.registerMetrics(reg)
@@ -160,6 +162,8 @@ type Pool struct {
systemRootToken string
installPublicKey ssh.PublicKey
tagKeyPrefix string
+ runnerCmdDefault string // crunch-run command to use if not deploying a binary
+ runnerArgs []string // extra args passed to crunch-run
// private state
subscribers map[<-chan struct{}]chan<- struct{}
@@ -881,7 +885,7 @@ func (wp *Pool) loadRunnerData() error {
if wp.runnerData != nil {
return nil
} else if wp.runnerSource == "" {
- wp.runnerCmd = "crunch-run"
+ wp.runnerCmd = wp.runnerCmdDefault
wp.runnerData = []byte{}
return nil
}
diff --git a/lib/dispatchcloud/worker/pool_test.go b/lib/dispatchcloud/worker/pool_test.go
index a85f7383a..0f5c5ee19 100644
--- a/lib/dispatchcloud/worker/pool_test.go
+++ b/lib/dispatchcloud/worker/pool_test.go
@@ -72,8 +72,8 @@ func (suite *PoolSuite) TestResumeAfterRestart(c *check.C) {
newExecutor := func(cloud.Instance) Executor {
return &stubExecutor{
response: map[string]stubResp{
- "crunch-run --list": {},
- "true": {},
+ "crunch-run-custom --list": {},
+ "true": {},
},
}
}
@@ -87,6 +87,7 @@ func (suite *PoolSuite) TestResumeAfterRestart(c *check.C) {
SyncInterval: arvados.Duration(time.Millisecond * 10),
TagKeyPrefix: "testprefix:",
},
+ CrunchRunCommand: "crunch-run-custom",
},
InstanceTypes: arvados.InstanceTypeMap{
type1.Name: type1,
diff --git a/lib/dispatchcloud/worker/runner.go b/lib/dispatchcloud/worker/runner.go
index 0fd99aeee..63561874c 100644
--- a/lib/dispatchcloud/worker/runner.go
+++ b/lib/dispatchcloud/worker/runner.go
@@ -9,6 +9,7 @@ import (
"encoding/json"
"fmt"
"net"
+ "strings"
"syscall"
"time"
@@ -22,6 +23,7 @@ type remoteRunner struct {
executor Executor
envJSON json.RawMessage
runnerCmd string
+ runnerArgs []string
remoteUser string
timeoutTERM time.Duration
timeoutSignal time.Duration
@@ -64,6 +66,7 @@ func newRemoteRunner(uuid string, wkr *worker) *remoteRunner {
executor: wkr.executor,
envJSON: envJSON,
runnerCmd: wkr.wp.runnerCmd,
+ runnerArgs: wkr.wp.runnerArgs,
remoteUser: wkr.instance.RemoteUser(),
timeoutTERM: wkr.wp.timeoutTERM,
timeoutSignal: wkr.wp.timeoutSignal,
@@ -81,7 +84,11 @@ func newRemoteRunner(uuid string, wkr *worker) *remoteRunner {
// assume the remote process _might_ have started, at least until it
// probes the worker and finds otherwise.
func (rr *remoteRunner) Start() {
- cmd := rr.runnerCmd + " --detach --stdin-env '" + rr.uuid + "'"
+ cmd := rr.runnerCmd + " --detach --stdin-env"
+ for _, arg := range rr.runnerArgs {
+ cmd += " '" + strings.Replace(arg, "'", "'\\''", -1) + "'"
+ }
+ cmd += " '" + rr.uuid + "'"
if rr.remoteUser != "root" {
cmd = "sudo " + cmd
}
diff --git a/lib/dispatchcloud/worker/worker_test.go b/lib/dispatchcloud/worker/worker_test.go
index cfb7a1bfb..4134788b2 100644
--- a/lib/dispatchcloud/worker/worker_test.go
+++ b/lib/dispatchcloud/worker/worker_test.go
@@ -236,6 +236,8 @@ func (suite *WorkerSuite) TestProbeAndUpdate(c *check.C) {
timeoutBooting: bootTimeout,
timeoutProbe: probeTimeout,
exited: map[string]time.Time{},
+ runnerCmdDefault: "crunch-run",
+ runnerArgs: []string{"--args=not used with --list"},
runnerCmd: "crunch-run",
runnerData: trial.deployRunner,
runnerMD5: md5.Sum(trial.deployRunner),
commit 6a9bbdcb1ebfd8a67b7a0154c2576c79ace5088b
Author: Tom Clegg <tom at curii.com>
Date: Sun Feb 14 23:04:13 2021 -0500
Fix panic when OIDC provider returns no name claim.
No issue #
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/login_oidc.go b/lib/controller/localdb/login_oidc.go
index 2b67a9504..74b8929a2 100644
--- a/lib/controller/localdb/login_oidc.go
+++ b/lib/controller/localdb/login_oidc.go
@@ -176,7 +176,7 @@ func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2.
if names := strings.Fields(strings.TrimSpace(name)); len(names) > 1 {
ret.FirstName = strings.Join(names[0:len(names)-1], " ")
ret.LastName = names[len(names)-1]
- } else {
+ } else if len(names) > 0 {
ret.FirstName = names[0]
}
ret.Email, _ = claims[ctrl.EmailClaim].(string)
commit 3c924b22e3705a35adbe12a82bc6ffdfe1c55835
Author: Ward Vandewege <ward at curii.com>
Date: Thu Feb 11 21:53:52 2021 -0500
Documentation: fix typo.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/api/methods/pipeline_instances.html.textile.liquid b/doc/api/methods/pipeline_instances.html.textile.liquid
index 56c071ef9..55baee9b5 100644
--- a/doc/api/methods/pipeline_instances.html.textile.liquid
+++ b/doc/api/methods/pipeline_instances.html.textile.liquid
@@ -21,7 +21,7 @@ Example UUID: @zzzzz-d1hrv-0123456789abcde@
h2. Resource
-Deprecated. A pipeline instance is a collection of jobs managed by @aravdos-run-pipeline-instance at .
+Deprecated. A pipeline instance is a collection of jobs managed by @arvados-run-pipeline-instance at .
Each PipelineInstance has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html:
commit 1919d3b07380831d8dc485bbf22857c5caf1da0f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Feb 10 16:01:43 2021 -0500
Put the missing images in the right place refs #17165
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/images/AddNew.png b/doc/user/cwl/images/AddNew.png
similarity index 100%
rename from doc/images/AddNew.png
rename to doc/user/cwl/images/AddNew.png
diff --git a/doc/images/Explorer.png b/doc/user/cwl/images/Explorer.png
similarity index 100%
rename from doc/images/Explorer.png
rename to doc/user/cwl/images/Explorer.png
diff --git a/doc/images/Extensions.png b/doc/user/cwl/images/Extensions.png
similarity index 100%
rename from doc/images/Extensions.png
rename to doc/user/cwl/images/Extensions.png
diff --git a/doc/images/RemoteExplorer.png b/doc/user/cwl/images/RemoteExplorer.png
similarity index 100%
rename from doc/images/RemoteExplorer.png
rename to doc/user/cwl/images/RemoteExplorer.png
diff --git a/doc/images/SSHTargets.png b/doc/user/cwl/images/SSHTargets.png
similarity index 100%
rename from doc/images/SSHTargets.png
rename to doc/user/cwl/images/SSHTargets.png
commit 87df20ebc9a695df24e92c0c6c44ee7d801733b9
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Feb 10 15:46:36 2021 -0500
Add missing images refs #17165
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/images/AddNew.png b/doc/images/AddNew.png
new file mode 100644
index 000000000..b2a4be4b1
Binary files /dev/null and b/doc/images/AddNew.png differ
diff --git a/doc/images/Explorer.png b/doc/images/Explorer.png
new file mode 100644
index 000000000..f91128b13
Binary files /dev/null and b/doc/images/Explorer.png differ
diff --git a/doc/images/Extensions.png b/doc/images/Extensions.png
new file mode 100644
index 000000000..7f797a1d0
Binary files /dev/null and b/doc/images/Extensions.png differ
diff --git a/doc/images/RemoteExplorer.png b/doc/images/RemoteExplorer.png
new file mode 100644
index 000000000..81672fdd5
Binary files /dev/null and b/doc/images/RemoteExplorer.png differ
diff --git a/doc/images/SSHTargets.png b/doc/images/SSHTargets.png
new file mode 100644
index 000000000..9e1356b6d
Binary files /dev/null and b/doc/images/SSHTargets.png differ
commit c1f0890ca1539f6b71946491827dd35d7d214354
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Feb 10 15:16:14 2021 -0500
17165: Sync up again
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/cwl/arvados-vscode-training.html.md.liquid b/doc/user/cwl/arvados-vscode-training.html.md.liquid
index 4d4dab078..80477e9d7 100644
--- a/doc/user/cwl/arvados-vscode-training.html.md.liquid
+++ b/doc/user/cwl/arvados-vscode-training.html.md.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
Imported from https://github.com/arvados/arvados-vscode-cwl-training
-git hash: 1f74405842f221c40b852cf142a393c25eefd15a
+git hash: e21c963c05fb35bd8be15e24f72045e9c1cec668
{% endcomment %}
@@ -72,20 +72,23 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Right click the newly added ssh target in the list and select “connect to host in current window`
1. If it asks `Select platform of the remote host` select `Linux`.
1. Vscode: On the left sidebar, go back to `Extensions` 
- 1. Search for "benten" and install `CWL (Rabix/Benten)`
- 1. You should see a message `Extension is enabled on 'SSH: ...' and disabled locally.`
+ 1. Search for "benten", then look for `CWL (Rabix/Benten)` and click `Install`
+ 1. On the information page for `CWL (Rabix/Benten)`
+ 1. If you see a warning `Install the extension on 'SSH: ...' to enable` then click the button `Install in SSH: ...`
+ 1. You should now see a message `Extension is enabled on 'SSH: ...' and disabled locally.`
1. Vscode: On the left sidebar, choose `Explorer` 
1. Select `Clone Repository` and enter [https://github.com/arvados/arvados-vscode-cwl-training](https://github.com/arvados/arvados-vscode-cwl-training), then click `Open`
1. If asked `Would you like to open the cloned repository?` choose `Open`
-1. Go to Arvados Workbench
- 1. Workbench: In the user menu, select `Current token`
- 1. Vscode: Click on the `Terminal` menu
+1. Vscode: Click on the `Terminal` menu
1. Vscode: Click `Run Task…`
1. Vscode: Select `Configure Arvados`
- 1. Workbench: Copy the string following `ARVADOS_API_HOST=`
+ 1. Go to Arvados Workbench
+ 1. Workbench: In the user menu, select `Current token`
+ 1. Workbench: Copy the text following `ARVADOS_API_HOST=` to the end of the line
1. Vscode: Paste the string at the `Value for ARVADOS_API_HOST` prompt
- 1. Workbench: Copy the string following `ARVADOS_API_TOKEN=`
+ 1. Workbench: Copy the text following `ARVADOS_API_TOKEN=` to the end of the line
1. Vscode: Paste the string at the `Value for ARVADOS_API_TOKEN` prompt
+ 1. This will create files called `API_HOST` and `API_TOKEN`
## 3. Register & run a workflow
@@ -93,6 +96,7 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Click on the `Terminal` menu
1. Click `Run Task…`
1. Select `Register or update CWL workflow on Arvados Workbench`
+ 1. This will create a file called `WORKFLOW_UUID`
1. Workbench: Go to `+NEW` and select `New project`
1. Enter a name for the project like "Lesson 1"
1. You should arrive at the panel for the new project
commit f7ae654bf33819bfae67d4be200e692394a507c5
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Feb 9 13:59:16 2021 -0500
17165: Update vscode training doc
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/cwl/arvados-vscode-training.html.md.liquid b/doc/user/cwl/arvados-vscode-training.html.md.liquid
index 8ade1c755..4d4dab078 100644
--- a/doc/user/cwl/arvados-vscode-training.html.md.liquid
+++ b/doc/user/cwl/arvados-vscode-training.html.md.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
Imported from https://github.com/arvados/arvados-vscode-cwl-training
-git hash: 0a544f917de8bc2538e1582a82af205c1000adf5
+git hash: 1f74405842f221c40b852cf142a393c25eefd15a
{% endcomment %}
@@ -25,18 +25,20 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
## 1. SSH Setup
-1. (Windows only) Install git for windows [https://git-scm.com/download/win](https://git-scm.com/download/win)
+1. (Windows only) Install Git for Windows [https://git-scm.com/download/win](https://git-scm.com/download/win)
1. Choose "64-bit Git for Windows Setup". It does not require admin privileges to install.
1. Hit "Next" a bunch of times to accept the defaults
- 1. The most important things is that "install git bash" and "install OpenSSH" are enabled.
+ 1. The most important things is that "install git bash" and "install OpenSSH" are enabled (this is the default).
1. At the end of the installation, you can launch tick a box to git bash directly.
1. Open "Git Bash" (installed in the "Git" folder of the start menu)
-1. From an open bash shell
+1. (All operating systems) Starting from bash shell (on MacOS or Linux you will open "Terminal")
1. Shell: Run `ssh-keygen`
1. Hit enter to save to a default location
1. You can choose to protect the key with a password, or just hit enter for no password.
- 1. Shell: Look for a message like `Your public key has been saved in /c/Users/MyUsername/.ssh/id_rsa.pub`
- 1. Shell: Run `cat /c/Users/MyUsername/.ssh/id_rsa.pub`
+ 1. Shell: Look for a message like `Your public key has been saved
+ in /c/Users/MyUsername/.ssh/id_rsa.pub` (Windows git bash
+ example, on MacOS or Linux this will probably start with `/Users` or `/home`)
+ 1. Shell: Run `cat /c/Users/MyUsername/.ssh/id_rsa.pub`
1. Shell: Use the pointer to highlight and copy the lines starting
with `ssh-rsa …` up to the next blank line. Right click and
select "Copy"
commit be34fa5f0ae5c616798fdc9accae9677c7e53508
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Feb 3 15:40:27 2021 -0500
17165: Update training after going back over it step-by-step
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/cwl/arvados-vscode-training.html.md.liquid b/doc/user/cwl/arvados-vscode-training.html.md.liquid
index 25c53744a..8ade1c755 100644
--- a/doc/user/cwl/arvados-vscode-training.html.md.liquid
+++ b/doc/user/cwl/arvados-vscode-training.html.md.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
Imported from https://github.com/arvados/arvados-vscode-cwl-training
-git hash: 4ea2b3e8cbfdf977eaba6bedc36b55aa172a69d8
+git hash: 0a544f917de8bc2538e1582a82af205c1000adf5
{% endcomment %}
@@ -17,20 +17,31 @@ These lessons give step by step instructions for using Visual Studio
Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Set up SSH
-1. Install vscode and necessary extensions
-1. Use vscode to connect to an Arvados shell node for development
-1. Create an simple command line tool, run it, and view the log
-1. Upload input, create a command line tool to process it, and view the output
+1. Install vscode and necessary extensions, then use vscode to connect to an Arvados shell node for development
+1. Register a workflow, run it on workbench, and view the log
+1. Upload input, run a workflow on it, and view the output
+1. Register a workflow with default inputs
+1. Run a workflow without registering it
## 1. SSH Setup
1. (Windows only) Install git for windows [https://git-scm.com/download/win](https://git-scm.com/download/win)
-1. Open bash shell (installed with git for Windows)
+ 1. Choose "64-bit Git for Windows Setup". It does not require admin privileges to install.
+ 1. Hit "Next" a bunch of times to accept the defaults
+ 1. The most important things is that "install git bash" and "install OpenSSH" are enabled.
+ 1. At the end of the installation, you can launch tick a box to git bash directly.
+ 1. Open "Git Bash" (installed in the "Git" folder of the start menu)
+1. From an open bash shell
1. Shell: Run `ssh-keygen`
- 1. Shell: Look for `Your public key has been saved in /c/Users/MyUsername/.ssh/id_rsa.pub`
+ 1. Hit enter to save to a default location
+ 1. You can choose to protect the key with a password, or just hit enter for no password.
+ 1. Shell: Look for a message like `Your public key has been saved in /c/Users/MyUsername/.ssh/id_rsa.pub`
1. Shell: Run `cat /c/Users/MyUsername/.ssh/id_rsa.pub`
- 1. Shell: Highlight and copy the lines starting with `ssh-rsa …`
-1. Open Arvados workbench
+ 1. Shell: Use the pointer to highlight and copy the lines starting
+ with `ssh-rsa …` up to the next blank line. Right click and
+ select "Copy"
+1. Open Arvados workbench 2. If necessary, go to the user menu and
+ select "Go to Workbench 2"
1. Workbench: Go to `SSH keys` in the user menu
1. Workbench:Click `+Add new ssh key`
1. Workbench: Paste the key into `Public key` and enter something for `name`
@@ -38,29 +49,32 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Workbench: Highlight and copy the value in in the `Command line` column.
1. At the git bash command line
1. Shell: paste the `ssh shell…` command line you got from workbench.
- 1. Shell: type "yes" if it asks `do you want to continue connecting`
- 1. Shell: You should now be logged into the Arvados shell node. Note:
- it can take up to two minutes for the SSH key to be copied to
- the shell node. If it doesn't work the first time, wait 60
+ 1. Shell: type "yes" if it asks `Are you sure you want to continue connecting`.
+ 1. Note: it can take up to two minutes for the SSH key to be copied to
+ the shell node. If you get "Permission denied" the first time, wait 60
seconds and try again.
+ 1. Shell: You should now be logged into the Arvados shell node.
1. Shell: Log out by typing `exit`
## 2. VSCode setup
1. Install [Visual Studio Code](https://code.visualstudio.com/) and start it up
-1. Vscode: go to `Extensions`
- 1. search for `remote development` and install the Remote Development extension pack from Microsoft
-1. Vscode: On the left side bar, choose `Remote explorer`
- 1. In the drop down, choose `SSH targets`
- 1. Click `Add new`
+1. Vscode: On the left sidebar, select `Extensions` 
+ 1. In `Search Extensions in Marketplace` enter "remote development".
+ 1. Choose and install the "Remote Development" extension pack from Microsoft
+1. Vscode: On the left sidebar, choose `Remote Explorer` 
+ 1. At the top of the Remote Explorer panel choose `SSH targets` 
+ 1. Click `Add New` 
1. Enter the `ssh shell…` command line you used in the previous section, step 1.4.1
- 1. Right click the ssh target in the list and select “connect to host in current window`
-1. Vscode: go to `Extensions`
- 1. Search for `benten` and install `CWL (Rabix/Benten)`
- 1. Choose `Install extension on the remote ssh host`
-1. Vscode: On the left side bar, choose `Explorer`
- 1. Select `Clone Repository` and then enter [https://github.com/arvados/arvados-vscode-cwl-training](https://github.com/arvados/arvados-vscode-cwl-training)
- 1. Choose `Open`
+ 1. If it asks you `Select SSH configuration file to update` choose the first one in the list.
+ 1. Right click the newly added ssh target in the list and select “connect to host in current window`
+ 1. If it asks `Select platform of the remote host` select `Linux`.
+1. Vscode: On the left sidebar, go back to `Extensions` 
+ 1. Search for "benten" and install `CWL (Rabix/Benten)`
+ 1. You should see a message `Extension is enabled on 'SSH: ...' and disabled locally.`
+1. Vscode: On the left sidebar, choose `Explorer` 
+ 1. Select `Clone Repository` and enter [https://github.com/arvados/arvados-vscode-cwl-training](https://github.com/arvados/arvados-vscode-cwl-training), then click `Open`
+ 1. If asked `Would you like to open the cloned repository?` choose `Open`
1. Go to Arvados Workbench
1. Workbench: In the user menu, select `Current token`
1. Vscode: Click on the `Terminal` menu
@@ -71,7 +85,7 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Workbench: Copy the string following `ARVADOS_API_TOKEN=`
1. Vscode: Paste the string at the `Value for ARVADOS_API_TOKEN` prompt
-# 3. Register & run a workflow
+## 3. Register & run a workflow
1. Vscode: Click on the `lesson1/main.cwl` file
1. Click on the `Terminal` menu
@@ -88,9 +102,13 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Click `Run process`
1. This should take you to a panel showing the workflow run status
1. Workbench: workflow run status panel
- 1. Click on the three vertical dots in the top-right corner
+ 1. Wait for the badge in the upper right to say `Completed`
+ 1. In the lower panel, double click on the `echo` workflow step
+ 1. This will take you to the status panel for the `echo` step
+ 1. Click on the three vertical dots in the top-right corner next to `Completed`
1. Choose `Log`
- 1. Under `event type` choose `stdout`
+ 1. This will take you to the log viewer panel
+ 1. Under `Event Type` choose `stdout`
1. You should see your message
## 4. Working with input and output files
@@ -100,14 +118,15 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Click `Run Task…`
1. Select `Register or update CWL workflow on Arvados Workbench`
1. Go to your desktop
- 1. Using a text editor such as notepad, create a file "message.txt"
- 1. Enter a message like "Hello world" and save
+ 1. Right click on the desktop, select `New > Text Document`
+ 1. Name the file `message`
+ 1. Enter a message like "Hello earth" and save
1. Workbench: Go to `+NEW` and select `New project`
1. Enter a name for the project like "Lesson 2"
1. You should arrive at the panel for the new project
1. Arvados workbench: With `Lesson 2` project selected
1. Click on +NEW and select `New collection`
- 1. Call the collection "my message"
+ 1. For Collection Name enter "my message"
1. Drag and drop `message.txt` into the browser
1. Click `Create a collection`
1. The file should be uploaded and then you will be on the collection page
@@ -125,13 +144,17 @@ Code (abbreviated "vscode") to develop CWL workflows on Arvados.
1. Click on the dot menu
1. Choose `Outputs`
1. Right click on `reverse.txt`
- 1. Click on `Download`
- 1. Open the downloaded file. It should have your results.
+ 1. Click on `Open in new tab`
+ 1. The results should be visible in a new browser tab.
## 5. Register a workflow with default inputs
The default value for the `message` parameter will taken from the `lesson3/defaults.yaml` file
+1. Vscode: Click on the `lesson3/main.cwl` file
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Register or update CWL workflow on Arvados Workbench`
1. Workbench: Go to `+NEW` and select `New project`
1. Enter a name for the project like "Lesson 3"
1. You should arrive at the panel for the new project
@@ -142,6 +165,8 @@ The default value for the `message` parameter will taken from the `lesson3/defau
1. The `#main/message` parameter will be pre-filled with your default value. You can choose to change it or use the default.
1. Click `Run process`
1. This should take you to the status page for this workflow
+ 1. The greeting will appear in the `Log` of the `echo` task, which
+ can be found the same way as described earlier in section 3.
## 6. Run a workflow without registering it
@@ -164,7 +189,21 @@ The `message` parameter will be taken from the file `lesson4/main-input.yaml`.
1. Vscode: In the bottom panel select the `Terminal` tab
1. In the upper right corner of the Terminal tab select `Task - Run CWL Workflow` from the drop-down
1. Look for logging text like `submitted container_request zzzzz-xvhdp-0123456789abcde`
- 1. Highlight and copy the workflow identifier (this the string containing -xvhdp- in the middle)
+ 1. Highlight and copy the workflow identifier (this the string containing `-xvhdp-` in the middle)
1. The results of this run will appear in the terminal when the run completes.
1. Workbench: Paste the workflow identifier into the search box
1. This will take you to the status page for this workflow
+
+
+## Notes
+
+If you need to change something about the environment of the user on
+the remote host (for example, the user has been added to a new unix
+group) you need to restart the vscode server that runs on the remote
+host. Do this in vscode:
+
+ctrl+shift+p: `Remote-SSH: Kill VS Code Server on Host`
+
+This is because the vscode server remains running on the remote host
+even after you disconnect, so exiting/restarting vscode on the desktop
+has no effect.
commit d26fce5afa74c38a094409867c18f6973512e439
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Feb 1 20:09:36 2021 -0500
17165: Adjust some titles
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/Rakefile b/doc/Rakefile
index baa3d9d30..ee87062f7 100644
--- a/doc/Rakefile
+++ b/doc/Rakefile
@@ -168,7 +168,7 @@ task :import_vscode_training do
---
layout: default
navsection: userguide
-title: "Developing Workflows with VSCode"
+title: "Developing CWL Workflows with VSCode"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
diff --git a/doc/_config.yml b/doc/_config.yml
index de00b070d..f519d229e 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -49,7 +49,7 @@ navbar:
- user/topics/collection-versioning.html.textile.liquid
- user/topics/storage-classes.html.textile.liquid
- Data Analysis with Workflows:
- - user/user/cwl/arvados-vscode-training.html.md.liquid
+ - user/cwl/arvados-vscode-training.html.md.liquid
- user/cwl/cwl-runner.html.textile.liquid
- user/cwl/cwl-run-options.html.textile.liquid
- user/tutorials/writing-cwl-workflow.html.textile.liquid
diff --git a/doc/user/cwl/arvados-vscode-training.html.md.liquid b/doc/user/cwl/arvados-vscode-training.html.md.liquid
index 2557fbcf8..25c53744a 100644
--- a/doc/user/cwl/arvados-vscode-training.html.md.liquid
+++ b/doc/user/cwl/arvados-vscode-training.html.md.liquid
@@ -1,7 +1,7 @@
---
layout: default
navsection: userguide
-title: "Developing Workflows with VSCode"
+title: "Developing CWL Workflows with VSCode"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
diff --git a/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid b/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
index 0166b8b52..9db3f8085 100644
--- a/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
+++ b/doc/user/tutorials/writing-cwl-workflow.html.textile.liquid
@@ -1,7 +1,7 @@
---
layout: default
navsection: userguide
-title: "Developing workflows with CWL"
+title: "CWL Resources"
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
commit 97502961518e9ac24dd9e6278dce6097d47b2bd6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Feb 1 16:18:25 2021 -0500
Update from upstream
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/cwl/arvados-vscode-training.html.md.liquid b/doc/user/cwl/arvados-vscode-training.html.md.liquid
index 883e0736a..2557fbcf8 100644
--- a/doc/user/cwl/arvados-vscode-training.html.md.liquid
+++ b/doc/user/cwl/arvados-vscode-training.html.md.liquid
@@ -9,156 +9,161 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
Imported from https://github.com/arvados/arvados-vscode-cwl-training
-git hash: 05cae0b323f4491a8f211fe1bdf300e4c761ce93
+git hash: 4ea2b3e8cbfdf977eaba6bedc36b55aa172a69d8
{% endcomment %}
These lessons give step by step instructions for using Visual Studio
-Code to develop CWL workflows on Arvados.
+Code (abbreviated "vscode") to develop CWL workflows on Arvados.
-1. Install & set up ssh client on Windows
-1. Install vscode and relevant extensions
-1. Use vscode to connect to an arvados shell node for development
+1. Set up SSH
+1. Install vscode and necessary extensions
+1. Use vscode to connect to an Arvados shell node for development
1. Create an simple command line tool, run it, and view the log
1. Upload input, create a command line tool to process it, and view the output
-# Windows SSH Setup
+## 1. SSH Setup
-Linux and MacOS users can skip this step.
-
-1. Install git for windows https://git-scm.com/download/win
-1. Open git bash
-1. Run “ssh-keygen”
-1. Look for “Your public key has been saved in /c/Users/MyUsername/.ssh/id_rsa.pub”
-1. Run “cat /c/Users/MyUsername/.ssh/id_rsa.pub”
-1. Copy the lines starting with “ssh-rsa …”
+1. (Windows only) Install git for windows [https://git-scm.com/download/win](https://git-scm.com/download/win)
+1. Open bash shell (installed with git for Windows)
+ 1. Shell: Run `ssh-keygen`
+ 1. Shell: Look for `Your public key has been saved in /c/Users/MyUsername/.ssh/id_rsa.pub`
+ 1. Shell: Run `cat /c/Users/MyUsername/.ssh/id_rsa.pub`
+ 1. Shell: Highlight and copy the lines starting with `ssh-rsa …`
1. Open Arvados workbench
-1. Go to “SSH keys” in the user menu
-1. Click + Add new ssh key
-1. Paste the key into “Public key” and enter something for “name”
-1. At the git bash command line, run “ssh shell…” and say “yes” if it asks “do you want to continue connecting”
-1. You should be logged into the Arvados shell node.
-1. Log out by typing “exit”
-
-# VSCode setup
-
-1. Install vscode https://code.visualstudio.com/ and start it up
-1. Vscode: go to “Extensions”
- 1. search for “remote development” and install the Remote Development extension pack from Microsoft
-1. Vscode: On the left side bar, choose “Remote explorer”
- 1. In the drop down, choose “SSH targets”
- 1. Click “Add new”
- 1. Enter the “ssh shell…” command line you used in step 1(j)
- 1. Right click the ssh target in the list and select “connect to host in current window”
-1. Vscode: go to “Extensions”
- 1. Search for “benten” and install “CWL (Rabix/Benten)”
- 1. Choose “Install extension on the remote ssh host”
-1. Vscode: On the left side bar, choose “Explorer”
- 1. Select “Clone Repository” and then enter “https://github.com/arvados/arvados-vscode-cwl-training”
- 1. Choose “Open”
+ 1. Workbench: Go to `SSH keys` in the user menu
+ 1. Workbench:Click `+Add new ssh key`
+ 1. Workbench: Paste the key into `Public key` and enter something for `name`
+ 1. Workbench: Go to `Virtual Machines` in the user menu
+ 1. Workbench: Highlight and copy the value in in the `Command line` column.
+1. At the git bash command line
+ 1. Shell: paste the `ssh shell…` command line you got from workbench.
+ 1. Shell: type "yes" if it asks `do you want to continue connecting`
+ 1. Shell: You should now be logged into the Arvados shell node. Note:
+ it can take up to two minutes for the SSH key to be copied to
+ the shell node. If it doesn't work the first time, wait 60
+ seconds and try again.
+ 1. Shell: Log out by typing `exit`
+
+## 2. VSCode setup
+
+1. Install [Visual Studio Code](https://code.visualstudio.com/) and start it up
+1. Vscode: go to `Extensions`
+ 1. search for `remote development` and install the Remote Development extension pack from Microsoft
+1. Vscode: On the left side bar, choose `Remote explorer`
+ 1. In the drop down, choose `SSH targets`
+ 1. Click `Add new`
+ 1. Enter the `ssh shell…` command line you used in the previous section, step 1.4.1
+ 1. Right click the ssh target in the list and select “connect to host in current window`
+1. Vscode: go to `Extensions`
+ 1. Search for `benten` and install `CWL (Rabix/Benten)`
+ 1. Choose `Install extension on the remote ssh host`
+1. Vscode: On the left side bar, choose `Explorer`
+ 1. Select `Clone Repository` and then enter [https://github.com/arvados/arvados-vscode-cwl-training](https://github.com/arvados/arvados-vscode-cwl-training)
+ 1. Choose `Open`
1. Go to Arvados Workbench
- 1. Workbench: In the user menu, select “Current token”
- 1. Vscode: Click on the “Terminal” menu
- 1. Vscode: Click “Run Task…”
- 1. Vscode: Select “Configure Arvados”
- 1. Workbench: Copy the string following “ARVADOS_API_HOST=”
- 1. Vscode: Paste the string at the “Value for ARVADOS_API_HOST” prompt
- 1. Workbench: Copy the string following “ARVADOS_API_TOKEN=”
- 1. Vscode: Paste the string at the “Value for ARVADOS_API_TOKEN” prompt
-
-# Register & run a workflow
-
-1. Vscode: Click on the “lesson1/main.cwl” file
- 1. Click on the “Terminal” menu
- 1. Click “Run Task…”
- 1. Select “Register or update CWL workflow on Arvados Workbench”
-1. Workbench: Go to “+NEW” and select “New project”
- 1. Enter a name for the project like “Lesson 1”
+ 1. Workbench: In the user menu, select `Current token`
+ 1. Vscode: Click on the `Terminal` menu
+ 1. Vscode: Click `Run Task…`
+ 1. Vscode: Select `Configure Arvados`
+ 1. Workbench: Copy the string following `ARVADOS_API_HOST=`
+ 1. Vscode: Paste the string at the `Value for ARVADOS_API_HOST` prompt
+ 1. Workbench: Copy the string following `ARVADOS_API_TOKEN=`
+ 1. Vscode: Paste the string at the `Value for ARVADOS_API_TOKEN` prompt
+
+# 3. Register & run a workflow
+
+1. Vscode: Click on the `lesson1/main.cwl` file
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Register or update CWL workflow on Arvados Workbench`
+1. Workbench: Go to `+NEW` and select `New project`
+ 1. Enter a name for the project like "Lesson 1"
1. You should arrive at the panel for the new project
-1. Workbench: With “Lesson 1” selected
- 1. Click on “+NEW” and select “Run a process”
- 1. Select “CWL training lesson 1” from the list and click “Next”
- 1. Enter a name for this run like “First training run”
- 1. Enter a message (under “#main/message”) like “Hello world”
- 1. Click “Run process”
+1. Workbench: With `Lesson 1` selected
+ 1. Click on `+NEW` and select `Run a process`
+ 1. Select `CWL training lesson 1` from the list and click `Next`
+ 1. Enter a name for this run like `First training run`
+ 1. Enter a message (under `#main/message`) like "Hello world"
+ 1. Click `Run process`
1. This should take you to a panel showing the workflow run status
1. Workbench: workflow run status panel
1. Click on the three vertical dots in the top-right corner
- 1. Choose “Log”
- 1. Under “event type” choose “stdout”
+ 1. Choose `Log`
+ 1. Under `event type` choose `stdout`
1. You should see your message
-# Working with input and output files
+## 4. Working with input and output files
-1. Vscode: Click on the “lesson2/main.cwl” file
- 1. Click on the “Terminal” menu
- 1. Click “Run Task…”
- 1. Select “Register or update CWL workflow on Arvados Workbench”
+1. Vscode: Click on the `lesson2/main.cwl` file
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Register or update CWL workflow on Arvados Workbench`
1. Go to your desktop
- 1. Using a text editor such as notepad, create a file “message.txt”
- 1. Enter a message like “Hello world” and save
-1. Workbench: Go to “+NEW” and select “New project”
- 1. Enter a name for the project like “Lesson 2”
+ 1. Using a text editor such as notepad, create a file "message.txt"
+ 1. Enter a message like "Hello world" and save
+1. Workbench: Go to `+NEW` and select `New project`
+ 1. Enter a name for the project like "Lesson 2"
1. You should arrive at the panel for the new project
-1. Arvados workbench: With “Lesson 2” project selected
- 1. Click on +NEW and select “New collection”
- 1. Call the collection “my message”
- 1. Drag and drop “message.txt” into the browser
- 1. Click “Create a collection”
+1. Arvados workbench: With `Lesson 2` project selected
+ 1. Click on +NEW and select `New collection`
+ 1. Call the collection "my message"
+ 1. Drag and drop `message.txt` into the browser
+ 1. Click `Create a collection`
1. The file should be uploaded and then you will be on the collection page
-1. Workbench: Select the “Lesson 2” project
- 1. Click on “+NEW” and select “Run a process”
- 1. Select “CWL training lesson 2” from the list and click “Next”
- 1. Enter a name for this run like “Second training run”
- 1. Click on “#main/message”
+1. Workbench: Select the `Lesson 2` project
+ 1. Click on `+NEW` and select `Run a process`
+ 1. Select `CWL training lesson 2` from the list and click `Next`
+ 1. Enter a name for this run like "Second training run"
+ 1. Click on `#main/message`
1. A selection dialog box will appear
- 1. Navigate to the collection you created in step (13) and choose “message.txt”
- 1. Click “Run process”
+ 1. Navigate to the collection you created in step (4.4.4) and choose `message.txt`
+ 1. Click `Run process`
1. This should take you to a panel showing the workflow run status
1. Workbench: workflow run status panel
1. Wait for the process to complete
1. Click on the dot menu
- 1. Choose “Outputs”
- 1. Right click on “reverse.txt”
- 1. Click on “Download”
+ 1. Choose `Outputs`
+ 1. Right click on `reverse.txt`
+ 1. Click on `Download`
1. Open the downloaded file. It should have your results.
-# Register a workflow with default inputs
+## 5. Register a workflow with default inputs
-The default value for the “message” parameter will taken from the “lesson3/defaults.yaml” file
+The default value for the `message` parameter will taken from the `lesson3/defaults.yaml` file
-1. Workbench: Go to “+NEW” and select “New project”
- 1. Enter a name for the project like “Lesson 3”
+1. Workbench: Go to `+NEW` and select `New project`
+ 1. Enter a name for the project like "Lesson 3"
1. You should arrive at the panel for the new project
-1. Workbench: With “Lesson 3” selected
- 1. Click on “+NEW” and select “Run a process”
- 1. Select “CWL training lesson 3” from the list and click “Next”
- 1. Enter a name for this run like “Third training run”
- 1. The “#main/message” parameter will be pre-filled with your default value. You can choose to change it or use the default.
- 1. Click “Run process”
+1. Workbench: With `Lesson 3` selected
+ 1. Click on `+NEW` and select `Run a process`
+ 1. Select `CWL training lesson 3` from the list and click `Next`
+ 1. Enter a name for this run like "Third training run"
+ 1. The `#main/message` parameter will be pre-filled with your default value. You can choose to change it or use the default.
+ 1. Click `Run process`
1. This should take you to the status page for this workflow
-# Run a workflow without registering it
+## 6. Run a workflow without registering it
-The “message” parameter will be taken from the file “lesson4/main-input.yaml”. This is useful during development.
+The `message` parameter will be taken from the file `lesson4/main-input.yaml`. This is useful during development.
-1. Workbench: Go to “+NEW” and select “New project”
- 1. Enter a name for the project like “Lesson 4”
+1. Workbench: Go to `+NEW` and select `New project`
+ 1. Enter a name for the project like "Lesson 4"
1. You should arrive at the panel for the new project
- 1. Click on “Additional info” in the upper right to expand the “info” panel
- 1. Under “Project UUID” click the “Copy to clipboard” button
-1. Vscode: Select the file “lesson4/main.cwl”
- 1. Click on the “Terminal” menu
- 1. Click “Run Task…”
- 1. Select “Set Arvados project UUID”
+ 1. Click on `Additional info` in the upper right to expand the `info` panel
+ 1. Under `Project UUID` click the `Copy to clipboard` button
+1. Vscode: Select the file `lesson4/main.cwl`
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Set Arvados project UUID`
1. Paste the project UUID from workbench at the prompt
-1. Vscode: Select the file “lesson4/main.cwl”
- 1. Click on the “Terminal” menu
- 1. Click “Run Task…”
- 1. Select “Run CWL workflow on Arvados”
-1. Vscode: In the bottom panel select the “Terminal” tab
- 1. In the upper right corner of the Terminal tab select “Task - Run CWL Workflow” from the drop-down
- 1. Look for logging text like “submitted container_request zzzzz-xvhdp-0123456789abcde”
+1. Vscode: Select the file `lesson4/main.cwl`
+ 1. Click on the `Terminal` menu
+ 1. Click `Run Task…`
+ 1. Select `Run CWL workflow on Arvados`
+1. Vscode: In the bottom panel select the `Terminal` tab
+ 1. In the upper right corner of the Terminal tab select `Task - Run CWL Workflow` from the drop-down
+ 1. Look for logging text like `submitted container_request zzzzz-xvhdp-0123456789abcde`
1. Highlight and copy the workflow identifier (this the string containing -xvhdp- in the middle)
1. The results of this run will appear in the terminal when the run completes.
1. Workbench: Paste the workflow identifier into the search box
commit e2d819b5216b056137fcc7863f788440a37a452d
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Feb 1 15:47:05 2021 -0500
17165: Add rake task to copy from arvados-vscode-cwl-training
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/Rakefile b/doc/Rakefile
index 3717f9f5f..baa3d9d30 100644
--- a/doc/Rakefile
+++ b/doc/Rakefile
@@ -157,6 +157,37 @@ task :linkchecker => [ :generate ] do
end
end
+task :import_vscode_training do
+ Dir.chdir("user") do
+ rm_rf "arvados-vscode-cwl-training"
+ `git clone https://github.com/arvados/arvados-vscode-cwl-training`
+ githash = `git --git-dir arvados-vscode-cwl-training/.git log -n1 --format=%H HEAD`
+ File.open("cwl/arvados-vscode-training.html.md.liquid", "w") do |fn|
+ File.open("arvados-vscode-cwl-training/README.md", "r") do |rd|
+ fn.write(<<-EOF
+---
+layout: default
+navsection: userguide
+title: "Developing Workflows with VSCode"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+
+Imported from https://github.com/arvados/arvados-vscode-cwl-training
+git hash: #{githash}
+{% endcomment %}
+
+EOF
+ )
+ fn.write(rd.read())
+ end
+ end
+ rm_rf "arvados-vscode-cwl-training"
+ end
+end
+
task :clean do
rm_rf "sdk/python/arvados"
rm_rf "sdk/R"
diff --git a/doc/_config.yml b/doc/_config.yml
index 4e0e2f873..de00b070d 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -49,6 +49,7 @@ navbar:
- user/topics/collection-versioning.html.textile.liquid
- user/topics/storage-classes.html.textile.liquid
- Data Analysis with Workflows:
+ - user/user/cwl/arvados-vscode-training.html.md.liquid
- user/cwl/cwl-runner.html.textile.liquid
- user/cwl/cwl-run-options.html.textile.liquid
- user/tutorials/writing-cwl-workflow.html.textile.liquid
diff --git a/doc/user/cwl/arvados-vscode-training.html.md.liquid b/doc/user/cwl/arvados-vscode-training.html.md.liquid
new file mode 100644
index 000000000..883e0736a
--- /dev/null
+++ b/doc/user/cwl/arvados-vscode-training.html.md.liquid
@@ -0,0 +1,165 @@
+---
+layout: default
+navsection: userguide
+title: "Developing Workflows with VSCode"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+
+Imported from https://github.com/arvados/arvados-vscode-cwl-training
+git hash: 05cae0b323f4491a8f211fe1bdf300e4c761ce93
+
+{% endcomment %}
+
+These lessons give step by step instructions for using Visual Studio
+Code to develop CWL workflows on Arvados.
+
+1. Install & set up ssh client on Windows
+1. Install vscode and relevant extensions
+1. Use vscode to connect to an arvados shell node for development
+1. Create an simple command line tool, run it, and view the log
+1. Upload input, create a command line tool to process it, and view the output
+
+# Windows SSH Setup
+
+Linux and MacOS users can skip this step.
+
+1. Install git for windows https://git-scm.com/download/win
+1. Open git bash
+1. Run “ssh-keygen”
+1. Look for “Your public key has been saved in /c/Users/MyUsername/.ssh/id_rsa.pub”
+1. Run “cat /c/Users/MyUsername/.ssh/id_rsa.pub”
+1. Copy the lines starting with “ssh-rsa …”
+1. Open Arvados workbench
+1. Go to “SSH keys” in the user menu
+1. Click + Add new ssh key
+1. Paste the key into “Public key” and enter something for “name”
+1. At the git bash command line, run “ssh shell…” and say “yes” if it asks “do you want to continue connecting”
+1. You should be logged into the Arvados shell node.
+1. Log out by typing “exit”
+
+# VSCode setup
+
+1. Install vscode https://code.visualstudio.com/ and start it up
+1. Vscode: go to “Extensions”
+ 1. search for “remote development” and install the Remote Development extension pack from Microsoft
+1. Vscode: On the left side bar, choose “Remote explorer”
+ 1. In the drop down, choose “SSH targets”
+ 1. Click “Add new”
+ 1. Enter the “ssh shell…” command line you used in step 1(j)
+ 1. Right click the ssh target in the list and select “connect to host in current window”
+1. Vscode: go to “Extensions”
+ 1. Search for “benten” and install “CWL (Rabix/Benten)”
+ 1. Choose “Install extension on the remote ssh host”
+1. Vscode: On the left side bar, choose “Explorer”
+ 1. Select “Clone Repository” and then enter “https://github.com/arvados/arvados-vscode-cwl-training”
+ 1. Choose “Open”
+1. Go to Arvados Workbench
+ 1. Workbench: In the user menu, select “Current token”
+ 1. Vscode: Click on the “Terminal” menu
+ 1. Vscode: Click “Run Task…”
+ 1. Vscode: Select “Configure Arvados”
+ 1. Workbench: Copy the string following “ARVADOS_API_HOST=”
+ 1. Vscode: Paste the string at the “Value for ARVADOS_API_HOST” prompt
+ 1. Workbench: Copy the string following “ARVADOS_API_TOKEN=”
+ 1. Vscode: Paste the string at the “Value for ARVADOS_API_TOKEN” prompt
+
+# Register & run a workflow
+
+1. Vscode: Click on the “lesson1/main.cwl” file
+ 1. Click on the “Terminal” menu
+ 1. Click “Run Task…”
+ 1. Select “Register or update CWL workflow on Arvados Workbench”
+1. Workbench: Go to “+NEW” and select “New project”
+ 1. Enter a name for the project like “Lesson 1”
+ 1. You should arrive at the panel for the new project
+1. Workbench: With “Lesson 1” selected
+ 1. Click on “+NEW” and select “Run a process”
+ 1. Select “CWL training lesson 1” from the list and click “Next”
+ 1. Enter a name for this run like “First training run”
+ 1. Enter a message (under “#main/message”) like “Hello world”
+ 1. Click “Run process”
+ 1. This should take you to a panel showing the workflow run status
+1. Workbench: workflow run status panel
+ 1. Click on the three vertical dots in the top-right corner
+ 1. Choose “Log”
+ 1. Under “event type” choose “stdout”
+ 1. You should see your message
+
+# Working with input and output files
+
+1. Vscode: Click on the “lesson2/main.cwl” file
+ 1. Click on the “Terminal” menu
+ 1. Click “Run Task…”
+ 1. Select “Register or update CWL workflow on Arvados Workbench”
+1. Go to your desktop
+ 1. Using a text editor such as notepad, create a file “message.txt”
+ 1. Enter a message like “Hello world” and save
+1. Workbench: Go to “+NEW” and select “New project”
+ 1. Enter a name for the project like “Lesson 2”
+ 1. You should arrive at the panel for the new project
+1. Arvados workbench: With “Lesson 2” project selected
+ 1. Click on +NEW and select “New collection”
+ 1. Call the collection “my message”
+ 1. Drag and drop “message.txt” into the browser
+ 1. Click “Create a collection”
+ 1. The file should be uploaded and then you will be on the collection page
+1. Workbench: Select the “Lesson 2” project
+ 1. Click on “+NEW” and select “Run a process”
+ 1. Select “CWL training lesson 2” from the list and click “Next”
+ 1. Enter a name for this run like “Second training run”
+ 1. Click on “#main/message”
+ 1. A selection dialog box will appear
+ 1. Navigate to the collection you created in step (13) and choose “message.txt”
+ 1. Click “Run process”
+ 1. This should take you to a panel showing the workflow run status
+1. Workbench: workflow run status panel
+ 1. Wait for the process to complete
+ 1. Click on the dot menu
+ 1. Choose “Outputs”
+ 1. Right click on “reverse.txt”
+ 1. Click on “Download”
+ 1. Open the downloaded file. It should have your results.
+
+# Register a workflow with default inputs
+
+The default value for the “message” parameter will taken from the “lesson3/defaults.yaml” file
+
+1. Workbench: Go to “+NEW” and select “New project”
+ 1. Enter a name for the project like “Lesson 3”
+ 1. You should arrive at the panel for the new project
+1. Workbench: With “Lesson 3” selected
+ 1. Click on “+NEW” and select “Run a process”
+ 1. Select “CWL training lesson 3” from the list and click “Next”
+ 1. Enter a name for this run like “Third training run”
+ 1. The “#main/message” parameter will be pre-filled with your default value. You can choose to change it or use the default.
+ 1. Click “Run process”
+ 1. This should take you to the status page for this workflow
+
+# Run a workflow without registering it
+
+The “message” parameter will be taken from the file “lesson4/main-input.yaml”. This is useful during development.
+
+1. Workbench: Go to “+NEW” and select “New project”
+ 1. Enter a name for the project like “Lesson 4”
+ 1. You should arrive at the panel for the new project
+ 1. Click on “Additional info” in the upper right to expand the “info” panel
+ 1. Under “Project UUID” click the “Copy to clipboard” button
+1. Vscode: Select the file “lesson4/main.cwl”
+ 1. Click on the “Terminal” menu
+ 1. Click “Run Task…”
+ 1. Select “Set Arvados project UUID”
+ 1. Paste the project UUID from workbench at the prompt
+1. Vscode: Select the file “lesson4/main.cwl”
+ 1. Click on the “Terminal” menu
+ 1. Click “Run Task…”
+ 1. Select “Run CWL workflow on Arvados”
+1. Vscode: In the bottom panel select the “Terminal” tab
+ 1. In the upper right corner of the Terminal tab select “Task - Run CWL Workflow” from the drop-down
+ 1. Look for logging text like “submitted container_request zzzzz-xvhdp-0123456789abcde”
+ 1. Highlight and copy the workflow identifier (this the string containing -xvhdp- in the middle)
+ 1. The results of this run will appear in the terminal when the run completes.
+1. Workbench: Paste the workflow identifier into the search box
+ 1. This will take you to the status page for this workflow
commit 165f300730a1c69b71a0b4008191baeed074f606
Author: Ward Vandewege <ward at curii.com>
Date: Wed Feb 10 09:19:22 2021 -0500
Documentation: include the cluster id namespace explanation on the
prerequisites page.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/architecture/federation.html.textile.liquid b/doc/architecture/federation.html.textile.liquid
index 751282843..1ae8b6006 100644
--- a/doc/architecture/federation.html.textile.liquid
+++ b/doc/architecture/federation.html.textile.liquid
@@ -20,9 +20,9 @@ h2(#cluster_id). Cluster identifiers
Clusters are identified by a five-digit alphanumeric id (numbers and lowercase letters). There are 36 ^5^ = 60466176 possible cluster identifiers.
-* For automated tests purposes, use "z****"
+* For automated test purposes, use "z****"
* For experimental/local-only/private clusters that won't ever be visible on the public Internet, use "x****"
-* For long-lived clusters, we recommend reserving a cluster id. Contact "info at curii.com":mailto:info at curii.com
+* For long-lived clusters, we recommend reserving a cluster id. Contact "info at curii.com":mailto:info at curii.com for more information.
Cluster identifiers are mapped API server hosts one of two ways:
diff --git a/doc/install/install-manual-prerequisites.html.textile.liquid b/doc/install/install-manual-prerequisites.html.textile.liquid
index 8f45b29a4..364e8cd2b 100644
--- a/doc/install/install-manual-prerequisites.html.textile.liquid
+++ b/doc/install/install-manual-prerequisites.html.textile.liquid
@@ -119,7 +119,13 @@ For a small demo installation, it is possible to run all the Arvados services on
h2(#clusterid). Arvados Cluster ID
-Each Arvados installation should have a cluster identifier, which is a unique 5-character lowercase alphanumeric string. Here is one way to make a random 5-character string:
+Each Arvados installation is identified by a cluster identifier, which is a unique 5-character lowercase alphanumeric string. There are 36 5 = 60466176 possible cluster identifiers.
+
+* For automated test purposes, use “z****”
+* For experimental/local-only/private clusters that won’t ever be visible on the public Internet, use “x****”
+* For long-lived clusters, we recommend reserving a cluster id. Contact "info at curii.com":mailto:info at curii.com for more information.
+
+Here is one way to make a random 5-character string:
<notextile>
<pre><code>~$ <span class="userinput">tr -dc 0-9a-z </dev/urandom | head -c5; echo</span>
commit 7df49ca80b5ed10c9efc644de5b07ff3eb1829c8
Author: Ward Vandewege <ward at curii.com>
Date: Tue Feb 9 15:14:10 2021 -0500
17355: keepstore should take the volume's AccessViaHosts ReadOnly flag
into account when reporting ReadOnly status of a volume on
startup.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/services/keepstore/volume.go b/services/keepstore/volume.go
index 4d8a0aec7..26e6b7318 100644
--- a/services/keepstore/volume.go
+++ b/services/keepstore/volume.go
@@ -315,7 +315,7 @@ func makeRRVolumeManager(logger logrus.FieldLogger, cluster *arvados.Cluster, my
if err != nil {
return nil, fmt.Errorf("error initializing volume %s: %s", uuid, err)
}
- logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly)
+ logger.Printf("started volume %s (%s), ReadOnly=%v", uuid, vol, cfgvol.ReadOnly || va.ReadOnly)
sc := cfgvol.StorageClasses
if len(sc) == 0 {
commit 6543a0758850a52c8c1d3d9879c8937a40fc1640
Author: Ward Vandewege <ward at curii.com>
Date: Tue Mar 9 16:06:59 2021 -0500
17119: one more fix based on review feedback.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index e1fc2691a..6e933fc00 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -107,9 +107,9 @@ func (rtr *router) sendResponse(w http.ResponseWriter, req *http.Request, resp i
rtr.mungeItemFields(item)
slice[i] = item
}
- if opts.Count == "none" {
- delete(tmp, "items_available")
- }
+ }
+ if opts.Count == "none" {
+ delete(tmp, "items_available")
}
} else {
tmp = applySelectParam(opts.Select, tmp)
commit 3474a8e3567f8119a7149dfa6b8beee16549e89b
Author: Ward Vandewege <ward at curii.com>
Date: Mon Mar 8 17:03:29 2021 -0500
17119: implement review feedback.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 09540a5f4..a9352098d 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -423,8 +423,7 @@ func (conn *Conn) GroupContents(ctx context.Context, options arvados.GroupConten
}
func (conn *Conn) GroupShared(ctx context.Context, options arvados.ListOptions) (arvados.GroupList, error) {
- // FIXME is this right?? We don't have options.UUID to cue the chooseBackend off
- return conn.chooseBackend(conn.cluster.ClusterID).GroupShared(ctx, options)
+ return conn.chooseBackend(options.ClusterID).GroupShared(ctx, options)
}
func (conn *Conn) GroupDelete(ctx context.Context, options arvados.DeleteOptions) (arvados.Group, error) {
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index aee2fc590..e1fc2691a 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -82,38 +82,34 @@ func (rtr *router) sendResponse(w http.ResponseWriter, req *http.Request, resp i
defaultItemKind = strings.TrimSuffix(respKind, "List")
}
- var items, included []interface{}
- var itemsOK, includedOK bool
- items, itemsOK = tmp["items"].([]interface{})
- included, includedOK = tmp["included"].([]interface{})
- if includedOK && len(included) > 0 {
- items = append(items, included...)
- }
-
- if itemsOK {
- for i, item := range items {
- // Fill in "kind" by inspecting UUID/PDH if
- // possible; fall back on assuming each
- // Items[] entry in an "arvados#fooList"
- // response should have kind="arvados#foo".
- item, _ := item.(map[string]interface{})
- infix := ""
- if uuid, _ := item["uuid"].(string); len(uuid) == 27 {
- infix = uuid[6:11]
+ if _, isListResponse := tmp["items"].([]interface{}); isListResponse {
+ items, _ := tmp["items"].([]interface{})
+ included, _ := tmp["included"].([]interface{})
+ for _, slice := range [][]interface{}{items, included} {
+ for i, item := range slice {
+ // Fill in "kind" by inspecting UUID/PDH if
+ // possible; fall back on assuming each
+ // Items[] entry in an "arvados#fooList"
+ // response should have kind="arvados#foo".
+ item, _ := item.(map[string]interface{})
+ infix := ""
+ if uuid, _ := item["uuid"].(string); len(uuid) == 27 {
+ infix = uuid[6:11]
+ }
+ if k := kind(infixMap[infix]); k != "" {
+ item["kind"] = k
+ } else if pdh, _ := item["portable_data_hash"].(string); pdh != "" {
+ item["kind"] = "arvados#collection"
+ } else if defaultItemKind != "" {
+ item["kind"] = defaultItemKind
+ }
+ item = applySelectParam(opts.Select, item)
+ rtr.mungeItemFields(item)
+ slice[i] = item
}
- if k := kind(infixMap[infix]); k != "" {
- item["kind"] = k
- } else if pdh, _ := item["portable_data_hash"].(string); pdh != "" {
- item["kind"] = "arvados#collection"
- } else if defaultItemKind != "" {
- item["kind"] = defaultItemKind
+ if opts.Count == "none" {
+ delete(tmp, "items_available")
}
- item = applySelectParam(opts.Select, item)
- rtr.mungeItemFields(item)
- items[i] = item
- }
- if opts.Count == "none" {
- delete(tmp, "items_available")
}
} else {
tmp = applySelectParam(opts.Select, tmp)
commit 198a6dd8ceb61dd2f6fca7b28d3b074f9b01f67d
Author: Ward Vandewege <ward at curii.com>
Date: Fri Mar 5 18:13:44 2021 -0500
17119: add (rudimentary) filter group support to workbench.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/apps/workbench/app/assets/javascripts/components/search.js b/apps/workbench/app/assets/javascripts/components/search.js
index fc6308678..83ed1a68d 100644
--- a/apps/workbench/app/assets/javascripts/components/search.js
+++ b/apps/workbench/app/assets/javascripts/components/search.js
@@ -127,6 +127,12 @@ window.Search = {
filters: [['group_class', '=', 'project']],
description: 'project',
},
+ {
+ wb_path: 'projects',
+ api_path: 'arvados/v1/groups',
+ filters: [['group_class', '=', 'filter']],
+ description: 'project',
+ },
{
wb_path: 'collections',
api_path: 'arvados/v1/collections',
diff --git a/apps/workbench/app/controllers/actions_controller.rb b/apps/workbench/app/controllers/actions_controller.rb
index 885f53936..b0b7a0b64 100644
--- a/apps/workbench/app/controllers/actions_controller.rb
+++ b/apps/workbench/app/controllers/actions_controller.rb
@@ -34,7 +34,7 @@ class ActionsController < ApplicationController
@object.link_class == 'name' and
ArvadosBase::resource_class_for_uuid(@object.head_uuid) == Collection
redirect_to collection_path(id: @object.uuid)
- elsif @object.is_a?(Group) and @object.group_class == 'project'
+ elsif @object.is_a?(Group) and (@object.group_class == 'project' or @object.group_class == 'filter')
redirect_to project_path(id: @object.uuid)
elsif @object
redirect_to @object
diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb
index 6d139cd5f..04449d5f1 100644
--- a/apps/workbench/app/controllers/application_controller.rb
+++ b/apps/workbench/app/controllers/application_controller.rb
@@ -95,7 +95,7 @@ class ApplicationController < ActionController::Base
# exception here than in a template.)
unless current_user.nil?
begin
- my_starred_projects current_user
+ my_starred_projects current_user, 'project'
build_my_wanted_projects_tree current_user
rescue ArvadosApiClient::ApiError
# Fall back to the default-setting code later.
@@ -824,7 +824,7 @@ class ApplicationController < ActionController::Base
helper_method :all_projects
def all_projects
@all_projects ||= Group.
- filter([['group_class','=','project']]).order('name')
+ filter([['group_class','IN',['project','filter']]]).order('name')
end
helper_method :my_projects
@@ -925,13 +925,17 @@ class ApplicationController < ActionController::Base
end
helper_method :my_starred_projects
- def my_starred_projects user
+ def my_starred_projects user, group_class
return if defined?(@starred_projects) && @starred_projects
links = Link.filter([['owner_uuid', 'in', ["#{Rails.configuration.ClusterID}-j7d0g-publicfavorites", user.uuid]],
['link_class', '=', 'star'],
['head_uuid', 'is_a', 'arvados#group']]).with_count("none").select(%w(head_uuid))
uuids = links.collect { |x| x.head_uuid }
- starred_projects = Group.filter([['uuid', 'in', uuids]]).order('name').with_count("none")
+ if group_class == ""
+ starred_projects = Group.filter([['uuid', 'in', uuids]]).order('name').with_count("none")
+ else
+ starred_projects = Group.filter([['uuid', 'in', uuids],['group_class', '=', group_class]]).order('name').with_count("none")
+ end
@starred_projects = starred_projects.results
end
@@ -949,7 +953,7 @@ class ApplicationController < ActionController::Base
@too_many_projects = false
@reached_level_limit = false
while from_top.size <= page_size*2
- current_level = Group.filter([['group_class','=','project'],
+ current_level = Group.filter([['group_class','IN',['project','filter']],
['owner_uuid', 'in', uuids]])
.order('name').limit(page_size*2)
break if current_level.results.size == 0
diff --git a/apps/workbench/app/controllers/groups_controller.rb b/apps/workbench/app/controllers/groups_controller.rb
index 5da55be0b..6abd2ff11 100644
--- a/apps/workbench/app/controllers/groups_controller.rb
+++ b/apps/workbench/app/controllers/groups_controller.rb
@@ -4,7 +4,7 @@
class GroupsController < ApplicationController
def index
- @groups = Group.filter [['group_class', '!=', 'project']]
+ @groups = Group.filter [['group_class', '!=', 'project'], ['group_class', '!=', 'filter']]
@group_uuids = @groups.collect &:uuid
@links_from = Link.where(link_class: 'permission', tail_uuid: @group_uuids).with_count("none")
@links_to = Link.where(link_class: 'permission', head_uuid: @group_uuids).with_count("none")
@@ -12,7 +12,7 @@ class GroupsController < ApplicationController
end
def show
- if @object.group_class == 'project'
+ if @object.group_class == 'project' or @object.group_class == 'filter'
redirect_to(project_path(@object))
else
super
diff --git a/apps/workbench/app/helpers/application_helper.rb b/apps/workbench/app/helpers/application_helper.rb
index 786716eb3..f22ab5016 100644
--- a/apps/workbench/app/helpers/application_helper.rb
+++ b/apps/workbench/app/helpers/application_helper.rb
@@ -176,7 +176,7 @@ module ApplicationHelper
raw(link_name)
else
controller_class = resource_class.to_s.tableize
- if controller_class.eql?('groups') and object.andand.group_class.eql?('project')
+ if controller_class.eql?('groups') and (object.andand.group_class.eql?('project') or object.andand.group_class.eql?('filter'))
controller_class = 'projects'
end
(link_to raw(link_name), { controller: controller_class, action: 'show', id: ((opts[:name_link].andand.uuid) || link_uuid) }, style_opts) + raw(tags)
diff --git a/apps/workbench/app/models/group.rb b/apps/workbench/app/models/group.rb
index 08b13bf34..ea3da2db5 100644
--- a/apps/workbench/app/models/group.rb
+++ b/apps/workbench/app/models/group.rb
@@ -20,6 +20,13 @@ class Group < ArvadosBase
ret
end
+ def editable?
+ if group_class == 'filter'
+ return false
+ end
+ super
+ end
+
def contents params={}
res = arvados_api_client.api self.class, "/#{self.uuid}/contents", {
_method: 'GET'
@@ -30,7 +37,7 @@ class Group < ArvadosBase
end
def class_for_display
- group_class == 'project' ? 'Project' : super
+ (group_class == 'project' or group_class == 'filter') ? 'Project' : super
end
def textile_attributes
diff --git a/apps/workbench/app/views/application/_projects_tree_menu.html.erb b/apps/workbench/app/views/application/_projects_tree_menu.html.erb
index 08d3b8111..805d5279c 100644
--- a/apps/workbench/app/views/application/_projects_tree_menu.html.erb
+++ b/apps/workbench/app/views/application/_projects_tree_menu.html.erb
@@ -2,7 +2,7 @@
SPDX-License-Identifier: AGPL-3.0 %>
-<% starred_projects = my_starred_projects current_user%>
+<% starred_projects = my_starred_projects current_user, '' %>
<% if starred_projects.andand.any? %>
<li role="presentation" class="dropdown-header">
My favorite projects
diff --git a/apps/workbench/app/views/projects/_choose.html.erb b/apps/workbench/app/views/projects/_choose.html.erb
index 8e5695e6d..633a9ba33 100644
--- a/apps/workbench/app/views/projects/_choose.html.erb
+++ b/apps/workbench/app/views/projects/_choose.html.erb
@@ -17,7 +17,7 @@ SPDX-License-Identifier: AGPL-3.0 %>
<div class="modal-body">
<div class="selectable-container" style="height: 15em; overflow-y: scroll">
- <% starred_projects = my_starred_projects current_user%>
+ <% starred_projects = my_starred_projects current_user, 'project' %>
<% if starred_projects.andand.any? %>
<% writable_projects = starred_projects.select(&:editable?) %>
<% writable_projects.each do |projectnode| %>
diff --git a/apps/workbench/app/views/projects/show.html.erb b/apps/workbench/app/views/projects/show.html.erb
index 6066335a1..60f2d2340 100644
--- a/apps/workbench/app/views/projects/show.html.erb
+++ b/apps/workbench/app/views/projects/show.html.erb
@@ -11,6 +11,9 @@ SPDX-License-Identifier: AGPL-3.0 %>
<%= render_editable_attribute @object, 'name', nil, { 'data-emptytext' => "New project" } %>
<% end %>
</h2>
+ <% if @object.class == Group and @object.group_class == 'filter' %>
+ This is a filter group.
+ <% end %>
<% end %>
<%
diff --git a/apps/workbench/test/controllers/projects_controller_test.rb b/apps/workbench/test/controllers/projects_controller_test.rb
index 27d7dedc9..2d379f864 100644
--- a/apps/workbench/test/controllers/projects_controller_test.rb
+++ b/apps/workbench/test/controllers/projects_controller_test.rb
@@ -523,12 +523,12 @@ EOT
use_token user
ctrl = ProjectsController.new
current_user = User.find(api_fixture('users')[user]['uuid'])
- my_starred_project = ctrl.send :my_starred_projects, current_user
+ my_starred_project = ctrl.send :my_starred_projects, current_user, ''
assert_equal(size, my_starred_project.andand.size)
ctrl2 = ProjectsController.new
current_user = User.find(api_fixture('users')[user]['uuid'])
- my_starred_project = ctrl2.send :my_starred_projects, current_user
+ my_starred_project = ctrl2.send :my_starred_projects, current_user, ''
assert_equal(size, my_starred_project.andand.size)
end
end
@@ -542,7 +542,7 @@ EOT
use_token :project_viewer
current_user = User.find(api_fixture('users')['project_viewer']['uuid'])
ctrl = ProjectsController.new
- my_starred_project = ctrl.send :my_starred_projects, current_user
+ my_starred_project = ctrl.send :my_starred_projects, current_user, ''
assert_equal(0, my_starred_project.andand.size)
# share it again
@@ -560,7 +560,7 @@ EOT
# verify that the project is again included in starred projects
use_token :project_viewer
ctrl = ProjectsController.new
- my_starred_project = ctrl.send :my_starred_projects, current_user
+ my_starred_project = ctrl.send :my_starred_projects, current_user, ''
assert_equal(1, my_starred_project.andand.size)
end
end
commit fe8a7a83e41dbc236621d5cd90128b0cf6721203
Author: Ward Vandewege <ward at curii.com>
Date: Fri Mar 5 10:44:23 2021 -0500
17119: add filter group support to arv-mount.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/services/fuse/arvados_fuse/fusedir.py b/services/fuse/arvados_fuse/fusedir.py
index 1fab2e0fb..cfef43fdb 100644
--- a/services/fuse/arvados_fuse/fusedir.py
+++ b/services/fuse/arvados_fuse/fusedir.py
@@ -683,9 +683,11 @@ and the directory will appear if it exists.
if group_uuid_pattern.match(k):
project = self.api.groups().list(
- filters=[['group_class', '=', 'project'], ["uuid", "=", k]]).execute(num_retries=self.num_retries)
+ filters=[["uuid", "=", k]]).execute(num_retries=self.num_retries)
if project[u'items_available'] == 0:
return False
+ if project[u'items'][0][u'group_class'] != u'project' and project[u'items'][0][u'group_class'] != u'filter':
+ return False
e = self.inodes.add_entry(ProjectDirectory(
self.inode, self.inodes, self.api, self.num_retries, project[u'items'][0]))
else:
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index b2816ac16..54316bb9a 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -129,7 +129,9 @@ class FuseMagicTest(MountTestBase):
self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
+ self.filter_group = run_test_server.fixture('groups')['afiltergroup']['uuid']
self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
+ self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
cw = arvados.CollectionWriter()
@@ -157,6 +159,11 @@ class FuseMagicTest(MountTestBase):
llfuse.listdir(os.path.join(self.mounttmp, self.test_project)))
self.assertIn(self.collection_in_test_project,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.test_project)))
+ self.assertIn(self.collection_in_filter_group,
+ llfuse.listdir(os.path.join(self.mounttmp, self.filter_group)))
+ self.assertIn(self.collection_in_filter_group,
+ llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.filter_group)))
+
mount_ls = llfuse.listdir(self.mounttmp)
self.assertIn('README', mount_ls)
@@ -166,6 +173,8 @@ class FuseMagicTest(MountTestBase):
self.assertIn(self.test_project, mount_ls)
self.assertIn(self.test_project,
llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
+ self.assertIn(self.filter_group,
+ llfuse.listdir(os.path.join(self.mounttmp, 'by_id')))
with self.assertRaises(OSError):
llfuse.listdir(os.path.join(self.mounttmp, 'by_id', self.non_project_group))
commit 1a427d62dd9ff3fc9294879b0ae5fe2b9b6195c5
Author: Ward Vandewege <ward at curii.com>
Date: Thu Mar 4 17:42:06 2021 -0500
17119: add documentation.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/_config.yml b/doc/_config.yml
index b0355e269..4e0e2f873 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -44,6 +44,7 @@ navbar:
- user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
- user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
- user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
+ - user/topics/projects.html.textile.liquid
- user/topics/arv-copy.html.textile.liquid
- user/topics/collection-versioning.html.textile.liquid
- user/topics/storage-classes.html.textile.liquid
diff --git a/doc/api/methods/groups.html.textile.liquid b/doc/api/methods/groups.html.textile.liquid
index f85e621db..73df7a066 100644
--- a/doc/api/methods/groups.html.textile.liquid
+++ b/doc/api/methods/groups.html.textile.liquid
@@ -25,8 +25,9 @@ Each Group has, in addition to the "Common resource fields":{{site.baseurl}}/api
table(table table-bordered table-condensed).
|_. Attribute|_. Type|_. Description|_. Example|
|name|string|||
-|group_class|string|Type of group. This does not affect behavior, but determines how the group is presented in the user interface. For example, @project@ indicates that the group should be displayed by Workbench and arv-mount as a project for organizing and naming objects.|@"project"@
-null|
+|group_class|string|Type of group. @project@ and @filter@ indicate that the group should be displayed by Workbench and arv-mount as a project for organizing and naming objects. @role@ means FIXME. |@"filter"@
+@"project"@
+@"role"@|
|description|text|||
|properties|hash|User-defined metadata, may be used in queries using "subproperty filters":{{site.baseurl}}/api/methods.html#subpropertyfilters ||
|writable_by|array|List of UUID strings identifying Users and other Groups that have write permission for this Group. Only users who are allowed to administer the Group will receive a full list. Other users will receive a partial list that includes the Group's owner_uuid and (if applicable) their own user UUID.||
@@ -34,6 +35,49 @@ null|
|delete_at|datetime|If @delete_at@ is non-null and in the past, the group and all objects directly or indirectly owned by the group may be permanently deleted.||
|is_trashed|datetime|True if @trash_at@ is in the past, false if not.||
+ at filter@ groups have a special @properties@ field named @filters@, which must be an array of arrays with 3 elements, each describing a filter. @filter@ groups are virtual groups; they can not own other objects. Filter attributes must include the object type (@collections@, @container_requests@, @groups@, @workflows@), separated with a dot from the field to be filtered on. Filters are applied with an implied *and* between them, but each filter only applies to the object type specified. The results are subject to the usual access controls - they are a subset of all objects the user can see. Here is an example:
+
+<pre>
+ "properties":{
+ "filters":[
+ [
+ "groups.name",
+ "like",
+ "Public%"
+ ]
+ ]
+ },
+</pre>
+
+This @filter@ group will return all groups (projects) that have a name starting with the word @Public@ and are visible to the user issuing the query. Because groups can contain many types of object, it will also return all objects of other types that the user can see.
+
+The 'is_a' filter operator is of particular interest to limit the @filter@ group 'content' to the desired object(s). When the 'is_a' operator is used, the attribute must be 'uuid'. The operand may be a string or an array which means objects of either type will match the filter. This example will return all groups (projects) that have a name starting with the word @Public@, as well as all collections that are in the project with uuid @zzzzz-j7d0g-0123456789abcde at .
+
+<pre>
+ "properties":{
+ "filters":[
+ [
+ "groups.name",
+ "like",
+ "Public%"
+ ],
+ [
+ "collections.owner_uuid",
+ "=",
+ "zzzzz-j7d0g-0123456789abcde"
+ ],
+ [
+ "uuid",
+ "is_a",
+ [
+ "arvados#group",
+ "arvados#collection"
+ ]
+ ]
+ ]
+ },
+ </pre>
+
h2. Methods
See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update at .
diff --git a/doc/api/permission-model.html.textile.liquid b/doc/api/permission-model.html.textile.liquid
index 54c4a3331..f54dc8bf2 100644
--- a/doc/api/permission-model.html.textile.liquid
+++ b/doc/api/permission-model.html.textile.liquid
@@ -26,7 +26,7 @@ There are four levels of permission: *none*, *can_read*, *can_write*, and *can_m
h2. Ownership
-All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group". For Group, the @group_class@ must be a "project".
+All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group". For Group, the @group_class@ must be "filter", "project" or "role".
The User or Group specified by @owner_uuid@ has *can_manage* permission on the object. This permission is one way: an object that is owned does not get any special permissions on the User or Group that owns it.
@@ -63,9 +63,15 @@ h2. Projects and Roles
A "project" is a subtype of Group that is displayed as a "Project" in Workbench, and as a directory by @arv-mount at .
* A project can own things (appear in @owner_uuid@)
* A project can be owned by a user or another project.
-* The name of a project is unique only among projects with the same owner_uuid.
+* The name of a project is unique only among projects and filters with the same owner_uuid.
* Projects can be targets (@head_uuid@) of permission links, but not origins (@tail_uuid@). Putting a project in a @tail_uuid@ field is an error.
+A "filter" is a subtype of Group that is displayed as a "Project" in Workbench, and as a directory by @arv-mount at . See "the groups API documentation":/api/methods/groups.html for more information.
+* A filter cannot own things (cannot appear in @owner_uuid@). Putting a role in an @owner_uuid@ field is an error.
+* A filter can be owned by a user or another project.
+* The name of a filter is unique only among projects and filters with the same owner_uuid.
+* Filters can be targets (@head_uuid@) of permission links, but not origins (@tail_uuid@). Putting a filter in a @tail_uuid@ field is an error.
+
A "role" is a subtype of Group that is treated in Workbench as a group of users who have permissions in common (typically an organizational group).
* A role cannot own things (cannot appear in @owner_uuid@). Putting a role in an @owner_uuid@ field is an error.
* All roles are owned by the system user.
diff --git a/doc/user/topics/projects.html.textile.liquid b/doc/user/topics/projects.html.textile.liquid
new file mode 100644
index 000000000..f4b236c4c
--- /dev/null
+++ b/doc/user/topics/projects.html.textile.liquid
@@ -0,0 +1,36 @@
+---
+layout: default
+navsection: userguide
+title: "Projects"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Arvados @projects@ are used to organize objects. Projects can contain @collections@, @container requests@, @workflows@, etc. Projects can also contain other projects. An object is part of a project if the @owner_uuid@ of the object is set to the uuid of the project.
+
+Projects are implemented as a subtype of the Arvados @group@ object type, with @group_class@ set to the value "project". More information is available in the "groups API reference":/api/methods/groups.html.
+
+Projects can be manipulated via Workbench, the cli tools, the SDKs, and the Arvados APIs.
+
+h2. The home project
+
+Each user has a @home project@, which is implemented differently. This is a virtual project that is comprised of all objects owned by the user, in other words, all objects with the @owner_uuid@ set to the @uuid@ of the user. The home project is accessible via Workbench, which makes it easy view its contents and to move objects from and to the home project. The home project is also accessible via FUSE, WebDAV and the S3 interface.
+
+The same thing can be done via the APIs. To put something in a user's home project via the cli or SDKs, one would set the @owner_uuid@ of the object to the user's @uuid at . This also implies that this user now has full ownership and control over that object.
+
+The contents of the home project can be accessed with the @group contents@ API, e.g. via the cli with this command:
+<pre>arv group contents --uuid zzzzz-tpzed-123456789012345</pre>
+In this command, `zzzzz-tpzed-123456789012345` is a @user@ uuid, which is unusual because we are using it as the argument to a @groups@ API. The @group contents@ API is normally used with a @group@ uuid.
+
+Because the home project is a virtual project, other operations via the @groups@ API are not supported.
+
+h2. Filter groups
+
+Filter groups are another type of virtual project. They are implemented as an Arvados @group@ object with @group_class@ set to the value "filter".
+
+Filter groups define one or more filters which are applied to all objects that the current user can see, and returned as the contents of the @group at . @filter@ groups are described in more detail in the "groups API reference":/api/methods/groups.html.
+
+Filter groups are accessible (read-only) via Workbench and the Arvados FUSE mount, WebDAV and S3 interface. Filter groups must currently be defined via the API, SDK or cli, there is no Workbench support yet.
commit 66fa80a47802b23a75b217e7e9a92e0bf6781b3e
Author: Ward Vandewege <ward at curii.com>
Date: Wed Mar 3 16:04:54 2021 -0500
17119: add filter validation for filter groups. Add a test for the
'is_a' filter.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/go/arvados/fs_project_test.go b/sdk/go/arvados/fs_project_test.go
index 9d77c31d8..0564e2fae 100644
--- a/sdk/go/arvados/fs_project_test.go
+++ b/sdk/go/arvados/fs_project_test.go
@@ -64,6 +64,15 @@ func (s *SiteFSSuite) TestFilterGroup(c *check.C) {
_, err = s.fs.OpenFile("/fg2/A Project", 0, 0)
c.Assert(err, check.IsNil)
+
+ // An 'is_a' 'arvados#collection' filter means only collections should be returned.
+ s.fs.MountProject("fg3", fixtureAFilterGroupThreeUUID)
+
+ _, err = s.fs.OpenFile("/fg3/baz_file", 0, 0)
+ c.Assert(err, check.IsNil)
+
+ _, err = s.fs.OpenFile("/fg3/A Subproject", 0, 0)
+ c.Assert(err, check.Not(check.IsNil))
}
func (s *SiteFSSuite) TestCurrentUserHome(c *check.C) {
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index 02f21ded5..b1c627f89 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -20,6 +20,7 @@ const (
fixtureAProjectUUID = "zzzzz-j7d0g-v955i6s2oi1cbso"
fixtureThisFilterGroupUUID = "zzzzz-j7d0g-thisfiltergroup"
fixtureAFilterGroupTwoUUID = "zzzzz-j7d0g-afiltergrouptwo"
+ fixtureAFilterGroupThreeUUID = "zzzzz-j7d0g-filtergroupthre"
fixtureFooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
fixtureFooCollectionName = "zzzzz-4zz18-fy296fx3hot09f7 added sometime"
fixtureFooCollectionPDH = "1f4b0bc7583c2a7f9102c395f4ffc5e3+45"
diff --git a/services/api/app/models/group.rb b/services/api/app/models/group.rb
index 870e0d0c4..1bf2cf5d5 100644
--- a/services/api/app/models/group.rb
+++ b/services/api/app/models/group.rb
@@ -18,6 +18,7 @@ class Group < ArvadosModel
validate :ensure_filesystem_compatible_name
validate :check_group_class
+ validate :check_filter_group_filters
before_create :assign_name
after_create :after_ownership_change
after_create :update_trash
@@ -56,6 +57,40 @@ class Group < ArvadosModel
end
end
+ def check_filter_group_filters
+ if group_class == 'filter'
+ if !self.properties.key?("filters")
+ return
+ end
+ if !self.properties["filters"].is_a?(Array)
+ errors.add :properties, "filters property must be an array of arrays, each with 3 elements"
+ return
+ end
+ self.properties["filters"].each do |filter|
+ if !filter.is_a?(Array)
+ errors.add :properties, "filters property must be an array of arrays, each with 3 elements"
+ return
+ end
+ if filter.length() != 3
+ errors.add :properties, "filters property must be an array of arrays, each with 3 elements"
+ return
+ end
+ if !filter[0].include?(".") and filter[0].downcase != "uuid"
+ errors.add :properties, "filter attribute must be 'uuid' or contain a dot (e.g. groups.name)"
+ return
+ end
+ if (filter[0].downcase != "uuid" and filter[1].downcase == "is_a")
+ errors.add :properties, "when filter operator is 'is_a', attribute must be 'uuid'"
+ return
+ end
+ if ! ["=","<","<=",">",">=","!=","like","ilike","in","not in","is_a","exists"].include?(filter[1].downcase)
+ errors.add :properties, "filter operator is not valid (must be =,<,<=,>,>=,!=,like,ilike,in,not in,is_a,exists)"
+ return
+ end
+ end
+ end
+ end
+
def update_trash
if saved_change_to_trash_at? or saved_change_to_owner_uuid?
# The group was added or removed from the trash.
diff --git a/services/api/test/fixtures/groups.yml b/services/api/test/fixtures/groups.yml
index da20f8be9..48925a270 100644
--- a/services/api/test/fixtures/groups.yml
+++ b/services/api/test/fixtures/groups.yml
@@ -133,6 +133,19 @@ afiltergroup2:
properties:
filters: []
+afiltergroup3:
+ uuid: zzzzz-j7d0g-filtergroupthre
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-04-21 15:37:48 -0400
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ modified_at: 2014-04-21 15:37:48 -0400
+ updated_at: 2014-04-21 15:37:48 -0400
+ name: A filter group with an is_a collection filter
+ group_class: filter
+ properties:
+ filters: [["uuid", "is_a", "arvados#collection"]]
+
future_project_viewing_group:
uuid: zzzzz-j7d0g-futrprojviewgrp
owner_uuid: zzzzz-tpzed-000000000000000
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list