[ARVADOS] updated: 2.1.0-98-g52f3ba321
Git user
git at public.arvados.org
Mon Feb 22 16:32:45 UTC 2021
Summary of changes:
build/run-build-packages.sh | 4 +--
.../install-shell-server.html.textile.liquid | 22 ++++++++++++++-
.../getting_started/setup-cli.html.textile.liquid | 20 +++++++++++++
lib/crunchrun/crunchrun.go | 33 ++++++++++++++--------
sdk/cwl/arvados_cwl/pathmapper.py | 9 ++++--
sdk/cwl/arvados_cwl/runner.py | 3 +-
.../cases/cat.cwl => 17267-broken-schemas.cwl} | 13 ++++-----
sdk/cwl/tests/arvados-tests.yml | 10 +++++++
sdk/ruby/lib/arvados.rb | 33 +++++++++++-----------
services/login-sync/bin/arvados-login-sync | 5 +++-
10 files changed, 108 insertions(+), 44 deletions(-)
create mode 100644 doc/user/getting_started/setup-cli.html.textile.liquid
copy sdk/cwl/tests/{federation/cases/cat.cwl => 17267-broken-schemas.cwl} (56%)
via 52f3ba321db95c6d8d8f9b5b13c7e91c77a00592 (commit)
via ee513a52b84e13ded32bd16420c3b4c7053c804c (commit)
via 964383400ae31d3e58a75fba5257095b8b4971dc (commit)
via c9877d5042490896035dab88134dd07421054e4f (commit)
via cc12638657fb78d4b4f31c8d41d92d5198b3e2d8 (commit)
via 7b433917b9476f43e894fbc522c17204a1a9061e (commit)
via 851f8f6d8a0e31be8a9544be95f8b5a391e75d3b (commit)
via ca903a18d44b09b4388b51f49586a237c4037243 (commit)
from f5c01e9ef17ce22b92a9f8661e29ea24e692e30a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 52f3ba321db95c6d8d8f9b5b13c7e91c77a00592
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Jan 19 16:39:30 2021 -0500
17267: Add integration test for broken link in $schemas
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/tests/17267-broken-schemas.cwl b/sdk/cwl/tests/17267-broken-schemas.cwl
new file mode 100644
index 000000000..0dca94520
--- /dev/null
+++ b/sdk/cwl/tests/17267-broken-schemas.cwl
@@ -0,0 +1,13 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.0
+class: CommandLineTool
+$schemas:
+ - http://example.com/schema.xml
+inputs: []
+outputs:
+ out: stdout
+baseCommand: [echo, "foo"]
+stdout: foo.txt
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index a46decd96..668604f88 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -345,3 +345,13 @@
}
tool: 16377-missing-default.cwl
doc: "Test issue 16377 - missing default fails even when it should be overridden by valid input"
+
+- job: hello.yml
+ output:
+ "out":
+ "checksum": "sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15"
+ "class": "File"
+ "location": "foo.txt"
+ "size": 4
+ tool: 17267-broken-schemas.cwl
+ doc: "Test issue 17267 - inaccessible $schemas URL is not a fatal error"
commit ee513a52b84e13ded32bd16420c3b4c7053c804c
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Jan 18 14:36:07 2021 -0500
17267: Don't immediately fail if an extension schema is missing
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/pathmapper.py b/sdk/cwl/arvados_cwl/pathmapper.py
index 5bad29077..60e92a399 100644
--- a/sdk/cwl/arvados_cwl/pathmapper.py
+++ b/sdk/cwl/arvados_cwl/pathmapper.py
@@ -99,9 +99,12 @@ class ArvPathMapper(PathMapper):
if srcobj["class"] == "Directory" and "listing" not in srcobj:
raise WorkflowException("Directory literal '%s' is missing `listing`" % src)
elif src.startswith("http:") or src.startswith("https:"):
- keepref = http_to_keep(self.arvrunner.api, self.arvrunner.project_uuid, src)
- logger.info("%s is %s", src, keepref)
- self._pathmap[src] = MapperEnt(keepref, keepref, srcobj["class"], True)
+ try:
+ keepref = http_to_keep(self.arvrunner.api, self.arvrunner.project_uuid, src)
+ logger.info("%s is %s", src, keepref)
+ self._pathmap[src] = MapperEnt(keepref, keepref, srcobj["class"], True)
+ except Exception as e:
+ logger.warning(str(e))
else:
self._pathmap[src] = MapperEnt(src, src, srcobj["class"], True)
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index f06453116..42cba5aef 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -428,7 +428,8 @@ def upload_dependencies(arvrunner, name, document_loader,
if "$schemas" in workflowobj:
sch = CommentedSeq()
for s in workflowobj["$schemas"]:
- sch.append(mapper.mapper(s).resolved)
+ if s in mapper:
+ sch.append(mapper.mapper(s).resolved)
workflowobj["$schemas"] = sch
return mapper
commit 964383400ae31d3e58a75fba5257095b8b4971dc
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Jan 6 16:39:19 2021 -0500
17224: Document LoginCluster configuration for login-sync
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/install/install-shell-server.html.textile.liquid b/doc/install/install-shell-server.html.textile.liquid
index 97854e524..43a4cdc72 100644
--- a/doc/install/install-shell-server.html.textile.liquid
+++ b/doc/install/install-shell-server.html.textile.liquid
@@ -24,7 +24,7 @@ Arvados support for shell nodes allows you to use Arvados permissions to grant L
A shell node runs the @arvados-login-sync@ service to manage user accounts, and typically has Arvados utilities and SDKs pre-installed. Users are allowed to log in and run arbitrary programs. For optimal performance, the Arvados shell server should be on the same LAN as the Arvados cluster.
-Because it _contains secrets_ shell nodes should *not* have a copy of the Arvados @config.yml at .
+Because Arvados @config.yml@ _contains secrets_ it should not *not* be present on shell nodes.
Shell nodes should be separate virtual machines from the VMs running other Arvados services. You may choose to grant root access to users so that they can customize the node, for example, installing new programs. This has security considerations depending on whether a shell node is single-user or multi-user.
@@ -89,6 +89,8 @@ If this is a multi-user shell node, then @ARVADOS_API_TOKEN@ should be an admini
Set @ARVADOS_VIRTUAL_MACHINE_UUID@ to the UUID from "Create record for VM":#vm-record
+h3. Standalone cluster
+
<notextile>
<pre>
<code>shellserver:# <span class="userinput">umask 0700; tee /etc/cron.d/arvados-login-sync <<EOF
@@ -100,6 +102,24 @@ EOF</span></code>
</pre>
</notextile>
+h3. Part of a LoginCLuster federation
+
+If this cluster is part of a "federation with centralized user management":../admin/federation.html#LoginCluster , the login sync script also needs to be given the host and user token for the login cluster.
+
+<notextile>
+<pre>
+<code>shellserver:# <span class="userinput">umask 0700; tee /etc/cron.d/arvados-login-sync <<EOF
+ARVADOS_API_HOST="<strong>ClusterID.example.com</strong>"
+ARVADOS_API_TOKEN="<strong>xxxxxxxxxxxxxxxxx</strong>"
+LOGINCLUSTER_ARVADOS_API_HOST="<strong>LoginClusterID.example.com</strong>"
+LOGINCLUSTER_ARVADOS_API_TOKEN="<strong>yyyyyyyyyyyyyyyyy</strong>"
+ARVADOS_VIRTUAL_MACHINE_UUID="<strong>zzzzz-2x53u-zzzzzzzzzzzzzzz</strong>"
+*/2 * * * * root arvados-login-sync
+EOF</span></code>
+</pre>
+</notextile>
+
+
h2(#confirm-working). Confirm working installation
A user should be able to log in to the shell server when the following conditions are satisfied:
commit c9877d5042490896035dab88134dd07421054e4f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Wed Dec 23 16:19:39 2020 -0500
17224: Provide alternate credentials to use for creating user tokens
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/services/login-sync/bin/arvados-login-sync b/services/login-sync/bin/arvados-login-sync
index 8162e22a2..a9bff0546 100755
--- a/services/login-sync/bin/arvados-login-sync
+++ b/services/login-sync/bin/arvados-login-sync
@@ -31,6 +31,9 @@ keys = ''
begin
arv = Arvados.new({ :suppress_ssl_warnings => false })
+ logincluster_arv = Arvados.new({ :api_host => (ENV['LOGINCLUSTER_ARVADOS_API_HOST'] || ENV['ARVADOS_API_HOST']),
+ :api_token => (ENV['LOGINCLUSTER_ARVADOS_API_TOKEN'] || ENV['ARVADOS_API_TOKEN']),
+ :suppress_ssl_warnings => false })
vm_uuid = ENV['ARVADOS_VIRTUAL_MACHINE_UUID']
@@ -190,7 +193,7 @@ begin
begin
if !File.exist?(tokenfile)
- user_token = arv.api_client_authorization.create(api_client_authorization: {owner_uuid: l[:user_uuid], api_client_id: 0})
+ user_token = logincluster_arv.api_client_authorization.create(api_client_authorization: {owner_uuid: l[:user_uuid], api_client_id: 0})
f = File.new(tokenfile, 'w')
f.write("ARVADOS_API_HOST=#{ENV['ARVADOS_API_HOST']}\n")
f.write("ARVADOS_API_TOKEN=v2/#{user_token[:uuid]}/#{user_token[:api_token]}\n")
commit cc12638657fb78d4b4f31c8d41d92d5198b3e2d8
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Sat Jan 2 11:59:57 2021 -0500
17224: Fix to have multiple clients connected to separate clusters
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/ruby/lib/arvados.rb b/sdk/ruby/lib/arvados.rb
index a89c21b01..d8a6be1b2 100644
--- a/sdk/ruby/lib/arvados.rb
+++ b/sdk/ruby/lib/arvados.rb
@@ -19,7 +19,6 @@ class Arvados
class TransactionFailedError < StandardError
end
- @@config = nil
@@debuglevel = 0
class << self
attr_accessor :debuglevel
@@ -31,12 +30,16 @@ class Arvados
@arvados_api_version = opts[:api_version] || 'v1'
- @arvados_api_host = opts[:api_host] ||
- config['ARVADOS_API_HOST'] or
- raise "#{$0}: no :api_host or ENV[ARVADOS_API_HOST] provided."
- @arvados_api_token = opts[:api_token] ||
- config['ARVADOS_API_TOKEN'] or
- raise "#{$0}: no :api_token or ENV[ARVADOS_API_TOKEN] provided."
+ @config = nil
+ [[:api_host, 'ARVADOS_API_HOST'],
+ [:api_token, 'ARVADOS_API_TOKEN']].each do |op, en|
+ if opts[op]
+ config[en] = opts[op]
+ end
+ if !config[en]
+ raise "#{$0}: no :#{op} or ENV[#{en}] provided."
+ end
+ end
if (opts[:suppress_ssl_warnings] or
%w(1 true yes).index(config['ARVADOS_API_HOST_INSECURE'].
@@ -91,17 +94,15 @@ class Arvados
# result looks like Arvados::A26949680::Job.
namespace_class.const_set classname, klass
- self.class.class_eval do
- define_method classname.underscore do
- klass
- end
+ self.define_singleton_method classname.underscore do
+ klass
end
end
end
def client
@client ||= Google::APIClient.
- new(:host => @arvados_api_host,
+ new(:host => config["ARVADOS_API_HOST"],
:application_name => @application_name,
:application_version => @application_version.to_s)
end
@@ -119,7 +120,7 @@ class Arvados
end
def config(config_file_path="~/.config/arvados/settings.conf")
- return @@config if @@config
+ return @config if @config
# Initialize config settings with environment variables.
config = {}
@@ -137,7 +138,7 @@ class Arvados
# Note: If we start using additional configuration settings from
# this file in the future, we might have to read the file anyway
# instead of returning here.
- return (@@config = config)
+ return (@config = config)
end
begin
@@ -164,7 +165,7 @@ class Arvados
debuglog "Ignoring error reading #{config_file_path}: #{e}", 0
end
- @@config = config
+ @config = config
end
class Model
@@ -202,7 +203,7 @@ class Arvados
:parameters => parameters,
:body_object => body,
:headers => {
- :authorization => 'OAuth2 '+arvados.config['ARVADOS_API_TOKEN']
+ :authorization => 'Bearer '+arvados.config['ARVADOS_API_TOKEN']
})
resp = JSON.parse result.body, :symbolize_names => true
if resp[:errors]
commit 7b433917b9476f43e894fbc522c17204a1a9061e
Author: Ward Vandewege <ward at curii.com>
Date: Wed Dec 23 11:58:03 2020 -0500
Fix arvados-src package version generation.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 0e74ac6f2..272c31e02 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -259,10 +259,8 @@ debug_echo -e "\nPython packages\n"
git checkout $DASHQ_UNLESS_DEBUG "$COMMIT_HASH"
echo "$COMMIT_HASH" >git-commit.version
- cd "$SRC_BUILD_DIR"
- PKG_VERSION=$(version_from_git)
cd $WORKSPACE/packages/$TARGET
- fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+ fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$arvados_src_version" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
rm -rf "$SRC_BUILD_DIR"
fi
commit 851f8f6d8a0e31be8a9544be95f8b5a391e75d3b
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Jan 25 15:26:54 2021 -0500
Add missing file closes #17281
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/user/getting_started/setup-cli.html.textile.liquid b/doc/user/getting_started/setup-cli.html.textile.liquid
new file mode 100644
index 000000000..46ea770ef
--- /dev/null
+++ b/doc/user/getting_started/setup-cli.html.textile.liquid
@@ -0,0 +1,20 @@
+---
+layout: default
+navsection: userguide
+title: Getting started at the command line
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Many operations in Arvados can be performed using either the web Workbench or through command line tools. Some operations can only be done using the command line.
+
+To use the command line tools, you can either log into an Arvados-managed VM instance where those tools are pre-installed, or install the Arvados tools on your own system.
+
+To log into an Arvados-managed VM, see instructions for "Webshell":{{site.baseurl}}/user/getting_started/vm-login-with-webshell.html or "Unix":{{site.baseurl}}/user/getting_started/ssh-access-unix.html or "Windows":{{site.baseurl}}/user/getting_started/ssh-access-windows.html .
+
+To install the Arvados tools on your own system, you should install the "Command line SDK":{{site.baseurl}}/sdk/cli/install.html (requires Ruby) and "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html (requires Python). You may also want to install "arvados-cwl-runner":{{site.baseurl}}/sdk/python/arvados-cwl-runner.html to submit workflows and "arvados-fuse":{{site.baseurl}}/sdk/python/arvados-fuse.html to mount keep as a filesystem.
+
+Once you are logged in or have command line tools installed, see "getting an API token":{{site.baseurl}}/user/reference/api-tokens.html and "check your environment":{{site.baseurl}}/user/getting_started/check-environment.html .
commit ca903a18d44b09b4388b51f49586a237c4037243
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jan 29 15:54:41 2021 -0500
17293: Update trash_at/deleted_at even on MarshalManifest error
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index c125b27a5..e05b352fa 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -1431,15 +1431,20 @@ func (runner *ContainerRunner) saveLogCollection(final bool) (response arvados.C
// Already finalized.
return
}
- mt, err := runner.LogCollection.MarshalManifest(".")
- if err != nil {
- err = fmt.Errorf("error creating log manifest: %v", err)
- return
- }
updates := arvadosclient.Dict{
- "name": "logs for " + runner.Container.UUID,
- "manifest_text": mt,
+ "name": "logs for " + runner.Container.UUID,
}
+ mt, err1 := runner.LogCollection.MarshalManifest(".")
+ if err1 == nil {
+ // Only send updated manifest text if there was no
+ // error.
+ updates["manifest_text"] = mt
+ }
+
+ // Even if flushing the manifest had an error, we still want
+ // to update the log record, if possible, to push the trash_at
+ // and delete_at times into the future. Details on bug
+ // #17293.
if final {
updates["is_trashed"] = true
} else {
@@ -1448,16 +1453,20 @@ func (runner *ContainerRunner) saveLogCollection(final bool) (response arvados.C
updates["delete_at"] = exp
}
reqBody := arvadosclient.Dict{"collection": updates}
+ var err2 error
if runner.logUUID == "" {
reqBody["ensure_unique_name"] = true
- err = runner.DispatcherArvClient.Create("collections", reqBody, &response)
+ err2 = runner.DispatcherArvClient.Create("collections", reqBody, &response)
} else {
- err = runner.DispatcherArvClient.Update("collections", runner.logUUID, reqBody, &response)
+ err2 = runner.DispatcherArvClient.Update("collections", runner.logUUID, reqBody, &response)
}
- if err != nil {
- return
+ if err2 == nil {
+ runner.logUUID = response.UUID
+ }
+
+ if err1 != nil || err2 != nil {
+ err = fmt.Errorf("error recording logs: %q, %q", err1, err2)
}
- runner.logUUID = response.UUID
return
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list