[ARVADOS] updated: 2d0d55e71d1bc158626c4bfe75141e4e30351a82
git at public.curoverse.com
git at public.curoverse.com
Wed Nov 12 14:03:30 EST 2014
Summary of changes:
.../app/controllers/application_controller.rb | 1 +
.../app/controllers/collections_controller.rb | 39 +--
apps/workbench/app/controllers/users_controller.rb | 6 +-
apps/workbench/app/models/arvados_api_client.rb | 7 +-
apps/workbench/app/models/arvados_base.rb | 4 +-
apps/workbench/app/models/arvados_resource_list.rb | 163 +++++++-----
.../app/views/application/_paging.html.erb | 22 +-
.../app/views/collections/_choose_rows.html.erb | 18 +-
.../app/views/collections/_index_tbody.html.erb | 2 +-
.../app/views/collections/_show_recent.html.erb | 4 +-
.../app/views/projects/_show_sharing.html.erb | 3 +-
.../views/users/_manage_virtual_machines.html.erb | 2 +
.../workbench/app/views/users/_show_admin.html.erb | 2 +-
.../test/unit/arvados_resource_list_test.rb | 97 +++++--
doc/_config.yml | 2 +
doc/sdk/cli/index.html.textile.liquid | 53 +---
doc/sdk/cli/install.html.textile.liquid | 42 +++
doc/sdk/cli/subcommands.html.textile.liquid | 247 ++++++++++++++++++
sdk/python/arvados/arvfile.py | 29 +++
sdk/python/arvados/collection.py | 111 ++++++--
sdk/python/arvados/keep.py | 80 ++++--
sdk/python/arvados/retry.py | 10 +-
sdk/python/arvados/stream.py | 121 ++++++---
sdk/python/arvados/util.py | 4 +-
sdk/python/setup.py | 1 +
sdk/python/tests/arvados_testutil.py | 42 +++
sdk/python/tests/test_collections.py | 287 +++++++++------------
sdk/python/tests/test_keep_client.py | 92 ++++++-
sdk/python/tests/test_retry.py | 4 +-
sdk/python/tests/test_stream.py | 174 ++++++++++++-
services/api/lib/load_param.rb | 7 +-
.../arvados/v1/groups_controller_test.rb | 67 ++---
services/nodemanager/arvnodeman/config.py | 3 +-
services/nodemanager/arvnodeman/daemon.py | 6 +-
services/nodemanager/arvnodeman/jobqueue.py | 10 +-
services/nodemanager/arvnodeman/launcher.py | 8 +-
services/nodemanager/doc/ec2.example.cfg | 4 +
services/nodemanager/doc/local.example.cfg | 1 +
services/nodemanager/tests/test_daemon.py | 13 +-
services/nodemanager/tests/test_jobqueue.py | 18 ++
40 files changed, 1254 insertions(+), 552 deletions(-)
create mode 100644 doc/sdk/cli/install.html.textile.liquid
create mode 100644 doc/sdk/cli/subcommands.html.textile.liquid
create mode 100644 sdk/python/arvados/arvfile.py
via 2d0d55e71d1bc158626c4bfe75141e4e30351a82 (commit)
via e158f485053be1e840073b321033d60d686a55a8 (commit)
via 5141c3ee23e89696773e227a93236ef2a51543c2 (commit)
via 45bfc1104dd30fb97a586de5ff96d6b739f7bb2b (commit)
via 6bcb827b1405ce8ca5d767108bc4356ecc2972f4 (commit)
via 093cebc3a4baed645fd5d4a5331c2ff837de8c77 (commit)
via 53381c90fd5ef26d3a0cc08c8d35318b0b92557a (commit)
via 91f0b180a8474321738577a12250b5a1b8f5bcbc (commit)
via a844af57de33b0094ce23c873d044f15de1ed138 (commit)
via d1cdd7eddf071e41643bc958b77d0056410d01ea (commit)
via 26aa2f1a1c8c9883beac1538c318279190f91c8a (commit)
via 7bf8f6c701e28e574c137b0c942522e8f8ee4d8c (commit)
via 7924077e3db2898de26c86599d9f311e02d6db46 (commit)
via c1d9150de2977f7eb85e7cb058d5c41ae6f06173 (commit)
via ee3e85858a60bffc2f132e2f95ba92d1deb78c2c (commit)
via c89f70e34e67642c5e594334a9149afe84490cf0 (commit)
via f34bf1aadbc6c940150518c8c4db38f7fed17282 (commit)
via 0dbc7a3469f687d1a4cef776a57e71e96561ad0d (commit)
via f4ceb0aab5df85254bf8828a74dddbac1cf169d2 (commit)
via 204419e149d2b6b19b3bc2965e1c524b7abd5881 (commit)
via e82cddfdbd763db27df889a6e316623df7b9c2a8 (commit)
via ec75fda0fc2c86a77d831dcd7962ece7a2d6ae6d (commit)
via d9d7af20beb9f9a09eb755e98d5e25f1a6e3e0e7 (commit)
via 7e8b0267597a52063395d3c258d4d4772d2e2240 (commit)
via ca136c579fc014b6428c18cc3a74a45550539543 (commit)
via a98086e04213ac68394e392761a2911b0e8a000c (commit)
via 69594694568da1abc2bd8e46134da81ec1f2a7c8 (commit)
via 0198b4937df2be79e41ea76ea6c31841441f4f2c (commit)
via 3ee90fd536fbfc9d8d91d5fac7c12d0ebe0df5ab (commit)
via fbd2326841005c1a288c65502dc038f26923ed65 (commit)
via 0f9bca457448372de1d15dcd9ed4548c324ff14f (commit)
via 3bbb988777079718338e3e6cb9c6c9b5399be800 (commit)
via 1fa1006e5b21cfc30e694890db969c20864af8c6 (commit)
via 83369774c123b427f519bd2a2229415fac383a6a (commit)
via 3993ba462c113090bc2db79db30838c017a5aa3b (commit)
via 2361a1b4d619a0f405f98c507cff6219b237bb2e (commit)
via 47530892a8a6b174786316c3881e22dc0864c859 (commit)
via c1f7fd8aa615d4a91f919ce18a6085ae766a5517 (commit)
from c7b7e4d44e9349c9104fbfcd33c5e4b97fe17a71 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2d0d55e71d1bc158626c4bfe75141e4e30351a82
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Nov 12 14:02:42 2014 -0500
2853: Remove historical artifact: check for None instead of catching an exception.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 23d8c20..793727b 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -294,9 +294,9 @@ class KeepClient(object):
return self.success_flag is not None
def last_status(self):
- try:
+ if self.last_result:
return self.last_result.status_code
- except AttributeError:
+ else:
return None
def get(self, locator, timeout=None):
commit e158f485053be1e840073b321033d60d686a55a8
Merge: c7b7e4d 5141c3e
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Nov 12 13:59:59 2014 -0500
2853: Merge branch 'master' into 2853-rendezvous
Conflicts:
sdk/python/arvados/keep.py
diff --cc doc/sdk/cli/index.html.textile.liquid
index 73acddc,6b0af21..08880b1
--- a/doc/sdk/cli/index.html.textile.liquid
+++ b/doc/sdk/cli/index.html.textile.liquid
@@@ -14,47 -12,7 +12,6 @@@ h3. Wrapper for API call
See the "command line interface":{{site.baseurl}}/user/reference/sdk-cli.html page in the user guide.
- h3. Installation
-
- If you are logged in to an Arvados VM, the @arv@ should be installed.
-
- To use @arv@ elsewhere, you can either install the @arvados-cli@ gem via RubyGems or build and install the package using the arvados source tree.
-
- h4. Prerequisites: Ruby >= 2.0.0 and curl libraries
-
- You can use "RVM":http://rvm.io/rvm/install to install and manage Ruby versions.
-
- <notextile>
- <pre>
- $ <code class="userinput">sudo apt-get install curl</code>
- $ <code class="userinput">sudo sh -c 'curl -sSL https://get.rvm.io | bash -s stable'</code>
- $ <code class="userinput">source /etc/profile.d/rvm.sh</code>
- </pre>
- </notextile>
-
- Install curl libraries with your system's package manager. For example, with Debian or Ubuntu:
-
- <notextile>
- <pre>
- $ <code class="userinput">sudo apt-get install libcurl3 libcurl3-gnutls libcurl4-openssl-dev</code>
- </pre>
- </notextile>
-
- h4. Option 1: install with RubyGems
-
- <notextile>
- <pre>
- $ <code class="userinput">sudo gem install arvados-cli</code>
- </pre>
- </notextile>
-
- h4. Option 2: build and install from source
+ h3. Subcommands
- <notextile>
- <pre>
- $ <code class="userinput">git clone https://github.com/curoverse/arvados.git</code>
- $ <code class="userinput">cd arvados/sdk/cli</code>
- $ <code class="userinput">gem build arvados-cli.gemspec</code>
- $ <code class="userinput">sudo gem install arvados-cli-*.gem</code>
- </pre>
- </notextile>
+ See the "arv subcommands":{{site.baseurl}}/sdk/cli/subcommands.html page.
-
diff --cc doc/sdk/cli/subcommands.html.textile.liquid
index 0000000,07be3cf..99f35ef
mode 000000,100644..100644
--- a/doc/sdk/cli/subcommands.html.textile.liquid
+++ b/doc/sdk/cli/subcommands.html.textile.liquid
@@@ -1,0 -1,248 +1,247 @@@
+ ---
+ layout: default
+ navsection: sdk
+ navmenu: CLI
+ title: "arv subcommands"
+
+ ...
+
+ The @arv@ CLI tool provides access to a number of subcommands which are described on this page.
+
+ h3(#arv-create). arv create
+
+ @arv create@ can be used to create Arvados objects from the command line. Arv create opens up the editor of your choice (set the EDITOR environment variable) and allows you to type or paste a json or yaml description. When saved the object will be created on the API server, if it passes validation.
+
+ <notextile>
+ <pre>
+ $ <code class="userinput">arv create --help</code>
+ Options:
+ --project-uuid, -p <s>: Project uuid in which to create the object
+ --help, -h: Show this message
+ </pre>
+ </notextile>
+
+ h3(#arv-edit). arv edit
+
+ @arv edit@ can be used to edit Arvados objects from the command line. Arv edit opens up the editor of your choice (set the EDITOR environment variable) with the json or yaml description of the object. Saving the file will update the Arvados object on the API server, if it passes validation.
+
+ <notextile>
+ <pre>
+ $ <code class="userinput">arv edit --help</code>
+ Arvados command line client
+ Usage: arv edit [uuid] [fields...]
+
+ Fetch the specified Arvados object, select the specified fields,
+ open an interactive text editor on a text representation (json or
+ yaml, use --format) and then update the object. Will use 'nano'
+ by default, customize with the EDITOR or VISUAL environment variable.
+ </pre>
+ </notextile>
+
+ h3(#arv-copy). arv copy
+
+ @arv copy@ can be used to copy a pipeline instance, template or collection from one Arvados instance to another. It takes care of copying the object and all its dependencies.
+
+ <notextile>
+ <pre>
+ $ <code class="userinput">arv copy --help</code>
+ usage: arv-copy [-h] [-v] [--progress] [--no-progress] [-f] --src
+ SOURCE_ARVADOS --dst DESTINATION_ARVADOS [--recursive]
+ [--no-recursive] [--dst-git-repo DST_GIT_REPO]
+ [--project-uuid PROJECT_UUID] [--retries RETRIES]
+ object_uuid
+
+ Copy a pipeline instance, template or collection from one Arvados instance to
+ another.
+
+ positional arguments:
+ object_uuid The UUID of the object to be copied.
+
+ optional arguments:
+ -h, --help show this help message and exit
+ -v, --verbose Verbose output.
+ --progress Report progress on copying collections. (default)
+ --no-progress Do not report progress on copying collections.
+ -f, --force Perform copy even if the object appears to exist at
+ the remote destination.
+ --src SOURCE_ARVADOS The name of the source Arvados instance (required).
+ May be either a pathname to a config file, or the
+ basename of a file in
+ $HOME/.config/arvados/instance_name.conf.
+ --dst DESTINATION_ARVADOS
+ The name of the destination Arvados instance
+ (required). May be either a pathname to a config file,
+ or the basename of a file in
+ $HOME/.config/arvados/instance_name.conf.
+ --recursive Recursively copy any dependencies for this object.
+ (default)
+ --no-recursive Do not copy any dependencies. NOTE: if this option is
+ given, the copied object will need to be updated
+ manually in order to be functional.
+ --dst-git-repo DST_GIT_REPO
+ The name of the destination git repository. Required
+ when copying a pipeline recursively.
+ --project-uuid PROJECT_UUID
+ The UUID of the project at the destination to which
+ the pipeline should be copied.
+ --retries RETRIES Maximum number of times to retry server requests that
+ encounter temporary failures (e.g., server down).
+ Default 3.
+ </pre>
+ </notextile>
+
+ h3(#arv-tag). arv tag
+
+ @arv tag@ is used to tag Arvados objects.
+
+ <notextile>
+ <pre>
+ $ <code class="userinput">arv tag --help</code>
+ arvados cli client
+ --dry-run, -n: Don't actually do anything
+ --verbose, -v: Print some things on stderr
+ --uuid, -u: Return the UUIDs of the objects in the response, one per
+ line (default)
+ --json, -j: Return the entire response received from the API server, as
+ a JSON object
+ --human, -h: Return the response received from the API server, as a JSON
+ object with whitespace added for human consumption
+ --pretty, -p: Synonym of --human
+ --yaml, -y: Return the response received from the API server, in YAML
+ format
+ --help, -e: Show this message
+ </pre>
+ </notextile>
+
+
+ h3(#arv-ws). arv ws
+
+ @arv ws@ provides access to the websockets event stream.
+
+ <notextile>
+ <pre>
+ $ <code class="userinput">arv ws --help</code>
+ usage: arv-ws [-h] [-u UUID] [-f FILTERS]
+ [--poll-interval POLL_INTERVAL | --no-poll]
+ [-p PIPELINE | -j JOB]
+
+ optional arguments:
+ -h, --help show this help message and exit
+ -u UUID, --uuid UUID Filter events on object_uuid
+ -f FILTERS, --filters FILTERS
+ Arvados query filter to apply to log events (JSON
+ encoded)
+ --poll-interval POLL_INTERVAL
+ If websockets is not available, specify the polling
+ interval, default is every 15 seconds
+ --no-poll Do not poll if websockets are not available, just fail
+ -p PIPELINE, --pipeline PIPELINE
+ Supply pipeline uuid, print log output from pipeline
+ and its jobs
+ -j JOB, --job JOB Supply job uuid, print log output from jobs
+ </pre>
+ </notextile>
+
+ h3(#arv-keep). arv keep
+
+ @arv keep@ provides access to the Keep storage service.
+
+ <notextile>
+ <pre>
+ $ <code class="userinput">arv keep --help</code>
+ Usage: arv keep [method] [--parameters]
+ Use 'arv keep [method] --help' to get more information about specific methods.
+
+ Available methods: ls, get, put, less, check, docker
+ </pre>
+ </notextile>
+
+ h3(#arv-pipeline-run). arv pipeline run
+
+ @arv pipeline run@ can be used to start a pipeline run from the command line.
+
+ The User Guide has a page with a bit more information on "using arv pipeline run":{{site.baseurl}}/user/topics/running-pipeline-command-line.html.
+
+ <notextile>
+ <pre>
+ $ <code class="userinput">arv pipeline run --help</code>
+ Options:
+ --dry-run, -n: Do not start any new jobs or wait for existing jobs to
+ finish. Just find out whether jobs are finished,
+ queued, or running for each component.
+ --status-text <s>: Store plain text status in given file. (Default:
+ /dev/stdout)
+ --status-json <s>: Store json-formatted pipeline in given file. (Default:
+ /dev/null)
+ --no-wait: Do not wait for jobs to finish. Just look up status,
+ submit new jobs if needed, and exit.
+ --no-reuse: Do not reuse existing jobs to satisfy pipeline
+ components. Submit a new job for every component.
+ --debug, -d: Print extra debugging information on stderr.
+ --debug-level <i>: Set debug verbosity level.
+ --template <s>: UUID of pipeline template, or path to local pipeline
+ template file.
+ --instance <s>: UUID of pipeline instance.
+ --submit: Submit the pipeline instance to the server, and exit.
+ Let the Crunch dispatch service satisfy the components
+ by finding/running jobs.
+ --run-pipeline-here: Manage the pipeline instance in-process. Submit jobs
+ to Crunch as needed. Do not exit until the pipeline
+ finishes (or fails).
+ --run-jobs-here: Run jobs in the local terminal session instead of
+ submitting them to Crunch. Implies
+ --run-pipeline-here. Note: this results in a
+ significantly different job execution environment, and
+ some Crunch features are not supported. It can be
+ necessary to modify a pipeline in order to make it run
+ this way.
+ --run-here: Synonym for --run-jobs-here.
+ --description <s>: Description for the pipeline instance.
+ --version, -v: Print version and exit
+ --help, -h: Show this message
+ </pre>
+ </notextile>
+
+ h3(#arv-run). arv run
+
+ The @arv-run@ command creates Arvados pipelines at the command line that fan out to multiple concurrent tasks across Arvado compute nodes.
+
+ The User Guide has a page on "using arv-run":{{site.baseurl}}/user/topics/arv-run.html.
+
+ <notextile>
+ <pre>
+ $ <code class="userinput">arv run --help</code>
+ usage: arv-run [-h] [--retries RETRIES] [--dry-run] [--local]
+ [--docker-image DOCKER_IMAGE] [--ignore-rcode] [--no-reuse]
+ [--no-wait] [--project-uuid PROJECT_UUID] [--git-dir GIT_DIR]
+ [--repository REPOSITORY] [--script-version SCRIPT_VERSION]
+ ...
+
+ positional arguments:
+ args
+
+ optional arguments:
+ -h, --help show this help message and exit
+ --retries RETRIES Maximum number of times to retry server requests that
+ encounter temporary failures (e.g., server down).
+ Default 3.
+ --dry-run Print out the pipeline that would be submitted and
+ exit
+ --local Run locally using arv-run-pipeline-instance
+ --docker-image DOCKER_IMAGE
+ Docker image to use, default arvados/jobs
+ --ignore-rcode Commands that return non-zero return codes should not
+ be considered failed.
+ --no-reuse Do not reuse past jobs.
+ --no-wait Do not wait and display logs after submitting command,
+ just exit.
+ --project-uuid PROJECT_UUID
+ Parent project of the pipeline
+ --git-dir GIT_DIR Git repository passed to arv-crunch-job when using
+ --local
+ --repository REPOSITORY
+ repository field of component, default 'arvados'
+ --script-version SCRIPT_VERSION
+ script_version field of component, default 'master'
+ </pre>
+ </notextile>
-
diff --cc sdk/python/arvados/keep.py
index 4c288f9,1fd8fa5..23d8c20
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@@ -21,7 -19,11 +19,8 @@@ import time
import datetime
import ssl
import socket
+ import requests
-_logger = logging.getLogger('arvados.keep')
-global_client_object = None
-
import arvados
import arvados.config as config
import arvados.errors
@@@ -461,13 -476,22 +479,22 @@@ class KeepClient(object)
api_client = arvados.api('v1')
self.api_client = api_client
self.api_token = api_client.api_token
- self.service_roots = None
+ self._keep_services = None
self.using_proxy = None
- self.static_service_roots = False
+ self._static_services_list = False
+ def current_timeout(self):
+ """Return the appropriate timeout to use for this client: the proxy
+ timeout setting if the backend service is currently a proxy,
+ the regular timeout setting otherwise.
+ """
+ # TODO(twp): the timeout should be a property of a
+ # KeepService, not a KeepClient. See #4488.
+ return self.proxy_timeout if self.using_proxy else self.timeout
+
- def build_service_roots(self, force_rebuild=False):
- if (self.static_service_roots or
- (self.service_roots and not force_rebuild)):
+ def build_services_list(self, force_rebuild=False):
+ if (self._static_services_list or
+ (self._keep_services and not force_rebuild)):
return
with self.lock:
try:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list