[ARVADOS] updated: fbbb1beaca00df2d1483087451b81ed3c40953a9
git at public.curoverse.com
git at public.curoverse.com
Thu Mar 27 21:03:05 EDT 2014
Summary of changes:
apps/workbench/README.textile | 2 +
apps/workbench/app/controllers/users_controller.rb | 10 +-
apps/workbench/app/helpers/provenance_helper.rb | 30 +-
apps/workbench/app/models/collection.rb | 7 +
apps/workbench/app/models/user.rb | 7 +
.../app/views/layouts/application.html.erb | 4 +-
.../pipeline_instances/_show_components.html.erb | 18 +-
.../workbench/app/views/users/_show_admin.html.erb | 6 +
apps/workbench/app/views/users/_tables.html.erb | 10 +-
apps/workbench/config/routes.rb | 1 +
apps/workbench/doc/README_FOR_APP | 2 -
.../integration/api_client_authorizations_test.rb | 11 -
apps/workbench/test/integration/logins_test.rb | 2 +-
apps/workbench/test/integration/smoke_test.rb | 39 +
apps/workbench/test/integration_helper.rb | 15 +-
apps/workbench/test/unit/collection_test.rb | 15 +-
doc/README.textile | 6 +
doc/_config.yml | 7 +
doc/_includes/_webring.liquid | 6 +-
doc/api/schema/ApiClient.html.textile.liquid | 4 +-
.../ApiClientAuthorization.html.textile.liquid | 2 +-
doc/api/schema/AuthorizedKey.html.textile.liquid | 5 +-
doc/api/schema/Collection.html.textile.liquid | 6 +-
doc/api/schema/Commit.html.textile.liquid | 4 +-
doc/api/schema/CommitAncestor.html.textile.liquid | 4 +-
doc/api/schema/Group.html.textile.liquid | 4 +-
doc/api/schema/Human.html.textile.liquid | 4 +-
doc/api/schema/Job.html.textile.liquid | 6 +-
doc/api/schema/JobTask.html.textile.liquid | 4 +-
doc/api/schema/KeepDisk.html.textile.liquid | 4 +-
doc/api/schema/Link.html.textile.liquid | 4 +-
doc/api/schema/Log.html.textile.liquid | 4 +-
doc/api/schema/Node.html.textile.liquid | 4 +-
.../schema/PipelineInstance.html.textile.liquid | 4 +-
.../schema/PipelineTemplate.html.textile.liquid | 4 +-
doc/api/schema/Repository.html.textile.liquid | 4 +-
doc/api/schema/Specimen.html.textile.liquid | 4 +-
doc/api/schema/Trait.html.textile.liquid | 4 +-
doc/api/schema/User.html.textile.liquid | 4 +-
doc/api/schema/VirtualMachine.html.textile.liquid | 4 +-
doc/gen_api_schema_docs.py | 3 +-
doc/images/dax-reading-book.png | Bin 0 -> 211000 bytes
doc/index.html.liquid | 4 +-
doc/sdk/cli/index.html.textile.liquid | 60 ++
doc/sdk/index.html.textile.liquid | 10 +-
doc/sdk/perl/index.html.textile.liquid | 99 +++
doc/sdk/ruby/index.html.textile.liquid | 125 ++++
doc/user/copying/agpl-3.0.html | 2 +-
.../job-and-pipeline-reference.html.textile.liquid | 223 ++++++
doc/user/topics/keep.html.textile.liquid | 2 +-
...nning-pipeline-command-line.html.textile.liquid | 7 +-
...rial-gatk-variantfiltration.html.textile.liquid | 1 +
doc/user/topics/tutorial-job1.html.textile.liquid | 15 +-
.../topics/tutorial-parallel.html.textile.liquid | 3 +-
.../running-external-program.html.textile.liquid | 5 +-
.../tutorial-firstscript.html.textile.liquid | 10 +-
.../tutorial-new-pipeline.html.textile.liquid | 9 +-
...tutorial-pipeline-workbench.html.textile.liquid | 4 +-
sdk/cli/bin/arv | 21 +-
sdk/cli/bin/arv-run-pipeline-instance | 104 +---
sdk/cli/bin/crunch-job | 2 +-
.../app/controllers/arvados/v1/jobs_controller.rb | 73 ++-
.../controllers/arvados/v1/schema_controller.rb | 25 +-
.../app/controllers/arvados/v1/users_controller.rb | 59 ++-
services/api/app/models/commit.rb | 122 +++-
services/api/app/models/job.rb | 9 +-
services/api/app/models/user.rb | 194 +++++
services/api/config/application.default.yml | 12 +-
services/api/config/application.yml.example | 7 +-
services/api/config/environment.rb | 6 +
services/api/config/routes.rb | 5 +-
...317135600_add_nondeterministic_column_to_job.rb | 9 +
...0547_separate_repository_from_script_version.rb | 31 +
.../20140321191343_add_repository_column_to_job.rb | 9 +
services/api/db/schema.rb | 22 +-
services/api/script/crunch-dispatch.rb | 55 +-
services/api/script/rails | 4 +-
services/api/script/setup-new-user.rb | 66 ++
services/api/test/fixtures/collections.yml | 2 +-
services/api/test/fixtures/jobs.yml | 21 +
services/api/test/fixtures/links.yml | 23 +-
services/api/test/fixtures/repositories.yml | 4 +
.../arvados/v1/commits_controller_test.rb | 86 +++
.../api/test/functional/arvados/v1/git_setup.rb | 27 +
.../arvados/v1/job_reuse_controller_test.rb | 167 +++++
.../functional/arvados/v1/jobs_controller_test.rb | 7 +
.../arvados/v1/schema_controller_test.rb | 2 +-
.../functional/arvados/v1/users_controller_test.rb | 758 ++++++++++++++++++++
services/api/test/test.git.tar | Bin 0 -> 112640 bytes
services/api/test/unit/user_test.rb | 280 +++++++-
services/keep/keep.go | 137 ++++
services/keep/keep_test.go | 125 ++++
92 files changed, 2979 insertions(+), 362 deletions(-)
delete mode 100644 apps/workbench/doc/README_FOR_APP
delete mode 100644 apps/workbench/test/integration/api_client_authorizations_test.rb
create mode 100644 apps/workbench/test/integration/smoke_test.rb
create mode 100644 doc/images/dax-reading-book.png
create mode 100644 doc/sdk/cli/index.html.textile.liquid
create mode 100644 doc/sdk/perl/index.html.textile.liquid
create mode 100644 doc/sdk/ruby/index.html.textile.liquid
create mode 100644 doc/user/reference/job-and-pipeline-reference.html.textile.liquid
create mode 100644 services/api/db/migrate/20140317135600_add_nondeterministic_column_to_job.rb
create mode 100644 services/api/db/migrate/20140319160547_separate_repository_from_script_version.rb
create mode 100644 services/api/db/migrate/20140321191343_add_repository_column_to_job.rb
create mode 100755 services/api/script/setup-new-user.rb
create mode 100644 services/api/test/fixtures/repositories.yml
create mode 100644 services/api/test/functional/arvados/v1/commits_controller_test.rb
create mode 100644 services/api/test/functional/arvados/v1/git_setup.rb
create mode 100644 services/api/test/functional/arvados/v1/job_reuse_controller_test.rb
create mode 100644 services/api/test/test.git.tar
create mode 100644 services/keep/keep.go
create mode 100644 services/keep/keep_test.go
via fbbb1beaca00df2d1483087451b81ed3c40953a9 (commit)
via 6232434d6b86192b0531bacd7e4f0d5ae589f368 (commit)
via b0af0440838407be80c6208b15f969bc2942008e (commit)
via 736c7c62cb2bd45637a54ac8937315ba4b64ed37 (commit)
via 0136e134c210fe60e6a1e0eee5e2f77d8bc939dc (commit)
via 3262e46dd0fff9d6289f8d91207a585df9de7074 (commit)
via 960e0078e260149cb82c27583bafa2aad6e995b7 (commit)
via 08a7509697639aa6fcbd7a16d00788db89dfcddf (commit)
via 96b6172daeeb0c4ca6ee9f3f72bccbcedc3b95d2 (commit)
via 5390a8a8578538ad3b2bf0e7e541485c342acbea (commit)
via 8a265d208908cd8fbfa808add745c3e842557dc0 (commit)
via 1158c797ee6479ec6d6b90b0512fbfa38da89a72 (commit)
via 06d0c1434c815ff2f00e5723f902fdf11b51ab14 (commit)
via 9c78924f523b5420a21fe5c7eba0f913de4c4d69 (commit)
via c0b66870cafce4d75625e16b5b36bed8b58d7c8e (commit)
via fbb4e2d894e5c6043e778991002629fbfdd08956 (commit)
via e40fdedbc285a0fa6af16e41b5b4f72e46e9987d (commit)
via 8d003a7abe0b4d633c6c125dabbf011394fde2e7 (commit)
via 31c6426b70e2b277087188dad2b9b346c904f30b (commit)
via ee86983a93c335413c508c5717a8707bbced93e6 (commit)
via 2c81dc8f676cad13b61c47b89fffd3dea93055fb (commit)
via 392c382ddaf8ea4c0c4b6655c7f508de73274d12 (commit)
via e423bbb1f70a3244af625165357acea8afbbfc85 (commit)
via 7a537556b3503bf957ad3da2bf27885518a33230 (commit)
via 1013c7cee5478538901fd9ba5fc0fb2ce30f6422 (commit)
via 231b9f25a701e9bdbb69b61135d4856647dcce55 (commit)
via b836fb6c2d2fb439787b68ecca278e1f8c868d50 (commit)
via 8e34082da3e9952c18e8bd9c0afac2516dea1287 (commit)
via 04e3622dad1430dfae82c8e7130b2fb9f4610cf7 (commit)
via 2a3f8231a1682e7bac956e4079ff19ccc8b133ec (commit)
via 10184eeb601e432f8e81aea00cbc2375f79bcf45 (commit)
via 73b1451ebad641dec4617292e98de61425b4c285 (commit)
via 52cbd3bc6d8b7b2fe0735cb8ccd6e57c89c8d797 (commit)
via c54e3ca6bf62f1f4e41af25e920b49ee33c97805 (commit)
via 402461a12dbff7d0ec00c8446b925b021aabc2ed (commit)
via 7c04fe3e0e93d22b9c884934dbbd80fed6072ee3 (commit)
via df4897f1390e58825dd8afcc053955c6d3894169 (commit)
via 921c13a95c2d13c94de74601e5bc26e5b0c71c31 (commit)
via 4e26baca96b4cb9e2145475bf31f18a0ffad2c26 (commit)
via e23961a316dc64068a6cc9b799d618f021f85ddb (commit)
via 56c13e4c3af29ec274a8c3c5564ff3dbec2505fc (commit)
via e102a3b3100577137def302ecd0f810bc46b808a (commit)
via 0ce7d041f53bea47f98f509ccf7c196b8ad393b0 (commit)
via eff1d4ece26ce459c1f19b202a2048b2f663bddc (commit)
via 23a65392c4dd88f28cbf94ac451bd1f1d9cd0be4 (commit)
via b98ffafdc9e1b5904b200013ebe106959334c0c3 (commit)
via 51a07a4abf5b21fb983e5df39dc2ea9ae58d0469 (commit)
via 4995783a3270e2f6d2d3b5226238fbbccf2864c1 (commit)
via 64cdcaf6b38437c459bce30445e547c274e3f99e (commit)
via 031ad8460650129f8bed088e801fb0762d4d29b3 (commit)
via b65cb64f0d7865266bd8674e681cd0f48fd476c7 (commit)
via b9e9a418b7db148352bfc6b9f616c37163f38c32 (commit)
via 9f5d16ade2d95497ed9610eefd03d76fb15eb747 (commit)
via ee21f45e958de5ef39970981ead6416e3790cd1a (commit)
via 03ab8baef455f5eb9cde83c2b05d82b42c46e216 (commit)
via 2013479619e9aa838037262564ac4f265f786ad0 (commit)
via fad09fd18d6c364f358f3d7c5782b4d0360c68ab (commit)
via 070332d12b114e0536ccc5025743bea854bd4c9e (commit)
via 5991a5bb4a4ccf74b763570941458dfee640f6d1 (commit)
via 31bb970d362490abc5238e895e04ab41d828e64c (commit)
via 48b87a694a78f09e8e7c6d05212f1e21aa2423d3 (commit)
via de10e681e82e0cb987ffb95109a6c09a13f49944 (commit)
via 91526e84bf213e08b602fdbd4e7b69a18d401483 (commit)
via a248323fdb1bfa315a3dc9af0a2362aa5b0a0f46 (commit)
via f75abd5da51c048fd26e7076a8a8fe96f6f5e0a2 (commit)
via 67a54670123669370bb4e664c7e40c71f7539b5f (commit)
via 0e560c4a9b3f1d46edcbd3fbc595beffc6efca47 (commit)
via 87501e7bf8d4f2303cc25a00e93476784911438b (commit)
via 2bfc24e6eb5cbd51bbb2717cd70b02a25b008ae5 (commit)
via fefc1d78df0a65c949099f98062877f93dfbec8a (commit)
via dff5bfa5afbdd49bf0003b75583b7120f21b7df7 (commit)
via 83fa2045a190732f8dd9bed3e5c6071ce66da07f (commit)
via 8c021e4b54a8179e86dd887e1eb8db9ef4dd9672 (commit)
via 6a18fc0d6f3f9859e905d4762796d813a157605b (commit)
via 35e245f09cfc5ace6259a671a1f12fd508750b13 (commit)
via 6fc29fa2b44072bc9a1cd015282dde30a0ee72c4 (commit)
via ee1abbc5a99dc2fa67de7713107ccd5473bd94a5 (commit)
via 1896a42e2da493b024dce40266a4814883c08003 (commit)
via 971452bda26fc20bfff3c07f6203d61fd4ca0df8 (commit)
via bf53b14daa894c77ea2b9413e9a574897446a2b7 (commit)
via 178545086f0752789ca79d212e92f8c45030ead9 (commit)
via 65a085e1a2812e48a6f4b21d5229430549fb8791 (commit)
via 2d2e5e107fc258c2d6c864f61cb0b03b901c2088 (commit)
via e9a25bfe507179a20c4bb8a994ad394759b5239f (commit)
via d51fa3200eef8da2798ab7d29e0e6180d3da71f7 (commit)
via eb0fdee4d4fdb516dd57998ae6a34840354e6e77 (commit)
via e8167f6b3a83525a07c79e03917bad570a6c5c26 (commit)
via 33166913f14878fd317b9a07e038f8d294453b94 (commit)
via 24742cd39eff59df5d4ac17d7d39c437c79a456b (commit)
via 71d1a30b467a43ef312fd7e81d795864799415b2 (commit)
via 994fa8b63a19f2253b6027d64f7ea72c43ea1192 (commit)
via 3e130a782321de2a0343fcec8a3365442ecde431 (commit)
via d65964ad6063b6cd5420004a5cbdc58fe2d0b194 (commit)
via c09ec8260b02c4767c82bcb5b346899193329828 (commit)
via 03aadd3864de6d5687e1e6c71815fafc4ec030af (commit)
via 6849ec08498a596d0604c2876dcd605ce6c6aaeb (commit)
via 4ad105bd22c9141e5f88139ad3bf334a52f7a5d1 (commit)
via 4d3a1c5346c99da360af58f1932b8f7a2ba87723 (commit)
via 5b039149732231ec9bd53fd8b5b005babf9eb82e (commit)
via 296a8cba6b65654156d3ff3abc07522a4824935e (commit)
via 127f1e032068d83365d783e8fe8cad5f0117638a (commit)
via 1f5561a43693243ad3b6737dcd1020cbe0cc809f (commit)
via 00a420599077a5063aba449fe9986a765db6bdb3 (commit)
via 24e20b5eb6f6e021b7f6789ce6e6b4c5a8c678be (commit)
via 620bcbd5734e860a1e67bb85e4899233bc4a6b28 (commit)
via 375c23fa047e5fcd09e71f5c473dbd05cbe14891 (commit)
via 3de8b7c50ba3d254acb888579f16de44ce693e36 (commit)
via e5804f99569634967d4affe322a382034d81cfa8 (commit)
via 23a7957589e03db8576c6547d3204477c2b0e7f3 (commit)
via 0496ff36445af8eae98914b91869ac52d951148e (commit)
via 86c192b3ae3614f69305cfd60664c9b720b84692 (commit)
via 7112a411de36be33f6857aea3bc750a87b139854 (commit)
via 3e5b4f4602903d00650aa7d57a6934617468dd9f (commit)
via 1b6d98d5000a0c21a5e06a5195cba70d0c2d4dde (commit)
via 87f67abc3c73634afbaf7ebad1445c1f234dc5d4 (commit)
via 240c16d4be50f3cb73d5d573421430efb7ece00f (commit)
via 9c3904d27dc7d6b6aa5834ff0f5815a8b3685e99 (commit)
via 4b57d5f64a980777888fa49657d6fe0617514218 (commit)
via 851f7d9cc9d3f6e4843a8fbd507338b1cc0e9f2c (commit)
via 107377eaf36cba7130e7bc9f42111147049cbd0b (commit)
via 8b7a06f6d4f3f40fc0b2a2d5debf4b4553cc4ba0 (commit)
via 9b10524c349d0c28a6dcf33ad0a473c3eb2d2b2f (commit)
via fe4b3736c374e703d53ac3c444da7eb49a54a6d6 (commit)
via dc01c0405fbc4c17e3fc947dd154e1adb915a790 (commit)
via 5351e44b4947d93c1027c70afde0b2fda150abb2 (commit)
via ac4facb25d9c5828c247ada1fa7618c5a2e8b2d8 (commit)
via dcbb5acc0f9767c6b358e5f5d6055b78290430ab (commit)
via daf4ff30727f49ee3606fe8adab3b67f35e00beb (commit)
via cbfc8eea7f3fc96f478530c77441b7175a043a17 (commit)
via 8993f82f9148553603d9d6ce5f638a253f7fce70 (commit)
via 501db79a2d00dcd79b3cf3757f6a93ef6c1c1696 (commit)
via 22b6371cb0ff61b9dd19cef581ec1bdaae1b0bd9 (commit)
from c95bf51af6261d2707421305342043f499b69f67 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit fbbb1beaca00df2d1483087451b81ed3c40953a9
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Mar 27 20:51:07 2014 -0400
Move "empty blob" magic number into a Collection class method, remove
some old unused code and commented-out debug statements.
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index 828fbfe..16f68dc 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -17,11 +17,6 @@ module ProvenanceHelper
else
return m[1]
end
- # Collection.where(uuid: ['contains', m[1]]).each do |u|
- # puts "fixup #{uuid} to #{u.uuid}"
- # return u.uuid
- # end
- #end
else
nil
end
@@ -52,15 +47,11 @@ module ProvenanceHelper
#"\"#{uuid}\" [label=\"#{rsc}\\n#{uuid}\",href=\"#{href}\"];\n"
if rsc == Collection
- #puts uuid
- if uuid == :"d41d8cd98f00b204e9800998ecf8427e+0"
+ if Collection.is_empty_blob_locator? uuid.to_s
# special case
- #puts "empty!"
return "\"#{uuid}\" [label=\"(empty collection)\"];\n"
end
- #puts "#{uuid.class} #{@pdata[uuid]}"
if @pdata[uuid]
- #puts @pdata[uuid]
if @pdata[uuid][:name]
return "\"#{uuid}\" [label=\"#{@pdata[uuid][:name]}\",href=\"#{href}\",shape=oval,#{bgcolor}];\n"
else
@@ -82,7 +73,6 @@ module ProvenanceHelper
if i < files.length
label += "\\n⋮"
end
- #puts "#{uuid} #{label} #{files}"
extra_s = @node_extra[uuid].andand.map { |k,v|
"#{k}=\"#{v}\""
}.andand.join ","
@@ -153,20 +143,16 @@ module ProvenanceHelper
end
unless node == ""
node += "']"
- #puts node
- #id = "#{job[:uuid]}_#{prefix}"
gr += "\"#{node}\" [label=\"#{node}\"];\n"
gr += edge(job_uuid(job), node, {:label => prefix})
end
when String
return '' if sp.empty?
m = GenerateGraph::collection_uuid(sp)
- #puts "#{m} pdata is #{@pdata[m.intern]}"
if m and (@pdata[m.intern] or (not @opts[:pdata_only]))
gr += edge(job_uuid(job), m, {:label => prefix})
gr += generate_provenance_edges(m)
elsif @opts[:all_script_parameters]
- #id = "#{job[:uuid]}_#{prefix}"
gr += "\"#{sp}\" [label=\"#{sp}\"];\n"
gr += edge(job_uuid(job), sp, {:label => prefix})
end
@@ -182,8 +168,6 @@ module ProvenanceHelper
uuid = uuid.intern if uuid
if (not uuid) or uuid.empty? or @visited[uuid]
-
- #puts "already @visited #{uuid}"
return ""
end
@@ -193,13 +177,9 @@ module ProvenanceHelper
@visited[uuid] = true
end
- #puts "visiting #{uuid.inspect}"
-
if m
# uuid is a collection
- if uuid != :"d41d8cd98f00b204e9800998ecf8427e+0"
- # not the empty collection
-
+ if not Collection.is_empty_blob_locator? uuid.to_s
@pdata.each do |k, job|
if job[:output] == uuid.to_s
extra = { label: 'output' }
@@ -251,8 +231,6 @@ module ProvenanceHelper
end
end
- #puts "finished #{uuid}"
-
gr
end
@@ -308,8 +286,6 @@ edge [fontsize=10];
gr += "edge [dir=back];"
end
- #puts "@pdata is #{pdata}"
-
g = GenerateGraph.new(pdata, opts)
pdata.each do |k, v|
@@ -321,8 +297,6 @@ edge [fontsize=10];
gr += "}"
svg = ""
- #puts gr
-
require 'open3'
Open3.popen2("dot", "-Tsvg") do |stdin, stdout, wait_thr|
diff --git a/apps/workbench/app/models/collection.rb b/apps/workbench/app/models/collection.rb
index 6bc55bd..5460e9a 100644
--- a/apps/workbench/app/models/collection.rb
+++ b/apps/workbench/app/models/collection.rb
@@ -1,5 +1,12 @@
class Collection < ArvadosBase
+ MD5_EMPTY = 'd41d8cd98f00b204e9800998ecf8427e'
+
+ # Return true if the given string is the locator of a zero-length blob
+ def self.is_empty_blob_locator? locator
+ !!locator.to_s.match("^#{MD5_EMPTY}(\\+.*)?\$")
+ end
+
def total_bytes
if files
tot = 0
diff --git a/apps/workbench/test/unit/collection_test.rb b/apps/workbench/test/unit/collection_test.rb
index 4f73670..bbfc983 100644
--- a/apps/workbench/test/unit/collection_test.rb
+++ b/apps/workbench/test/unit/collection_test.rb
@@ -1,7 +1,16 @@
require 'test_helper'
class CollectionTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ test 'recognize empty blob locator' do
+ ['d41d8cd98f00b204e9800998ecf8427e+0',
+ 'd41d8cd98f00b204e9800998ecf8427e',
+ 'd41d8cd98f00b204e9800998ecf8427e+0+Xyzzy'].each do |x|
+ assert_equal true, Collection.is_empty_blob_locator?(x)
+ end
+ ['d41d8cd98f00b204e9800998ecf8427e0',
+ 'acbd18db4cc2f85cedef654fccc4a4d8+3',
+ 'acbd18db4cc2f85cedef654fccc4a4d8+0'].each do |x|
+ assert_equal false, Collection.is_empty_blob_locator?(x)
+ end
+ end
end
commit 6232434d6b86192b0531bacd7e4f0d5ae589f368
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Mar 27 20:30:08 2014 -0400
Fix some whitespace complaints.
diff --git a/apps/workbench/app/views/pipeline_instances/_show_components.html.erb b/apps/workbench/app/views/pipeline_instances/_show_components.html.erb
index 69dd62e..eafaf8d 100644
--- a/apps/workbench/app/views/pipeline_instances/_show_components.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_show_components.html.erb
@@ -38,7 +38,7 @@
<% render_pipeline_jobs.each do |pj| %>
<tr>
<td>
- <% job_status = render(partial: 'job_status_label',
+ <% job_status = render(partial: 'job_status_label',
locals: { :j => pj[:job], :title => pj[:name] }) %>
<% if pj[:job].andand[:uuid] %>
<%= link_to(job_status, job_url(id: pj[:job][:uuid])) %>
@@ -51,7 +51,7 @@
</td><td>
<%= pj[:progress_bar] %>
</td><td>
- <%= render(partial: 'job_status_label',
+ <%= render(partial: 'job_status_label',
locals: { :j => pj[:job] }) %>
</td><td>
<%= link_to_if_arvados_object pj[:output] %>
@@ -71,7 +71,7 @@ setInterval(function(){$('a.refresh').click()}, 15000);
<% content_for :tab_line_buttons do %>
<%= form_tag @object, :method => :put do |f| %>
-
+
<%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => false %>
<%= button_tag "Stop pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
@@ -86,7 +86,7 @@ setInterval(function(){$('a.refresh').click()}, 15000);
<% content_for :tab_line_buttons do %>
<%= form_tag @object, :method => :put do |f| %>
-
+
<%= hidden_field @object.class.to_s.underscore.singularize.to_sym, :active, :value => true %>
<%= button_tag "Run pipeline", {class: 'btn btn-primary pull-right', id: "run-pipeline-button"} %>
@@ -94,5 +94,5 @@ setInterval(function(){$('a.refresh').click()}, 15000);
<% end %>
<%= render partial: 'pipeline_templates/show_components_template', locals: {:template => template, :obj => @object} %>
-
+
<% end %>
diff --git a/doc/sdk/cli/index.html.textile.liquid b/doc/sdk/cli/index.html.textile.liquid
index 3b63079..73acddc 100644
--- a/doc/sdk/cli/index.html.textile.liquid
+++ b/doc/sdk/cli/index.html.textile.liquid
@@ -58,4 +58,3 @@ $ <code class="userinput">gem build arvados-cli.gemspec</code>
$ <code class="userinput">sudo gem install arvados-cli-*.gem</code>
</pre>
</notextile>
-
diff --git a/doc/user/tutorials/tutorial-firstscript.html.textile.liquid b/doc/user/tutorials/tutorial-firstscript.html.textile.liquid
index e2d7a87..03c76f6 100644
--- a/doc/user/tutorials/tutorial-firstscript.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-firstscript.html.textile.liquid
@@ -129,7 +129,7 @@ EOF
* The component is listed with a human-readable name (@"do_hash"@ in this example)
* @"script"@ specifies the name of the script to run. The script is searched for in the "crunch_scripts/" subdirectory of the @git@ checkout specified by @"script_version"@.
* @"repository"@ is the git repository to search for the script version. You can access a list of available @git@ repositories on the Arvados workbench under "Compute %(rarr)→% Code repositories":https://{{site.arvados_workbench_host}}//repositories .
-* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit @git@ revision hash, a tag, or a branch (in which case it will take the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
+* @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit @git@ revision hash, a tag, or a branch (in which case it will take the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
* @"script_parameters"@ describes the parameters for the script. In this example, there is one parameter called @input@ which is @required@ and is a @Collection at .
* @"output_is_persistent"@ indicates whether the output of the job is considered valuable. If this value is false (or not given), the output will be treated as intermediate data and eventually deleted to reclaim disk space.
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 090be61..8a16492 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -455,7 +455,7 @@ class WhRunPipelineInstance
if (c[:job][:running] or
not (c[:job][:finished_at] or c[:job][:cancelled_at]))
# Job is running so update copy of job record
- c[:job] = JobCache.get(c[:job][:uuid])
+ c[:job] = JobCache.get(c[:job][:uuid])
end
if c[:job][:success]
@@ -547,7 +547,7 @@ class WhRunPipelineInstance
end
end
end
-
+
if ended == @components.length or failed > 0
@instance[:active] = false
@instance[:success] = (succeeded == @components.length)
diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index b35bc4e..178b48f 100644
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -6,11 +6,11 @@ class Arvados::V1::JobsController < ApplicationController
skip_before_filter :render_404_if_no_object, :only => :queue
def create
- [:repository, :script, :script_version, :script_parameters].each do |r|
+ [:repository, :script, :script_version, :script_parameters].each do |r|
if !resource_attrs[r]
return render json: {
:error => "#{r} attribute must be specified"
- }, status: :unprocessable_entity
+ }, status: :unprocessable_entity
end
end
@@ -26,8 +26,8 @@ class Arvados::V1::JobsController < ApplicationController
Job.readable_by(current_user).where(script: resource_attrs[:script],
script_version: r).
each do |j|
- if j.nondeterministic != true and
- j.success != false and
+ if j.nondeterministic != true and
+ j.success != false and
j.script_parameters == resource_attrs[:script_parameters]
# Record the first job in the list
if !@object
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index c5fc38d..fe2d0da 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -4,7 +4,7 @@ class Arvados::V1::UsersController < ApplicationController
skip_before_filter :render_404_if_no_object, only:
[:activate, :event_stream, :current, :system, :setup]
before_filter :admin_required, only: [:setup, :unsetup]
-
+
def current
@object = current_user
show
@@ -29,7 +29,7 @@ class Arvados::V1::UsersController < ApplicationController
end
end
end
-
+
def event_stream
channel = current_user.andand.uuid
if current_user.andand.is_admin
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index c09b67c..0539247 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -134,7 +134,7 @@ class User < ArvadosModel
end
return [oid_login_perm] + user.setup_repo_vm_links(repo_name, vm_uuid)
- end
+ end
# create links
def setup_repo_vm_links(repo_name, vm_uuid)
@@ -143,7 +143,7 @@ class User < ArvadosModel
group_perm = create_user_group_link
return [repo_perm, vm_login_perm, group_perm, self].compact
- end
+ end
# delete user signatures, login, repo, and vm perms, and mark as inactive
def unsetup
@@ -185,7 +185,7 @@ class User < ArvadosModel
# mark the user as inactive
self.is_active = false
self.save!
- end
+ end
protected
@@ -276,7 +276,7 @@ class User < ArvadosModel
link_class: 'permission',
name: 'can_write')
if repo_perms.any?
- logger.warn "User already has repository access " +
+ logger.warn "User already has repository access " +
repo_perms.collect { |p| p[:uuid] }.inspect
return repo_perms.first
end
@@ -299,9 +299,9 @@ class User < ArvadosModel
# create login permission for the given vm_uuid, if it does not already exist
def create_vm_login_permission_link(vm_uuid, repo_name)
begin
-
+
# vm uuid is optional
- if vm_uuid
+ if vm_uuid
vm = VirtualMachine.where(uuid: vm_uuid).first
if not vm
@@ -309,7 +309,7 @@ class User < ArvadosModel
raise "No vm found for #{vm_uuid}"
end
else
- return
+ return
end
logger.info { "vm uuid: " + vm[:uuid] }
@@ -363,7 +363,7 @@ class User < ArvadosModel
link_class: 'permission',
name: 'can_read')
logger.info { "group permission: " + group_perm[:uuid] }
- else
+ else
group_perm = group_perms.first
end
diff --git a/services/api/script/setup-new-user.rb b/services/api/script/setup-new-user.rb
index 19f5aeb..ebff192 100755
--- a/services/api/script/setup-new-user.rb
+++ b/services/api/script/setup-new-user.rb
@@ -24,7 +24,7 @@ claim the account.
end
log.level = (ENV['DEBUG'] || opts.debug) ? Logger::DEBUG : Logger::WARN
-
+
if ARGV.count != 3
Trollop::die "required arguments are missing"
end
@@ -39,8 +39,8 @@ begin
found_user = arv.user.get(uuid: user_arg)
rescue Arvados::TransactionFailedError
found = arv.user.list(where: {email: user_arg})[:items]
-
- if found.count == 0
+
+ if found.count == 0
if !user_arg.match(/\w\@\w+\.\w+/)
abort "About to create new user, but #{user_arg.inspect} " +
"does not look like an email address. Stop."
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index b15b99b..6f41902 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -66,7 +66,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
repo_name: repo_name,
openid_prefix: 'https://www.google.com/accounts/o8/id',
user: {
- uuid: "this_is_agreeable",
+ uuid: "this_is_agreeable",
first_name: "in_create_test_first_name",
last_name: "test_last_name",
email: "foo at example.com"
@@ -80,7 +80,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
assert_not_nil created['uuid'], 'expected non-null uuid for the new user'
assert_equal 'this_is_agreeable', created['uuid']
assert_not_nil created['email'], 'expected non-nil email'
- assert_nil created['identity_url'], 'expected no identity_url'
+ assert_nil created['identity_url'], 'expected no identity_url'
# arvados#user, repo link and link add user to 'All users' group
verify_num_links @all_links_at_start, 3
@@ -198,7 +198,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
get :current
assert_response :success
inactive_user = JSON.parse(@response.body)
-
+
authorize_with :admin
post :setup, {
@@ -214,7 +214,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
assert_equal inactive_user['uuid'], resp_obj['uuid']
- assert_equal inactive_user['email'], resp_obj['email'],
+ assert_equal inactive_user['email'], resp_obj['email'],
'expecting inactive user email'
# expect repo and vm links
@@ -230,7 +230,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
get :current
assert_response :success
inactive_user = JSON.parse(@response.body)
-
+
authorize_with :admin
post :setup, {
@@ -245,7 +245,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
assert_equal inactive_user['uuid'], resp_obj['uuid']
- assert_equal inactive_user['email'], resp_obj['email'],
+ assert_equal inactive_user['email'], resp_obj['email'],
'expecting inactive user email'
end
@@ -254,7 +254,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
get :current
assert_response :success
inactive_user = JSON.parse(@response.body)
-
+
authorize_with :admin
post :setup, {
@@ -269,7 +269,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
assert_not_nil resp_obj['uuid'], 'expected uuid for the new user'
assert_equal inactive_user['uuid'], resp_obj['uuid']
- assert_equal inactive_user['email'], resp_obj['email'],
+ assert_equal inactive_user['email'], resp_obj['email'],
'expecting inactive user email'
end
@@ -305,7 +305,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
response_body = JSON.parse(@response.body)
response_errors = response_body['errors']
assert_not_nil response_errors, 'Expected error in response'
- assert (response_errors.first.include? "No vm found for no_such_vm"),
+ assert (response_errors.first.include? "No vm found for no_such_vm"),
'Expected RuntimeError: No vm found for no_such_vm'
end
@@ -337,7 +337,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
openid_prefix: 'https://www.google.com/accounts/o8/id'
}
- assert_response :success
+ assert_response :success
response_items = JSON.parse(@response.body)['items']
response_object = find_obj_in_resp response_items, 'User', nil
assert_not_nil response_object['uuid'], 'expected uuid for new user'
@@ -365,7 +365,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
response_object = find_obj_in_resp response_items, 'User', nil
assert_not_nil response_object['uuid'], 'expected uuid for new user'
assert_equal response_object['email'], 'foo at example.com', 'expected given email'
- assert_equal 'test_first_name', response_object['first_name'],
+ assert_equal 'test_first_name', response_object['first_name'],
'expecting first name'
# four extra links; login link, group link, repo link and vm link
@@ -399,7 +399,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
assert_response :success
response_items = JSON.parse(@response.body)['items']
response_object2 = find_obj_in_resp response_items, 'User', nil
- assert_not_equal response_object['uuid'], response_object2['uuid'],
+ assert_not_equal response_object['uuid'], response_object2['uuid'],
'expected same uuid as first create operation'
assert_equal response_object['email'], 'foo at example.com', 'expected given email'
@@ -428,7 +428,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
assert_equal 'in_create_test_first_name', created['first_name']
assert_not_nil created['uuid'], 'expected uuid for new user'
assert_not_nil created['email'], 'expected non-nil email'
- assert_nil created['identity_url'], 'expected no identity_url'
+ assert_nil created['identity_url'], 'expected no identity_url'
# verify links
# 3 new links: arvados#user, repo, and 'All users' group.
@@ -488,7 +488,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
assert_equal 'in_create_test_first_name', created['first_name']
assert_not_nil created['uuid'], 'expected uuid for new user'
assert_not_nil created['email'], 'expected non-nil email'
- assert_nil created['identity_url'], 'expected no identity_url'
+ assert_nil created['identity_url'], 'expected no identity_url'
# expect 4 new links: arvados#user, repo, vm and 'All users' group link
verify_num_links @all_links_at_start, 4
@@ -502,7 +502,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
verify_link response_items, 'arvados#group', true, 'permission', 'can_read',
'All users', created['uuid'], 'arvados#group', true, 'Group'
- verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
+ verify_link response_items, 'arvados#virtualMachine', true, 'permission', 'can_login',
@vm_uuid, created['uuid'], 'arvados#virtualMachine', false, 'VirtualMachine'
end
@@ -516,7 +516,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
response_body = JSON.parse(@response.body)
response_errors = response_body['errors']
assert_not_nil response_errors, 'Expected error in response'
- assert (response_errors.first.include? 'PermissionDenied'),
+ assert (response_errors.first.include? 'PermissionDenied'),
'Expected PermissionDeniedError'
end
@@ -531,7 +531,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
response_body = JSON.parse(@response.body)
response_errors = response_body['errors']
assert_not_nil response_errors, 'Expected error in response'
- assert (response_errors.first.include? 'Forbidden'),
+ assert (response_errors.first.include? 'Forbidden'),
'Expected Forbidden error'
end
@@ -661,7 +661,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
created2 = JSON.parse(@response.body)
assert_not_nil created2['uuid'], 'expected uuid for the newly created user'
assert_equal created['uuid'], created2['uuid'], 'expected uuid not found'
-
+
verify_link_existence created['uuid'], created['email'], false, false, false, false
end
@@ -724,7 +724,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
link = find_obj_in_resp response_items, 'Link', link_object_name
- if !expect_link
+ if !expect_link
assert_nil link, "Expected no link for #{link_object_name}"
return
end
@@ -738,16 +738,16 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
end
assert_equal link['link_class'], link_class,
"did not find expected link_class for #{link_object_name}"
-
+
assert_equal link['name'], link_name,
"did not find expected link_name for #{link_object_name}"
-
+
assert_equal link['tail_uuid'], tail_uuid,
"did not find expected tail_uuid for #{link_object_name}"
-
+
assert_equal link['head_kind'], head_kind,
"did not find expected head_kind for #{link_object_name}"
-
+
assert_equal link['head_uuid'], head_uuid,
"did not find expected head_uuid for #{link_object_name}"
end
@@ -788,7 +788,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
signed_uuids = Link.where(link_class: 'signature',
tail_kind: 'arvados#user',
tail_uuid: uuid)
-
+
if expect_signatures
assert signed_uuids.any?, "expected singnatures"
else
diff --git a/services/api/test/unit/user_test.rb b/services/api/test/unit/user_test.rb
index 82f61e0..9f78b1a 100644
--- a/services/api/test/unit/user_test.rb
+++ b/services/api/test/unit/user_test.rb
@@ -1,7 +1,281 @@
require 'test_helper'
class UserTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+
+ # The fixture services/api/test/fixtures/users.yml serves as the input for this test case
+ setup do
+ @all_users = User.find(:all)
+
+ @all_users.each do |user|
+ if user.is_admin && user.is_active
+ @admin_user = user
+ elsif user.is_active && !user.is_admin
+ @active_user = user
+ elsif !user.is_active && !user.is_invited
+ @uninvited_user = user
+ end
+ end
+ end
+
+ test "check non-admin active user properties" do
+ assert !@active_user.is_admin, 'is_admin should not be set for a non-admin user'
+ assert @active_user.is_active, 'user should be active'
+ assert @active_user.is_invited, 'is_invited should be set'
+ assert_not_nil @active_user.prefs, "user's preferences should be non-null, but may be size zero"
+ assert (@active_user.can? :read=>"#{@active_user.uuid}"), "user should be able to read own object"
+ assert (@active_user.can? :write=>"#{@active_user.uuid}"), "user should be able to write own object"
+ assert (@active_user.can? :manage=>"#{@active_user.uuid}"), "user should be able to manage own object"
+
+ assert @active_user.groups_i_can(:read).size > 0, "active user should be able read at least one group"
+
+ # non-admin user cannot manage or write other user objects
+ assert !(@active_user.can? :read=>"#{@uninvited_user.uuid}")
+ assert !(@active_user.can? :write=>"#{@uninvited_user.uuid}")
+ assert !(@active_user.can? :manage=>"#{@uninvited_user.uuid}")
+ end
+
+ test "check admin user properties" do
+ assert @admin_user.is_admin, 'is_admin should be set for admin user'
+ assert @admin_user.is_active, 'admin user cannot be inactive'
+ assert @admin_user.is_invited, 'is_invited should be set'
+ assert_not_nil @admin_user.uuid.size, "user's uuid should be non-null"
+ assert_not_nil @admin_user.prefs, "user's preferences should be non-null, but may be size zero"
+ assert @admin_user.identity_url.size > 0, "user's identity url is expected"
+ assert @admin_user.can? :read=>"#{@admin_user.uuid}"
+ assert @admin_user.can? :write=>"#{@admin_user.uuid}"
+ assert @admin_user.can? :manage=>"#{@admin_user.uuid}"
+
+ assert @admin_user.groups_i_can(:read).size > 0, "admin active user should be able read at least one group"
+ assert @admin_user.groups_i_can(:write).size > 0, "admin active user should be able write to at least one group"
+ assert @admin_user.groups_i_can(:manage).size > 0, "admin active user should be able manage at least one group"
+
+ # admin user can also write or manage other users
+ assert @admin_user.can? :read=>"#{@uninvited_user.uuid}"
+ assert @admin_user.can? :write=>"#{@uninvited_user.uuid}"
+ assert @admin_user.can? :manage=>"#{@uninvited_user.uuid}"
+ end
+
+ test "check inactive and uninvited user properties" do
+ assert !@uninvited_user.is_admin, 'is_admin should not be set for a non-admin user'
+ assert !@uninvited_user.is_active, 'user should be inactive'
+ assert !@uninvited_user.is_invited, 'is_invited should not be set'
+ assert @uninvited_user.can? :read=>"#{@uninvited_user.uuid}"
+ assert @uninvited_user.can? :write=>"#{@uninvited_user.uuid}"
+ assert @uninvited_user.can? :manage=>"#{@uninvited_user.uuid}"
+
+ assert @uninvited_user.groups_i_can(:read).size == 0, "inactive and uninvited user should not be able read any groups"
+ assert @uninvited_user.groups_i_can(:write).size == 0, "inactive and uninvited user should not be able write to any groups"
+ assert @uninvited_user.groups_i_can(:manage).size == 0, "inactive and uninvited user should not be able manage any groups"
+ end
+
+ test "find user method checks" do
+ User.find(:all).each do |user|
+ assert_not_nil user.uuid, "non-null uuid expected for " + user.full_name
+ end
+
+ user = users(:active) # get the active user
+
+ found_user = User.find(user.id) # find a user by the row id
+
+ assert_equal found_user.full_name, user.first_name + ' ' + user.last_name
+ assert_equal found_user.identity_url, user.identity_url
+ end
+
+ test "create new user" do
+ Thread.current[:user] = @admin_user # set admin user as the current user
+
+ user = User.new
+ user.first_name = "first_name_for_newly_created_user"
+ user.save
+
+ # verify there is one extra user in the db now
+ assert (User.find(:all).size == @all_users.size+1)
+
+ user = User.find(user.id) # get the user back
+ assert_equal(user.first_name, 'first_name_for_newly_created_user')
+ assert_not_nil user.uuid, 'uuid should be set for newly created user'
+ assert_nil user.email, 'email should be null for newly created user, because it was not passed in'
+ assert_nil user.identity_url, 'identity_url should be null for newly created user, because it was not passed in'
+
+ user.first_name = 'first_name_for_newly_created_user_updated'
+ user.save
+ user = User.find(user.id) # get the user back
+ assert_equal(user.first_name, 'first_name_for_newly_created_user_updated')
+ end
+
+ test "update existing user" do
+ Thread.current[:user] = @active_user # set active user as current user
+ @active_user.first_name = "first_name_changed"
+ @active_user.save
+
+ @active_user = User.find(@active_user.id) # get the user back
+ assert_equal(@active_user.first_name, 'first_name_changed')
+
+ # admin user also should be able to update the "active" user info
+ Thread.current[:user] = @admin_user # set admin user as current user
+ @active_user.first_name = "first_name_changed_by_admin_for_active_user"
+ @active_user.save
+
+ @active_user = User.find(@active_user.id) # get the user back
+ assert_equal(@active_user.first_name, 'first_name_changed_by_admin_for_active_user')
+ end
+
+ test "delete a user and verify" do
+ active_user_uuid = @active_user.uuid
+
+ Thread.current[:user] = @admin_user
+ @active_user.delete
+
+ found_deleted_user = false
+ User.find(:all).each do |user|
+ if user.uuid == active_user_uuid
+ found_deleted_user = true
+ break
+ end
+ end
+ assert !found_deleted_user, "found deleted user: "+active_user_uuid
+
+ end
+
+ test "create new user as non-admin user" do
+ Thread.current[:user] = @active_user
+
+ begin
+ user = User.new
+ user.save
+ rescue ArvadosModel::PermissionDeniedError => e
+ end
+ assert (e.message.include? 'PermissionDeniedError'),
+ 'Expected PermissionDeniedError'
+ end
+
+ test "setup new user" do
+ Thread.current[:user] = @admin_user
+
+ email = 'foo at example.com'
+ openid_prefix = 'http://openid/prefix'
+
+ user = User.new
+ user.email = email
+ user.uuid = 'abcdefghijklmnop'
+
+ vm = VirtualMachine.create
+
+ response = User.setup user, openid_prefix, 'test_repo', vm.uuid
+
+ resp_user = find_obj_in_resp response, 'User'
+ verify_user resp_user, email
+
+ oid_login_perm = find_obj_in_resp response, 'Link', 'arvados#user'
+ verify_link oid_login_perm, 'permission', 'can_login', resp_user[:email],
+ resp_user[:uuid]
+ assert_equal openid_prefix, oid_login_perm[:properties][:identity_url_prefix],
+ 'expected identity_url_prefix not found for oid_login_perm'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ vm_perm = find_obj_in_resp response, 'Link', 'arvados#virtualMachine'
+ verify_link vm_perm, 'permission', 'can_login', resp_user[:uuid], vm.uuid
+ end
+
+ test "setup new user in multiple steps" do
+ Thread.current[:user] = @admin_user
+
+ email = 'foo at example.com'
+ openid_prefix = 'http://openid/prefix'
+
+ user = User.new
+ user.email = email
+ user.uuid = 'abcdefghijklmnop'
+
+ response = User.setup user, openid_prefix
+
+ resp_user = find_obj_in_resp response, 'User'
+ verify_user resp_user, email
+
+ oid_login_perm = find_obj_in_resp response, 'Link', 'arvados#user'
+ verify_link oid_login_perm, 'permission', 'can_login', resp_user[:email],
+ resp_user[:uuid]
+ assert_equal openid_prefix, oid_login_perm[:properties][:identity_url_prefix],
+ 'expected identity_url_prefix not found for oid_login_perm'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ # invoke setup again with repo_name
+ response = User.setup user, openid_prefix, 'test_repo'
+ resp_user = find_obj_in_resp response, 'User', nil
+ verify_user resp_user, email
+ assert_equal user.uuid, resp_user[:uuid], 'expected uuid not found'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ # invoke setup again with a vm_uuid
+ vm = VirtualMachine.create
+
+ response = User.setup user, openid_prefix, 'test_repo', vm.uuid
+
+ resp_user = find_obj_in_resp response, 'User', nil
+ verify_user resp_user, email
+ assert_equal user.uuid, resp_user[:uuid], 'expected uuid not found'
+
+ group_perm = find_obj_in_resp response, 'Link', 'arvados#group'
+ verify_link group_perm, 'permission', 'can_read', resp_user[:uuid], nil
+
+ repo_perm = find_obj_in_resp response, 'Link', 'arvados#repository'
+ verify_link repo_perm, 'permission', 'can_write', resp_user[:uuid], nil
+
+ vm_perm = find_obj_in_resp response, 'Link', 'arvados#virtualMachine'
+ verify_link vm_perm, 'permission', 'can_login', resp_user[:uuid], vm.uuid
+ end
+
+ def find_obj_in_resp (response, object_type, head_kind=nil)
+ return_obj = nil
+ response.each { |x|
+ if x.class.name == object_type
+ if head_kind
+ if x.head_kind == head_kind
+ return_obj = x
+ break
+ end
+ else
+ return_obj = x
+ break
+ end
+ end
+ }
+ return return_obj
+ end
+
+ def verify_user (resp_user, email)
+ assert_not_nil resp_user, 'expected user object'
+ assert_not_nil resp_user['uuid'], 'expected user object'
+ assert_equal email, resp_user['email'], 'expected email not found'
+
+ end
+
+ def verify_link (link_object, link_class, link_name, tail_uuid, head_uuid)
+ assert_not_nil link_object, 'expected link for #{link_class} #{link_name}'
+ assert_not_nil link_object[:uuid],
+ 'expected non-nil uuid for link for #{link_class} #{link_name}'
+ assert_equal link_class, link_object[:link_class],
+ 'expected link_class not found for #{link_class} #{link_name}'
+ assert_equal link_name, link_object[:name],
+ 'expected link_name not found for #{link_class} #{link_name}'
+ assert_equal tail_uuid, link_object[:tail_uuid],
+ 'expected tail_uuid not found for #{link_class} #{link_name}'
+ if head_uuid
+ assert_equal head_uuid, link_object[:head_uuid],
+ 'expected head_uuid not found for #{link_class} #{link_name}'
+ end
+ end
+
end
commit b0af0440838407be80c6208b15f969bc2942008e
Merge: c95bf51 736c7c6
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Mar 27 20:25:07 2014 -0400
Merge branch 'master' into 1932-job-output-persistent
Conflicts:
doc/user/tutorials/tutorial-firstscript.html.textile.liquid
doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid
sdk/cli/bin/arv-run-pipeline-instance
services/api/db/schema.rb
diff --cc doc/user/tutorials/tutorial-firstscript.html.textile.liquid
index ad2f0a4,a3b7e54..e2d7a87
--- a/doc/user/tutorials/tutorial-firstscript.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-firstscript.html.textile.liquid
@@@ -111,8 -111,8 +111,9 @@@ Next, create a file that contains the p
"dataclass": "Collection"
}
},
- "script_version":"<b>you</b>:master",
+ "repository":"<b>you</b>",
- "script_version":"master"
++ "script_version":"master",
+ "output_is_persistent":true
}
}
}
@@@ -127,9 -127,9 +128,10 @@@ EO
* @"components"@ is a set of scripts that make up the pipeline
* The component is listed with a human-readable name (@"do_hash"@ in this example)
* @"script"@ specifies the name of the script to run. The script is searched for in the "crunch_scripts/" subdirectory of the @git@ checkout specified by @"script_version"@.
- * @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit @git@ revision hash, or in the form "repository:branch" (in which case it will take the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run. You can access a list of available @git@ repositories on the Arvados workbench under "Compute %(rarr)→% Code repositories":http://{{site.arvados_workbench_host}}//repositories .
+ * @"repository"@ is the git repository to search for the script version. You can access a list of available @git@ repositories on the Arvados workbench under "Compute %(rarr)→% Code repositories":https://{{site.arvados_workbench_host}}//repositories .
+ * @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit @git@ revision hash, a tag, or a branch (in which case it will take the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run.
* @"script_parameters"@ describes the parameters for the script. In this example, there is one parameter called @input@ which is @required@ and is a @Collection at .
+* @"output_is_persistent"@ indicates whether the output of the job is considered valuable. If this value is false (or not given), the output will be treated as intermediate data and eventually deleted to reclaim disk space.
Now, use @arv pipeline_template create@ tell Arvados about your pipeline template:
diff --cc doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid
index 6852886,132bc9d..fc98d01
--- a/doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid
+++ b/doc/user/tutorials/tutorial-new-pipeline.html.textile.liquid
@@@ -43,8 -43,8 +43,9 @@@ Next, create a file that contains the p
"dataclass": "Collection"
}
},
- "script_version":"<b>you</b>:master",
+ "repository":"<b>you</b>",
- "script_version":"master"
++ "script_version":"master",
+ "output_is_persistent":false
},
"filter":{
"script":"0-filter.py",
@@@ -53,8 -53,8 +54,9 @@@
"output_of":"do_hash"
}
},
- "script_version":"<b>you</b>:master",
+ "repository":"<b>you</b>",
- "script_version":"master"
++ "script_version":"master",
+ "output_is_persistent":true
}
}
}
diff --cc sdk/cli/bin/arv-run-pipeline-instance
index 7578abc,e0e002f..090be61
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@@ -435,83 -424,28 +426,31 @@@ class WhRunPipelineInstanc
moretodo = false
@components.each do |cname, c|
job = nil
-
+ c_already_finished = (c[:job] &&
+ c[:job][:uuid] &&
+ !c[:job][:success].nil?)
if !c[:job] and
- c[:script_parameters].select { |pname, p| p.is_a? Hash }.empty?
- # Job is fully specified (all parameter values are present) but
- # no particular job has been found.
-
- debuglog "component #{cname} ready to satisfy."
-
- c.delete :wait
- second_place_job = nil # satisfies component, but not finished yet
-
- (@options[:no_reuse] ? [] : JobCache.
- where(script: c[:script],
- script_parameters: c[:script_parameters],
- script_version_descends_from: c[:script_version])
- ).each do |candidate_job|
- candidate_params_downcase = Hash[candidate_job[:script_parameters].
- map { |k,v| [k.downcase,v] }]
- c_params_downcase = Hash[c[:script_parameters].
- map { |k,v| [k.downcase,v] }]
-
- debuglog "component #{cname} considering job #{candidate_job[:uuid]} version #{candidate_job[:script_version]} parameters #{candidate_params_downcase.inspect}", 3
-
- unless candidate_params_downcase == c_params_downcase
- next
- end
-
- if c[:script_version] !=
- candidate_job[:script_version][0,c[:script_version].length]
- debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if script_version matched.", 2
- next
- end
-
- unless candidate_job[:success] || candidate_job[:running] ||
- (!candidate_job[:started_at] && !candidate_job[:cancelled_at])
- debuglog "component #{cname} would be satisfied by job #{candidate_job[:uuid]} if it were running or successful.", 2
- next
- end
-
- if candidate_job[:success]
- unless @options[:no_reuse_finished]
- job = candidate_job
- $stderr.puts "using #{job[:uuid]} (finished at #{job[:finished_at]}) for component #{cname}"
- c[:job] = job
- end
- else
- second_place_job ||= candidate_job
- end
- break
- end
- if not c[:job] and second_place_job
- job = second_place_job
- $stderr.puts "using #{job[:uuid]} (running since #{job[:started_at]}) for component #{cname}"
+ c[:script_parameters].select { |pname, p| p.is_a? Hash and p[:output_of]}.empty?
+ # No job yet associated with this component and is component inputs
+ # are fully specified (any output_of script_parameters are resolved
+ # to real value)
+ job = JobCache.create({:script => c[:script],
+ :script_parameters => c[:script_parameters],
+ :script_version => c[:script_version],
+ :repository => c[:repository],
+ :minimum_script_version => c[:minimum_script_version],
+ :exclude_script_versions => c[:exclude_minimum_script_versions],
+ :nondeterministic => c[:nondeterministic],
- :no_reuse => @options[:no_reuse]})
++ :no_reuse => @options[:no_reuse],
++ :output_is_persistent => c[:output_is_persistent] || false})
+ if job
+ debuglog "component #{cname} new job #{job[:uuid]}"
c[:job] = job
+ else
+ debuglog "component #{cname} new job failed"
end
- if not c[:job]
- debuglog "component #{cname} not satisfied by any existing job."
- if !@options[:dry_run]
- debuglog "component #{cname} new job."
- job = JobCache.create(:script => c[:script],
- :script_parameters => c[:script_parameters],
- :runtime_constraints => c[:runtime_constraints] || {},
- :script_version => c[:script_version] || 'master',
- :output_is_persistent => c[:output_is_persistent] || false)
- if job
- debuglog "component #{cname} new job #{job[:uuid]}"
- c[:job] = job
- else
- debuglog "component #{cname} new job failed"
- end
- end
- end
- else
- c[:wait] = true
end
+
if c[:job] and c[:job][:uuid]
if (c[:job][:running] or
not (c[:job][:finished_at] or c[:job][:cancelled_at]))
@@@ -529,43 -465,9 +470,44 @@@
end
end
end
+ unless c_already_finished
+ if c[:output_is_persistent]
+ # This is my first time discovering that the job
+ # succeeded. I need to make sure a resources/wants
+ # link is in place to protect the output from garbage
+ # collection. (Normally Crunch does this for me, but
+ # here I might be reusing the output of someone else's
+ # job and I need to make sure it's understood that the
+ # output is valuable to me, too.)
+ wanted = c[:job][:output]
+ debuglog "checking for existing persistence link for #{wanted}"
+ @my_user_uuid ||= $arv.user.current[:uuid]
+ links = $arv.link.list(limit: 1,
+ filters:
+ [%w(link_class = resources),
+ %w(name = wants),
+ %w(tail_uuid =) + [@my_user_uuid],
+ %w(head_uuid =) + [wanted]
+ ])[:items]
+ if links.any?
+ debuglog "link already exists, uuid #{links.first[:uuid]}"
+ else
+ newlink = $arv.link.create link: \
+ {
+ link_class: 'resources',
+ name: 'wants',
+ tail_kind: 'arvados#user',
+ tail_uuid: @my_user_uuid,
+ head_kind: 'arvados#collection',
+ head_uuid: wanted
+ }
+ debuglog "added link, uuid #{newlink[:uuid]}"
+ end
+ end
+ end
elsif c[:job][:running] ||
(!c[:job][:started_at] && !c[:job][:cancelled_at])
+ # Job is still running
moretodo = true
elsif c[:job][:cancelled_at]
debuglog "component #{cname} job #{c[:job][:uuid]} cancelled."
diff --cc services/api/db/schema.rb
index 5c2fe29,97e6e9c..8ab6f5f
--- a/services/api/db/schema.rb
+++ b/services/api/db/schema.rb
@@@ -189,7 -189,8 +189,9 @@@ ActiveRecord::Schema.define(:version =
t.string "log"
t.text "tasks_summary"
t.text "runtime_constraints"
+ t.boolean "nondeterministic"
+ t.string "repository"
+ t.boolean "output_is_persistent", :default => false, :null => false
end
add_index "jobs", ["created_at"], :name => "index_jobs_on_created_at"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list