[ARVADOS] updated: e2cfa981e8a18453b4eff1edcf79497dc648fe40
Git user
git at public.curoverse.com
Tue Nov 1 20:48:12 EDT 2016
Summary of changes:
COPYING | 14 +-
.../app/assets/javascripts/work_unit_component.js | 18 +
.../app/controllers/application_controller.rb | 38 +-
.../app/controllers/collections_controller.rb | 13 +-
.../app/controllers/projects_controller.rb | 17 +-
.../controllers/work_unit_templates_controller.rb | 6 +-
.../app/controllers/work_units_controller.rb | 66 ++-
apps/workbench/app/models/container.rb | 2 +-
apps/workbench/app/models/container_request.rb | 2 +-
apps/workbench/app/models/container_work_unit.rb | 6 +-
apps/workbench/app/models/job.rb | 2 +-
apps/workbench/app/models/job_task.rb | 2 +-
apps/workbench/app/models/pipeline_instance.rb | 2 +-
.../app/models/pipeline_instance_work_unit.rb | 4 +-
apps/workbench/app/models/proxy_work_unit.rb | 9 +-
apps/workbench/app/models/work_unit.rb | 4 +
.../pipeline_instances/_show_components.html.erb | 2 +-
.../views/work_units/_component_detail.html.erb | 28 +-
.../app/views/work_units/_show_child.html.erb | 29 +-
.../app/views/work_units/_show_component.html.erb | 47 +-
.../app/views/work_units/_show_status.html.erb | 2 +-
apps/workbench/config/routes.rb | 4 +-
.../test/controllers/work_units_controller_test.rb | 23 -
apps/workbench/test/integration/jobs_test.rb | 2 +-
apps/workbench/test/integration/work_units_test.rb | 25 +
build/go-python-package-scripts/postinst | 64 ++-
build/go-python-package-scripts/prerm | 37 +-
.../test-package-python-arvados-cwl-runner.sh | 1 +
.../test-package-python-arvados-fuse.sh | 1 +
.../test-package-python-arvados-python-client.sh | 1 +
...t-package-python27-python-arvados-cwl-runner.sh | 6 +
...ackage-python27-python-arvados-python-client.sh | 2 +-
build/run-build-docker-images.sh | 2 +-
build/run-build-docker-jobs-image.sh | 69 ++-
build/run-build-packages.sh | 8 +-
build/run-library.sh | 12 +-
doc/_includes/_install_docker_cleaner.liquid | 35 +-
docker/jobs/Dockerfile | 12 +-
sdk/cli/bin/crunch-job | 2 +-
sdk/cwl/arvados_cwl/__init__.py | 33 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 21 +-
sdk/cwl/arvados_cwl/arvdocker.py | 14 +-
sdk/cwl/arvados_cwl/arvjob.py | 28 +-
sdk/cwl/arvados_cwl/crunch_script.py | 19 +-
sdk/cwl/arvados_cwl/runner.py | 8 +
sdk/cwl/gittaggers.py | 38 +-
sdk/cwl/setup.py | 15 +
sdk/cwl/tests/test_job.py | 36 +-
sdk/cwl/tests/test_submit.py | 17 +-
sdk/python/arvados/arvfile.py | 78 ++--
sdk/python/arvados/commands/put.py | 516 +++++++++++++++------
sdk/python/tests/test_arv_put.py | 178 +++----
.../api/app/controllers/application_controller.rb | 8 +
.../arvados/v1/containers_controller.rb | 17 +
.../app/controllers/user_sessions_controller.rb | 3 +
services/api/app/models/container.rb | 48 +-
services/api/app/models/user.rb | 36 +-
services/api/config/application.default.yml | 5 +-
services/api/config/routes.rb | 1 +
services/api/lib/josh_id.rb | 1 +
.../test/fixtures/api_client_authorizations.yml | 14 +
services/api/test/fixtures/collections.yml | 13 +
services/api/test/fixtures/containers.yml | 4 +-
.../arvados/v1/containers_controller_test.rb | 19 +
.../functional/arvados/v1/jobs_controller_test.rb | 13 +
.../arvados/v1/schema_controller_test.rb | 8 +-
.../api/test/integration/user_sessions_test.rb | 21 +-
services/api/test/unit/container_request_test.rb | 2 +-
services/api/test/unit/container_test.rb | 92 +++-
services/crunch-run/crunchrun.go | 15 +
services/crunch-run/crunchrun_test.go | 49 ++
.../dockercleaner/arvados-docker-cleaner.service | 7 +-
services/dockercleaner/arvados_docker/cleaner.py | 21 +-
services/dockercleaner/setup.py | 1 +
services/dockercleaner/tests/test_cleaner.py | 6 +
services/fuse/arvados_fuse/__init__.py | 14 +
services/fuse/tests/integration_test.py | 1 -
services/fuse/tests/test_mount.py | 34 --
services/fuse/tests/test_token_expiry.py | 68 +++
services/login-sync/bin/arvados-login-sync | 56 ++-
services/login-sync/test/test_add_user.rb | 2 +-
tools/arvbox/bin/arvbox | 28 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.base | 20 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 2 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.dev | 11 +-
tools/arvbox/lib/arvbox/docker/common.sh | 7 +-
.../lib/arvbox/docker/service/sdk/run-service | 8 +-
87 files changed, 1568 insertions(+), 707 deletions(-)
create mode 100644 apps/workbench/app/assets/javascripts/work_unit_component.js
create mode 120000 build/package-testing/test-package-python-arvados-cwl-runner.sh
create mode 120000 build/package-testing/test-package-python-arvados-fuse.sh
create mode 120000 build/package-testing/test-package-python-arvados-python-client.sh
create mode 100755 build/package-testing/test-package-python27-python-arvados-cwl-runner.sh
mode change 120000 => 100644 sdk/cwl/gittaggers.py
create mode 100644 services/fuse/tests/test_token_expiry.py
via e2cfa981e8a18453b4eff1edcf79497dc648fe40 (commit)
via 0d686814570f4fca6d3b35d5e37a321752c8d50a (commit)
via cf569180b5121c1d2210a0fc1aa986e207d09af3 (commit)
via 9e2e220996943f6de9a059a6b57c458f60f10e8e (commit)
via ceabcac63dc3832f89cefd84857dd4ff658a31df (commit)
via b13ad9d7173ac5dff945d4a2ec572510ff0fd5da (commit)
via 69abf05f9257431a2084897dd3e7f1a4b76ae462 (commit)
via 62645650fcd451d2ea569c8e55526f6ed0db4fea (commit)
via 15d8499a8d95abbc4bc2dbbd0bcfd2a4c6666408 (commit)
via 4e9fd075b8136613e7edbb0465bcb96ccf5b1f45 (commit)
via 734335da27f27e2177d3b931b1e5e9e8e83a042f (commit)
via 5c05b488a1f2318e73780b67f6f654b4a12c32b3 (commit)
via c14d6c107c8b3e2d110be12148e3a04d8ee04ef6 (commit)
via ceec074ede6c1e4d8e588ee66dbb6704bf903b68 (commit)
via 95fc98726d64c71ed0b3a8c2270ee62c1c5d1bb5 (commit)
via 4c0b3359319e4c778371e3d61368dec6635c3ef2 (commit)
via 246413f02ca8711f117a8575fccfec82e5dbfe23 (commit)
via d966c816e478d1747c6e0e39d3430526120c655f (commit)
via 7898b516dfc7e5e9177753d5bcdfc980dc055308 (commit)
via 0066dc77abc461090fe98bcee7c6e324a5ca43a1 (commit)
via 07b6eaeada7c10c3efb9b917579c474bd66685b0 (commit)
via 078bdb166766423e1a423523e5285966aff7ec6b (commit)
via 6f04833d7837659acdfd01d4fbcf4ae36e8e5f5c (commit)
via 68463d51530a921dd965075e1721744c75322703 (commit)
via 40cad1b6c6b969c78fb415ee7be59dac32c45891 (commit)
via 85292497b7a89b9a0681fc38d307eea26681924c (commit)
via 8d6ffdf2fba938e7fbb5f128664d828d2669bcfe (commit)
via c95fd352d669bc65c0dbc9970ca11291cc933577 (commit)
via f830fb6b5a8113aabe6a377cadbb50dc52d5d1e0 (commit)
via 4304baee8e6cc9483a0cae397149526a0e8356fb (commit)
via 28e0fb829175415c533ea783d90ade8a39bdb236 (commit)
via 9ac31a42d4c777c158b2b9a7623194217d7a0772 (commit)
via 182dd3521c1ea587a94ed2bfa4e1a3e935d493c7 (commit)
via 4b037a38dbe848a77224dd05dd035736b5d0c0c1 (commit)
via 7bf89e17945bb6610d6339409a6a411803851434 (commit)
via 5015b0a3ab4ba407b162abf673d114924ca86c92 (commit)
via b4e586171f3fa3eab6b11f4fa4c71c346b2812e7 (commit)
via f6288c66e7ae5a2c736e1c9897acedf6d35b0a10 (commit)
via 4c86081e1ee5149f09f05cd3b77e59fc652d4ab3 (commit)
via 633781b1ac0f5b23cb43c43d0dda4627c2b9924e (commit)
via 09a87e19162124ca40cbf13cc90d2aed72f90478 (commit)
via cd0052e87bfa9d7e3d501427c7ff47e08c68d8d5 (commit)
via b73985d8a0c9173aec57f6a81fe540b2813a5bff (commit)
via dfe0ec7bfec3fd72cd40d3962e5c8af08d2413d2 (commit)
via 6349237efb19ca7fcad94333090d694fda3d4374 (commit)
via 842850ea58a856b93755d5f8990e7473cb034504 (commit)
via 2431a079d91545770bc931fc03afbca81dfa2004 (commit)
via 670b4ba238d93910ff087794d359da0d5ac469fa (commit)
via e8a76617eeec9429ddf7879048a310ba6e15e0d8 (commit)
via a70a555fbd30524568a0e019df184468f28556f0 (commit)
via 6362c17d947b16db1e2b71f7a06da9c930ef9778 (commit)
via f61a39eb70046280bd4611ea2dad38a5602303dc (commit)
via b8a53fdbe406c1c46ba39facccec63c5055001bf (commit)
via 91dc5f1d7f5ad9eb2640f6089e2d0476cbf87c8e (commit)
via 68387de8c5b5c5950f2e10d315172658297835b9 (commit)
via 9290528fb9700db4541ebe29c94eda3503f509f4 (commit)
via d6e4e65cf3aae701c3e02a814329757e869a1d6d (commit)
via 1c7de9d85417202a48ea82409ab390dfcab7f2a5 (commit)
via e14484fbba64222e4b32c353e3ea057c6f10acd0 (commit)
via 4d59e4f59204ff6e278ad7f37b55ead3911412e1 (commit)
via 617ef14a3f2fac4dad5f699af7000187143ebbe3 (commit)
via 0e5198142fdba0ce2af6eb2852d45dff46ffb2e2 (commit)
via 3d984f3ee6182391094ece7e32d61c40be7ba903 (commit)
via 754913fc3e5037903048863f5545c9512fbf7bfc (commit)
via d5524135b1495b919de332df4f952926664961f5 (commit)
via cfa2c529c7c8105e4d424b404db2c48ff2066e27 (commit)
via f84926fc3f934ea1958c810637851ec91994fc16 (commit)
via 7e54f217895d2f401563c54704c73a196e2d236d (commit)
via 8ffceeb0bddd457cee62586d405afd8e082e1d6f (commit)
via 1f208cec34df32a83683110b552a1f62767a4852 (commit)
via 360d5f885a6613a506e283ff74b18382249cf929 (commit)
via e8e45c5267f216909382aad7fc976dca82008284 (commit)
via cde03921959a7edbac9f6e969b3b8b99ee3c5b86 (commit)
via 957068e7b647db0bf2188b2c9ed4bc673f0744f2 (commit)
via 162766a3f8e91024f0f40fbc3d5955b899df73a9 (commit)
via f99aa5b85a1aa3bff1f1140f6a109fd308b03d30 (commit)
via e144534106f2031b4a9bec0d80eebf094e33a285 (commit)
via 1658a97ba74320c2da1bed1d5fbfbaa767203e1c (commit)
via b8573fab8e641eab5adc123e9f335510bff65172 (commit)
via 9aecc4974eecdfaf94db275112c2a37601802fe7 (commit)
via f0bc71892f7ba3b2bb42e96598302676faa4a652 (commit)
via 61060582be06315a7a44347acbab9613bf1c4c93 (commit)
via d71020ac08d7b6e84d2d8f8d2c9b22d512144baa (commit)
via 23a37f77f36717f60884c2b8054a9670b35e611b (commit)
via b1158db281c2107b191ddd41dd6b20f32ff04940 (commit)
via ed16f0c1f600acd7bab81796ec22fde90ecdfa7f (commit)
via 17f2b714df870dffc6320173d6a2348243ff7992 (commit)
from 2ee3d94756197297876afb64b7d5bc2789419aaf (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit e2cfa981e8a18453b4eff1edcf79497dc648fe40
Merge: 0d68681 cf56918
Author: radhika <radhika at curoverse.com>
Date: Tue Nov 1 20:47:28 2016 -0400
Merge branch 'master' into 10310-workbench-disable-apis
commit 0d686814570f4fca6d3b35d5e37a321752c8d50a
Author: radhika <radhika at curoverse.com>
Date: Tue Nov 1 20:46:31 2016 -0400
10310: invoke index and show actions only when they are not blacklisted by the api server.
diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb
index f68250b..46dad10 100644
--- a/apps/workbench/app/controllers/application_controller.rb
+++ b/apps/workbench/app/controllers/application_controller.rb
@@ -13,6 +13,7 @@ class ApplicationController < ActionController::Base
# Methods that don't require login should
# skip_around_filter :require_thread_api_token
around_filter :require_thread_api_token, except: ERROR_ACTIONS
+ before_filter :ensure_arvados_api_exists, only: [:index, :show]
before_filter :set_cache_buster
before_filter :accept_uuid_as_id_param, except: ERROR_ACTIONS
before_filter :check_user_agreements, except: ERROR_ACTIONS
@@ -213,11 +214,34 @@ class ApplicationController < ActionController::Base
end
end
+ def ensure_arvados_api_exists
+ if params['action'] == 'show'
+ method = :get
+ else
+ method = params['action'].to_sym
+ end
+
+ exists = arvados_api_exists params['controller'].to_sym, method
+ if !exists
+ @errors = ["#{params['action']} method is not supported for #{params['controller']}"]
+ return render_error(status: 404)
+ end
+ end
+
def index
find_objects_for_index if !@objects
render_index
end
+ helper_method :arvados_api_exists
+ def arvados_api_exists ctrl, method
+ if [:jobs, :job_tasks, :pipeline_instances, :pipeline_templates].include?(ctrl)
+ arvados_api_client.discovery[:resources][ctrl].andand[:methods].andand[method]
+ else
+ true
+ end
+ end
+
helper_method :next_page_offset
def next_page_offset objects=nil
if !objects
@@ -760,7 +784,11 @@ class ApplicationController < ActionController::Base
}
@@notification_tests.push lambda { |controller, current_user|
- PipelineInstance.limit(1).where(created_by: current_user.uuid).each do
+ begin
+ PipelineInstance.limit(1).where(created_by: current_user.uuid).each do
+ return nil
+ end
+ rescue ArvadosApiClient::NotFoundException
return nil
end
return lambda { |view|
@@ -856,12 +884,14 @@ class ApplicationController < ActionController::Base
def recent_processes lim
lim = 12 if lim.nil?
- cols = %w(uuid owner_uuid created_at modified_at pipeline_template_uuid name state started_at finished_at)
- pipelines = PipelineInstance.select(cols).limit(lim).order(["created_at desc"])
+ if arvados_api_exists :pipeline_instances, :index
+ cols = %w(uuid owner_uuid created_at modified_at pipeline_template_uuid name state started_at finished_at)
+ pipelines = PipelineInstance.select(cols).limit(lim).order(["created_at desc"])
+ pipelines.results.each { |pi| procs[pi] = pi.created_at }
+ end
crs = ContainerRequest.limit(lim).order(["created_at desc"]).filter([["requesting_container_uuid", "=", nil]])
procs = {}
- pipelines.results.each { |pi| procs[pi] = pi.created_at }
crs.results.each { |c| procs[c] = c.created_at }
Hash[procs.sort_by {|key, value| value}].keys.reverse.first(lim)
diff --git a/apps/workbench/app/controllers/collections_controller.rb b/apps/workbench/app/controllers/collections_controller.rb
index 20b227c..ebcc392 100644
--- a/apps/workbench/app/controllers/collections_controller.rb
+++ b/apps/workbench/app/controllers/collections_controller.rb
@@ -239,12 +239,15 @@ class CollectionsController < ApplicationController
render 'hash_matches'
return
else
- jobs_with = lambda do |conds|
- Job.limit(RELATION_LIMIT).where(conds)
- .results.sort_by { |j| j.finished_at || j.created_at }
+ if arvados_api_exists :jobs, :index
+ jobs_with = lambda do |conds|
+ Job.limit(RELATION_LIMIT).where(conds)
+ .results.sort_by { |j| j.finished_at || j.created_at }
+ end
+ @output_of = jobs_with.call(output: @object.portable_data_hash)
+ @log_of = jobs_with.call(log: @object.portable_data_hash)
end
- @output_of = jobs_with.call(output: @object.portable_data_hash)
- @log_of = jobs_with.call(log: @object.portable_data_hash)
+
@project_links = Link.limit(RELATION_LIMIT).order("modified_at DESC")
.where(head_uuid: @object.uuid, link_class: 'name').results
project_hash = Group.where(uuid: @project_links.map(&:tail_uuid)).to_hash
diff --git a/apps/workbench/app/controllers/projects_controller.rb b/apps/workbench/app/controllers/projects_controller.rb
index 16212a8..a0b48fd 100644
--- a/apps/workbench/app/controllers/projects_controller.rb
+++ b/apps/workbench/app/controllers/projects_controller.rb
@@ -53,6 +53,17 @@ class ProjectsController < ApplicationController
# It also seems to me that something like these could be used to configure the contents of the panes.
def show_pane_list
pane_list = []
+
+ procs = ["arvados#containerRequest"]
+ if arvados_api_exists(:pipeline_instances, :index)
+ procs << "arvados#pipelineInstance"
+ end
+
+ workflows = ["arvados#workflow"]
+ if arvados_api_exists(:pipeline_templates, :index)
+ workflows << "arvados#pipelineTemplate"
+ end
+
if @object.uuid != current_user.andand.uuid
pane_list << 'Description'
end
@@ -64,12 +75,12 @@ class ProjectsController < ApplicationController
pane_list <<
{
:name => 'Pipelines_and_processes',
- :filters => [%w(uuid is_a) + [%w(arvados#containerRequest arvados#pipelineInstance)]]
+ :filters => [%w(uuid is_a) + [procs]]
}
pane_list <<
{
:name => 'Pipeline_templates',
- :filters => [%w(uuid is_a) + [%w(arvados#pipelineTemplate arvados#workflow)]]
+ :filters => [%w(uuid is_a) + [workflows]]
}
pane_list <<
{
@@ -213,6 +224,8 @@ class ProjectsController < ApplicationController
@name_link_for = {}
kind_filters.each do |attr,op,val|
(val.is_a?(Array) ? val : [val]).each do |type|
+ next if(!arvados_api_exists(type.split('#')[-1].underscore.pluralize.to_sym, :index))
+
filters = @filters - kind_filters + [['uuid', 'is_a', type]]
if type == 'arvados#containerRequest'
filters = filters + [['container_requests.requesting_container_uuid', '=', nil]]
diff --git a/apps/workbench/app/controllers/work_unit_templates_controller.rb b/apps/workbench/app/controllers/work_unit_templates_controller.rb
index 6b5f114..9926d5c 100644
--- a/apps/workbench/app/controllers/work_unit_templates_controller.rb
+++ b/apps/workbench/app/controllers/work_unit_templates_controller.rb
@@ -6,8 +6,10 @@ class WorkUnitTemplatesController < ApplicationController
@filters = @filters || []
# get next page of pipeline_templates
- filters = @filters + [["uuid", "is_a", ["arvados#pipelineTemplate"]]]
- pipelines = PipelineTemplate.limit(@limit).order(["created_at desc"]).filter(filters)
+ if arvados_api_exists :pipeline_templates, :index
+ filters = @filters + [["uuid", "is_a", ["arvados#pipelineTemplate"]]]
+ pipelines = PipelineTemplate.limit(@limit).order(["created_at desc"]).filter(filters)
+ end
# get next page of workflows
filters = @filters + [["uuid", "is_a", ["arvados#workflow"]]]
diff --git a/apps/workbench/app/controllers/work_units_controller.rb b/apps/workbench/app/controllers/work_units_controller.rb
index fe6bff1..d94dcda 100644
--- a/apps/workbench/app/controllers/work_units_controller.rb
+++ b/apps/workbench/app/controllers/work_units_controller.rb
@@ -14,12 +14,16 @@ class WorkUnitsController < ApplicationController
@filters = @filters || []
# get next page of pipeline_instances
- filters = @filters + [["uuid", "is_a", ["arvados#pipelineInstance"]]]
- pipelines = PipelineInstance.limit(@limit).order(["created_at desc"]).filter(filters)
+ if arvados_api_exists :pipeline_instances, :index
+ filters = @filters + [["uuid", "is_a", ["arvados#pipelineInstance"]]]
+ pipelines = PipelineInstance.limit(@limit).order(["created_at desc"]).filter(filters)
+ end
# get next page of jobs
- filters = @filters + [["uuid", "is_a", ["arvados#job"]]]
- jobs = Job.limit(@limit).order(["created_at desc"]).filter(filters)
+ if arvados_api_exists :jobs, :index
+ filters = @filters + [["uuid", "is_a", ["arvados#job"]]]
+ jobs = Job.limit(@limit).order(["created_at desc"]).filter(filters)
+ end
# get next page of container_requests
filters = @filters + [["uuid", "is_a", ["arvados#containerRequest"]]]
commit ceabcac63dc3832f89cefd84857dd4ff658a31df
Merge: 2ee3d94 69abf05
Author: radhika <radhika at curoverse.com>
Date: Mon Oct 31 15:58:38 2016 -0400
Merge branch 'master' into 10310-workbench-disable-apis
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list