[ARVADOS] updated: 02d1074b356837f1a06a25ac3dc353ecf541b444
Git user
git at public.curoverse.com
Thu May 18 14:53:55 EDT 2017
Summary of changes:
.../workbench/app/assets/javascripts/filterable.js | 2 +-
.../app/controllers/application_controller.rb | 7 ++
.../controllers/container_requests_controller.rb | 61 ++++++++--
apps/workbench/app/helpers/provenance_helper.rb | 106 ++++++++++-------
apps/workbench/app/models/container_work_unit.rb | 2 +-
.../application/_show_text_with_locators.html.erb | 29 +++--
.../_show_recent.html.erb | 22 ++--
.../container_requests/_show_recent_rows.html.erb | 36 ++++++
.../app/views/container_requests/index.html.erb | 11 ++
.../views/work_units/_component_detail.html.erb | 21 +++-
.../container_requests_controller_test.rb | 4 +-
.../test/integration/container_requests_test.rb | 28 +++++
sdk/cwl/arvados_cwl/arvjob.py | 1 +
sdk/cwl/tests/test_job.py | 2 +-
sdk/python/arvados/arvfile.py | 2 +-
sdk/python/arvados/collection.py | 4 +-
sdk/python/arvados/keep.py | 6 +-
sdk/python/tests/test_collections.py | 43 +++++++
sdk/python/tests/test_keep_locator.py | 5 +
.../arvados/v1/containers_controller.rb | 9 ++
services/api/app/models/arvados_model.rb | 12 ++
services/api/app/models/container.rb | 127 +++++++++++++++------
services/api/app/models/job.rb | 15 +--
services/api/lib/log_reuse_info.rb | 12 ++
.../test/fixtures/api_client_authorizations.yml | 7 ++
services/api/test/fixtures/containers.yml | 1 +
.../arvados/v1/containers_controller_test.rb | 23 ++++
services/api/test/unit/container_test.rb | 30 ++++-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 12 +-
.../crunch-dispatch-slurm_test.go | 91 +++++++++++----
services/crunch-run/crunchrun.go | 123 ++++++++++++++++----
services/crunch-run/crunchrun_test.go | 94 ++++++++++++++-
32 files changed, 771 insertions(+), 177 deletions(-)
copy apps/workbench/app/views/{pipeline_instances => container_requests}/_show_recent.html.erb (60%)
create mode 100644 apps/workbench/app/views/container_requests/_show_recent_rows.html.erb
create mode 100644 apps/workbench/app/views/container_requests/index.html.erb
create mode 100644 services/api/lib/log_reuse_info.rb
via 02d1074b356837f1a06a25ac3dc353ecf541b444 (commit)
via 627fbeb0406fcdd9cdc9e90a6e9529f48dc37321 (commit)
via df55660fc8748a317219043153f0bb54150086a8 (commit)
via 84906c7124ab9707b5ccca94c83471fa551d9992 (commit)
via b3f88934721fc97973b40786cae97e55e84b31d5 (commit)
via 6e5079ff9a0349c57ab2cf06398413018a921cc0 (commit)
via 78390940029ab08efecab923615c545b9da922b4 (commit)
via e9481fd0320a8f0dbecaa93eadbe3143c98bac53 (commit)
via 80407a6b686d5038bf89bbe12978ceb633e27332 (commit)
via fbc576b76977938cf7b742f9770ab90559136dc8 (commit)
via 867c69d1443b44311ab35f58304b304750c38066 (commit)
via 7a4bee5299767c8491bdfe7efaa8a936f2c2bf6a (commit)
via f0e917d2305c66cb7120125aaf6592ce2b9e3855 (commit)
via 9d33e0c4f98da668b23b85c54d20d22fe4b0f342 (commit)
via f26f70d0a60798065c5f7a5cb91b95587cc9e9ef (commit)
via aceb2d1ed239fa82fcb8bb352b632a8d92251dac (commit)
via f5fbc48810d1397df9e6244c16cf07c05162d36a (commit)
via b9b4502bcddeccd794614bf6979d643f9f350877 (commit)
via a340487a7d406e73e51479a765a3d08bdb92b8d0 (commit)
via 94b92f075dbfb60a25fbe28e5741a553ac4985fd (commit)
via eee2c470dfa879c769eccb515861419a6b900101 (commit)
via 795bf007cbe24775bd348fb40fc5c28d93c8f23d (commit)
via 34f9129a3d7d2a625455fccbd01c94fc18f6685a (commit)
via 1903e0e26b3677d9686e1d19cea897690945e3ed (commit)
via bbe86c4a80d53807b325b46dd51557a7a01670ae (commit)
via ed56c26720184ca3d07f452faf014e177f1a2c2f (commit)
via 2aaef36d5b642e8e2b0bcd9bdca0a25217e49590 (commit)
via 8794708d1e16935173f6b20c551fdd661b60883d (commit)
via c70fbca19235ebff6ab560ced462496cea7d2f72 (commit)
via 86c4a20bcf282407f92cb3f3e0a022de79107aca (commit)
via 75953fd324f2629f282f6b98e96dc1cef8bc2da4 (commit)
via 8c9b3ded23d4594d2185056032e8f0bf22dd57e1 (commit)
via 93e4548df0ba8e98589dea7cdde60a851a2d9c2f (commit)
via 9cd1bd27bada41cd13fb9813144fe694a67c2d00 (commit)
via 7f1e7f793bd1b5efa73df51b3070c5acb8fcdc82 (commit)
via bc65e5b981806594fde9f4c40bf5aa7d8e48b040 (commit)
via 7bb1d40eed9436c2f7fbd5b11106f6fd8f020441 (commit)
via a309f8c5b4842075d6c83f99a8f2a1e1016976f5 (commit)
via 3fa8a8a4142aac0fc6d3900b3f77de347b76268b (commit)
via 97d759d73e24017c9cab74325d866960389c11a5 (commit)
via 303d374c370f334426570864258df6364b9f3f0b (commit)
via f8675ad473b45387b1286c6b7a41edf36148ebac (commit)
via b29ca38e4409fe0149aea28a0e26a84d56ccca17 (commit)
via b20770d7e5a4ba0a974a7b98fce87b81aead95d9 (commit)
via 4259263d270a5e377009d20835a463551c708d82 (commit)
via e018237856b7bdd61e46468696bf3783a8f745df (commit)
via 39755f7642bf974e6e2e1cf3828b5240f489566c (commit)
from 0d5962b720872b8f4cbd6bc0a3f44d896bff7daf (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 02d1074b356837f1a06a25ac3dc353ecf541b444
Author: Tom Clegg <tom at curoverse.com>
Date: Thu May 18 14:53:38 2017 -0400
11590: Add container logging tests.
diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 9a859c6..9dbd569 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -340,6 +340,19 @@ class ContainerTest < ActiveSupport::TestCase
assert_nil reused
end
+ test "find_reusable with logging disabled" do
+ set_user_from_auth :active
+ Rails.logger.expects(:info).never
+ Container.find_reusable(REUSABLE_COMMON_ATTRS)
+ end
+
+ test "find_reusable with logging enabled" do
+ set_user_from_auth :active
+ Rails.configuration.log_reuse_decisions = true
+ Rails.logger.expects(:info).at_least(3)
+ Container.find_reusable(REUSABLE_COMMON_ATTRS)
+ end
+
test "Container running" do
c, _ = minimal_new priority: 1
commit 627fbeb0406fcdd9cdc9e90a6e9529f48dc37321
Author: Tom Clegg <tom at curoverse.com>
Date: Thu May 18 12:19:13 2017 -0400
11590: Log container reuse decisions.
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index a22d620..60911bd 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -1,3 +1,4 @@
+require 'log_reuse_info'
require 'whitelist_update'
require 'safe_json'
@@ -8,6 +9,7 @@ class Container < ArvadosModel
include WhitelistUpdate
extend CurrentApiClient
extend DbCurrentTime
+ extend LogReuseInfo
serialize :environment, Hash
serialize :mounts, Hash
@@ -175,41 +177,75 @@ class Container < ArvadosModel
end
def self.find_reusable(attrs)
- candidates = Container.
- where_serialized(:command, attrs[:command]).
- where('cwd = ?', attrs[:cwd]).
- where_serialized(:environment, attrs[:environment]).
- where('output_path = ?', attrs[:output_path]).
- where('container_image = ?', resolve_container_image(attrs[:container_image])).
- where_serialized(:mounts, resolve_mounts(attrs[:mounts])).
- where_serialized(:runtime_constraints, resolve_runtime_constraints(attrs[:runtime_constraints]))
+ log_reuse_info { "starting with #{Container.all.count} container records in database" }
+ candidates = Container.where_serialized(:command, attrs[:command])
+ log_reuse_info { "have #{candidates.count} candidates after filtering on command #{attrs[:command].inspect}" }
+
+ candidates = candidates.where('cwd = ?', attrs[:cwd])
+ log_reuse_info { "have #{candidates.count} candidates after filtering on cwd #{attrs[:cwd].inspect}" }
+
+ candidates = candidates.where_serialized(:environment, attrs[:environment])
+ log_reuse_info { "have #{candidates.count} candidates after filtering on environment #{attrs[:environment].inspect}" }
+
+ candidates = candidates.where('output_path = ?', attrs[:output_path])
+ log_reuse_info { "have #{candidates.count} candidates after filtering on output_path #{attrs[:output_path].inspect}" }
+
+ image = resolve_container_image(attrs[:container_image])
+ candidates = candidates.where('container_image = ?', image)
+ log_reuse_info { "have #{candidates.count} candidates after filtering on container_image #{image.inspect} (resolved from #{attrs[:container_image].inspect})" }
+
+ candidates = candidates.where_serialized(:mounts, resolve_mounts(attrs[:mounts]))
+ log_reuse_info { "have #{candidates.count} candidates after filtering on mounts #{attrs[:mounts].inspect}" }
+
+ candidates = candidates.where_serialized(:runtime_constraints, resolve_runtime_constraints(attrs[:runtime_constraints]))
+ log_reuse_info { "have #{candidates.count} candidates after filtering on runtime_constraints #{attrs[:runtime_constraints].inspect}" }
# Check for Completed candidates whose output and log are both readable.
select_readable_pdh = Collection.
readable_by(current_user).
select(:portable_data_hash).
to_sql
- usable = candidates.
- where(state: Complete).
- where(exit_code: 0).
- where("log IN (#{select_readable_pdh})").
- where("output IN (#{select_readable_pdh})").
- order('finished_at ASC').
- limit(1).
- first
- return usable if usable
+
+ usable = candidates.where(state: Complete, exit_code: 0)
+ log_reuse_info { "have #{usable.count} with state=Complete, exit_code=0" }
+
+ usable = usable.where("log IN (#{select_readable_pdh})")
+ log_reuse_info { "have #{usable.count} with log readable by current user #{current_user.uuid}" }
+
+ usable = usable.where("output IN (#{select_readable_pdh})")
+ log_reuse_info { "have #{usable.count} with output readable by current user #{current_user.uuid}" }
+
+ usable = usable.order('finished_at ASC').
+ limit(1).first
+ if usable
+ log_reuse_info { "done, reusing completed container #{usable.uuid}" }
+ return usable
+ end
# Check for Running candidates and return the most likely to finish sooner.
running = candidates.where(state: Running).
- order('progress desc, started_at asc').limit(1).first
- return running if not running.nil?
+ order('progress desc, started_at asc').
+ limit(1).first
+ if running
+ log_reuse_info { "done, reusing container #{running.uuid} with state=Running" }
+ return running
+ else
+ log_reuse_info { "have no containers in Running state" }
+ end
# Check for Locked or Queued ones and return the most likely to start first.
- locked_or_queued = candidates.where("state IN (?)", [Locked, Queued]).
- order('state asc, priority desc, created_at asc').limit(1).first
- return locked_or_queued if not locked_or_queued.nil?
+ locked_or_queued = candidates.
+ where("state IN (?)", [Locked, Queued]).
+ order('state asc, priority desc, created_at asc').
+ limit(1).first
+ if locked_or_queued
+ log_reuse_info { "done, reusing container #{locked_or_queued.uuid} with state=#{locked_or_queued.state}" }
+ return locked_or_queued
+ else
+ log_reuse_info { "have no containers in Locked or Queued state" }
+ end
- # No suitable candidate found.
+ log_reuse_info { "done, no reusable container found" }
nil
end
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index 5344d45..7536c3b 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -1,3 +1,4 @@
+require 'log_reuse_info'
require 'safe_json'
class Job < ArvadosModel
@@ -5,6 +6,7 @@ class Job < ArvadosModel
include KindAndEtag
include CommonApiTemplate
extend CurrentApiClient
+ extend LogReuseInfo
serialize :components, Hash
attr_protected :arvados_sdk_version, :docker_image_locator
serialize :script_parameters, Hash
@@ -198,17 +200,6 @@ class Job < ArvadosModel
filters
end
- # log_reuse_info logs whatever the given block returns, if
- # log_reuse_decisions is enabled. It accepts a block instead of a
- # string because in some cases constructing the strings involves
- # doing database queries, and we want to skip those queries when
- # logging is disabled.
- def self.log_reuse_info
- if Rails.configuration.log_reuse_decisions
- Rails.logger.info("find_reusable: " + yield)
- end
- end
-
def self.find_reusable attrs, params, filters, read_users
if filters.empty? # Translate older creation parameters into filters.
filters =
diff --git a/services/api/lib/log_reuse_info.rb b/services/api/lib/log_reuse_info.rb
new file mode 100644
index 0000000..1bf050a
--- /dev/null
+++ b/services/api/lib/log_reuse_info.rb
@@ -0,0 +1,12 @@
+module LogReuseInfo
+ # log_reuse_info logs whatever the given block returns, if
+ # log_reuse_decisions is enabled. It accepts a block instead of a
+ # string because in some cases constructing the strings involves
+ # doing database queries, and we want to skip those queries when
+ # logging is disabled.
+ def log_reuse_info
+ if Rails.configuration.log_reuse_decisions
+ Rails.logger.info("find_reusable: " + yield)
+ end
+ end
+end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list