[ARVADOS] updated: a02b4f41cb495ecfea49ac0da1f6da0aec385f7d
Git user
git at public.curoverse.com
Wed Sep 14 09:42:55 EDT 2016
Summary of changes:
.../app/assets/javascripts/pipeline_instances.js | 20 ---
.../app/assets/javascripts/work_unit_log.js | 68 +++++++
.../app/controllers/work_units_controller.rb | 8 +-
apps/workbench/app/models/container_work_unit.rb | 21 +--
apps/workbench/app/models/job.rb | 3 +-
apps/workbench/app/models/pipeline_instance.rb | 2 +-
apps/workbench/app/models/proxy_work_unit.rb | 9 +
.../app/views/work_units/_show_child.html.erb | 4 +-
.../app/views/work_units/_show_component.html.erb | 64 +++----
.../test/controllers/workflows_controller_test.rb | 2 +-
.../test/helpers/fake_websocket_helper.rb | 18 ++
.../test/integration/application_layout_test.rb | 75 --------
.../test/integration/container_requests_test.rb | 4 +-
apps/workbench/test/integration/websockets_test.rb | 200 +++++++++------------
apps/workbench/test/integration/work_units_test.rb | 150 ++++++++++++++++
apps/workbench/test/integration_helper.rb | 23 ++-
apps/workbench/test/support/fake_websocket.js | 15 ++
apps/workbench/test/test_helper.rb | 10 +-
apps/workbench/test/unit/work_unit_test.rb | 14 ++
build/package-build-dockerfiles/centos6/Dockerfile | 3 +-
build/package-test-dockerfiles/centos6/Dockerfile | 3 +-
build/run-build-docker-jobs-image.sh | 6 +-
build/run-build-packages.sh | 34 ++--
doc/_config.yml | 2 +-
doc/user/cwl/cwl-style.html.textile.liquid | 168 +++++++++++++++++
sdk/cli/bin/crunch-job | 42 ++++-
sdk/cwl/arvados_cwl/__init__.py | 3 +-
sdk/cwl/setup.py | 4 +-
sdk/go/dispatch/dispatch.go | 20 ++-
.../arvados/v1/container_requests_controller.rb | 1 +
.../arvados/v1/containers_controller.rb | 10 ++
services/api/app/models/arvados_model.rb | 94 +++++-----
services/api/app/models/container.rb | 88 +++++----
services/api/app/models/job.rb | 12 --
services/api/app/models/log.rb | 18 ++
services/api/app/models/workflow.rb | 14 +-
services/api/config/routes.rb | 2 +
...20160909181442_rename_workflow_to_definition.rb | 10 ++
services/api/db/structure.sql | 8 +-
services/api/lib/eventbus.rb | 2 +-
services/api/lib/simulate_job_log.rb | 3 +
services/api/test/fixtures/container_requests.yml | 21 ++-
services/api/test/fixtures/containers.yml | 22 +--
services/api/test/fixtures/jobs.yml | 6 +-
services/api/test/fixtures/logs.yml | 4 +-
services/api/test/fixtures/workflows.yml | 16 +-
.../arvados/v1/containers_controller_test.rb | 44 ++++-
.../functional/arvados/v1/jobs_controller_test.rb | 2 +-
services/api/test/integration/permissions_test.rb | 10 +-
services/api/test/integration/websocket_test.rb | 60 +++----
services/api/test/unit/container_request_test.rb | 33 ++++
services/api/test/unit/container_test.rb | 104 +++++++++--
services/api/test/unit/job_test.rb | 22 +++
services/api/test/unit/log_test.rb | 3 +-
services/api/test/unit/permission_test.rb | 23 +++
services/api/test/unit/workflow_test.rb | 66 +++----
.../crunch-dispatch-local_test.go | 8 +-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 21 +--
services/keep-balance/balance.go | 19 +-
59 files changed, 1192 insertions(+), 549 deletions(-)
create mode 100644 apps/workbench/app/assets/javascripts/work_unit_log.js
create mode 100644 apps/workbench/test/helpers/fake_websocket_helper.rb
create mode 100644 apps/workbench/test/support/fake_websocket.js
create mode 100644 doc/user/cwl/cwl-style.html.textile.liquid
create mode 100644 services/api/db/migrate/20160909181442_rename_workflow_to_definition.rb
via a02b4f41cb495ecfea49ac0da1f6da0aec385f7d (commit)
via 6139a7b24e650aadd16857ed4383cfe4363ea45b (commit)
via 93e448118cfb620196967d05e8914fadd68b9d67 (commit)
via a47f0ade77ea89419f23b7746f51d49d24576cf3 (commit)
via 1d0bae2cf9ab3c493dc974cd06262a657eb608fb (commit)
via 62121f9b2f15cd35a6ce5b6e66ccaaae1c1af834 (commit)
via a61c6238265db896c91902f27f692f98f780393c (commit)
via ae71ca7f426bfb59e154350370c96cc2b31d6e0d (commit)
via 50b696969d71739e9fd083664de6a81db7e211b3 (commit)
via a673edc3450f2acee47e811a6197a6eb087ee275 (commit)
via faf5328304cd7c0b3ec1669c6f7f18c3f4daf25c (commit)
via f82ee19a440e31f05381511cf6c0d7932d804b9d (commit)
via c0d988e71f9657d0dea5a36c6b9904e1688e2448 (commit)
via b54478ea1b7c8aaeaf565d591f32769bcdc09b8f (commit)
via e51906ca834222fa0e85d01568507a39af4fde36 (commit)
via e636f4ec762410391aa8df7502468f98612ebb42 (commit)
via 19ad5c59b064088c58136f5387fdf029b754ee36 (commit)
via d51c716d0d547ecea25819678c6e50b647051da9 (commit)
via 2d392cf1c0726dcde178199fdafc094ee62a954a (commit)
via b758d48df3373a401cef74b0c8c99f918918afee (commit)
via 7c335bf3dc935f6ac0f1845f65063d5b40ddb5ed (commit)
via 23bfb867e544d4c470802ca775da593779bffdc1 (commit)
via 7846843453df9846c346f85c20a8d6d051066f52 (commit)
via 07ad9aa37c3b907adf85821a192041b5bb478fec (commit)
via 795125d3d998bc1fdbaf31d7d3dd048564f8e685 (commit)
via 9bad58fd3334bab5c9a09f365912a37c5f63fcdc (commit)
via 706abfc3fe0788584d930ae2297b2b01dab8cf36 (commit)
via cdc4b9c4561f09b4cfb59f4b61689a661823ad85 (commit)
via 8eaed3be04eb6356920b764f8ae370ddf8ace94c (commit)
via 5afa718bbfbe9cded6a852af216788629c3afc72 (commit)
via c0650d1d845b08668445fa5245d0e532b38afb95 (commit)
via 15a9dff29d3e5c16d936650c16b498fba1002860 (commit)
via 934694fb672f0768ec61701009a4eb2e149d6431 (commit)
via cc13854bd4675d9f1de807c38dfada0315bf3291 (commit)
via 4418adf838ac2b3d9a09d03cb87f911b323ba6d3 (commit)
via e378ed1d86cbab36f0c7093e4dc189f6184834bf (commit)
via ea8dcefcfdfea0e90ba5c45e2e692f5e59088e6b (commit)
via 19aca702ed17c7c89a5ba7d828a77105131e54db (commit)
via 96185e5347e2303d0b5215ed61bd59c0f2bc84a1 (commit)
via 3f06710d50c75c9d0c990fe05f9c1be017c55f87 (commit)
via b4a79e9c1de2e53acee8850e6d1ef3866aff0d3a (commit)
via 64d7948de09c56afdc2d7b1d9fa002a655eedf65 (commit)
via 3db1a8bbb9f14799e0aabd12cff3c980c7da0167 (commit)
via f9e3477a6a6b39ea32a0299d95971b03ebc3d50c (commit)
via 2d721e0fe63f0cf127f44a45c75201c696a8cd9e (commit)
via 5aaaed7d7b1287c0133dd66c9492d6df62790288 (commit)
via 183d42044fbb540b7db6a093d25cca9808bfb369 (commit)
via b046d1f162990361418a8e926aead5b307402a42 (commit)
via bf6dfd5785f6767d2700dadfebb8591a8595d5f3 (commit)
via 96cc8940e7926453f4728c5aec1374e7b99db201 (commit)
via 27680aebaeff734ce65f7385b0e150a39dc1653b (commit)
via dcb6eaa5012bf1eea607c61209bee18723769c24 (commit)
via 7af78c9694656d42c2bbb090e5aac62e7b3a0362 (commit)
via af0ece600054c97aae9661ed06731af47873a7ff (commit)
via 16e1811a346d7ed20999a6f6420c7a57937dbd09 (commit)
via d5ebfb1d35d2458742c545af4ee16c9f14de08bb (commit)
via 3ea8a43af615fc16cb8884a7bda78093c449542c (commit)
via ef439ae62e3b73ee81b428b64e62fe1fe13b32d0 (commit)
via feaf44094afa6774cd6583f1a121454d6754ba0c (commit)
via 8f5ef3058658c046cd25e4fff7b2f0ffafb79b77 (commit)
via 5054334a7417e35990e020aecf852a5eb3056108 (commit)
via 67afdf7bd1d53164e91e38e3b3a801ae83da778d (commit)
via 716fc68b444bb4d43ecad56bcb90e9ebf4bda854 (commit)
via dcf7f7d492830f11b58e691d127849c6781c8244 (commit)
via 3090a8d0b4020b291bf34958680e3cf4cd0b8c21 (commit)
via 730eb182b2160040a902feb09276b75a3f9c137c (commit)
via 415cb3b63e8cfafd4c6b92a8971a0cff74097337 (commit)
via 2cc397021b57dc43449730f4d13ac99d4e6be41d (commit)
via a1c3389d7fe54a7311609e7f74b5a9b3ea825c94 (commit)
via 694836fc68fbe6414d5248b48889ea3be1c33b75 (commit)
via 203cf7998836c7a00cfad2eeaf59b59f07d8e3ed (commit)
via 262a16e15c0dcfa9245e9c8fd88d383f54f51471 (commit)
via 661399e4fd26be1a932d94d0b2c419cd3a33a4f1 (commit)
via 9fd1f52e97c96f9a0a0fc76b37520828f0d9221d (commit)
via 5497c52dae7a3832e75d8e414492d48df1192ddc (commit)
via 3d53281b9e3ba0840e5e3eaeb0371e0f50a345ad (commit)
via 61ee10791b2b1f3724e6a8736c93161a625b5246 (commit)
via 87cb6b04df6f71d5bce9b0b3c1257e0b5082668d (commit)
via 2bc21107e3fa6c4ecea7efb22e4dfc7e30b3233d (commit)
via 71ec472aa11b47a58beb3b050fee38244674869e (commit)
via 579195def22d6b02f7484933d6db065bc33088d9 (commit)
via 08224a6abb60ff61f3967abc9665f50f9157791b (commit)
via 08262a4e18c0b3b65946878058e9546823b82535 (commit)
via 95e6adc8ac2c9449aba651ef0d932cd7d27aea5e (commit)
from f137e22a65cc29d00146a4735fe54d72b299984b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a02b4f41cb495ecfea49ac0da1f6da0aec385f7d
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Sep 14 10:41:11 2016 -0300
9623: Added test cases on container request to check for container reuse flow. Also added more container reuse tests to cover missing cases.
diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb
index 797fd16..7083b35 100644
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@ -395,4 +395,31 @@ class ContainerRequestTest < ActiveSupport::TestCase
assert_not_empty Container.readable_by(users(:active)).where(uuid: containers(:running).uuid)
assert_empty Container.readable_by(users(:spectator)).where(uuid: containers(:running).uuid)
end
+
+ [
+ [{"var" => "value1"}, {"var" => "value1"}],
+ [{"var" => "value1"}, {"var" => "value2"}]
+ ].each do |env1, env2|
+ test "Container request #{(env1 == env2) ? 'does' : 'does not'} reuse container when committed" do
+ common_attrs = {cwd: "test",
+ priority: 1,
+ command: ["echo", "hello"],
+ output_path: "test",
+ runtime_constraints: {"vcpus" => 4,
+ "ram" => 12000000000},
+ mounts: {"test" => {"kind" => "json"}}}
+ set_user_from_auth :active
+ cr1 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Committed,
+ environment: env1}))
+ cr2 = create_minimal_req!(common_attrs.merge({state: ContainerRequest::Uncommitted,
+ environment: env2}))
+ assert_nil cr2.container_uuid
+
+ # Update cr2 to commited state and check for container equality on both cases,
+ # when env1 and env2 are equal the same container should be assigned, and
+ # when env1 and env2 are different, cr2 container should be different.
+ cr2.update_attributes!({state: ContainerRequest::Committed})
+ assert_equal (env1 == env2), (cr1.container_uuid == cr2.container_uuid)
+ end
+ end
end
diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index fec3caa..6501f21 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -142,7 +142,32 @@ class ContainerTest < ActiveSupport::TestCase
assert_equal reused.uuid, c_recent.uuid
end
- test "find_reusable method should select running container most likely to finish sooner" do
+ test "find_reusable method should not select completed container when inconsistent outputs exist" do
+ set_user_from_auth :active
+ common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "complete"}})
+ completed_attrs = {
+ state: Container::Complete,
+ exit_code: 0,
+ log: 'test',
+ }
+
+ c_older, _ = minimal_new(common_attrs)
+ c_recent, _ = minimal_new(common_attrs)
+
+ set_user_from_auth :dispatch1
+ c_older.update_attributes!({state: Container::Locked})
+ c_older.update_attributes!({state: Container::Running})
+ c_older.update_attributes!(completed_attrs.merge({output: 'output 1'}))
+
+ c_recent.update_attributes!({state: Container::Locked})
+ c_recent.update_attributes!({state: Container::Running})
+ c_recent.update_attributes!(completed_attrs.merge({output: 'output 2'}))
+
+ reused = Container.find_reusable(common_attrs)
+ assert_nil reused
+ end
+
+ test "find_reusable method should select running container by start date" do
set_user_from_auth :active
common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running"}})
c_slower, _ = minimal_new(common_attrs)
@@ -160,9 +185,32 @@ class ContainerTest < ActiveSupport::TestCase
progress: 0.15})
reused = Container.find_reusable(common_attrs)
assert_not_nil reused
+ # Winner is the one that started first
assert_equal reused.uuid, c_faster_started_first.uuid
end
+ test "find_reusable method should select running container by progress" do
+ set_user_from_auth :active
+ common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "running2"}})
+ c_slower, _ = minimal_new(common_attrs)
+ c_faster_started_first, _ = minimal_new(common_attrs)
+ c_faster_started_second, _ = minimal_new(common_attrs)
+ set_user_from_auth :dispatch1
+ c_slower.update_attributes!({state: Container::Locked})
+ c_slower.update_attributes!({state: Container::Running,
+ progress: 0.1})
+ c_faster_started_first.update_attributes!({state: Container::Locked})
+ c_faster_started_first.update_attributes!({state: Container::Running,
+ progress: 0.15})
+ c_faster_started_second.update_attributes!({state: Container::Locked})
+ c_faster_started_second.update_attributes!({state: Container::Running,
+ progress: 0.2})
+ reused = Container.find_reusable(common_attrs)
+ assert_not_nil reused
+ # Winner is the one with most progress done
+ assert_equal reused.uuid, c_faster_started_second.uuid
+ end
+
test "find_reusable method should select locked container most likely to start sooner" do
set_user_from_auth :active
common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "locked"}})
@@ -181,6 +229,26 @@ class ContainerTest < ActiveSupport::TestCase
assert_equal reused.uuid, c_high_priority_older.uuid
end
+ test "find_reusable method should select running over failed container" do
+ set_user_from_auth :active
+ common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "failed_vs_running"}})
+ c_failed, _ = minimal_new(common_attrs)
+ c_running, _ = minimal_new(common_attrs)
+ set_user_from_auth :dispatch1
+ c_failed.update_attributes!({state: Container::Locked})
+ c_failed.update_attributes!({state: Container::Running})
+ c_failed.update_attributes!({state: Container::Complete,
+ exit_code: 42,
+ log: "test",
+ output: "test"})
+ c_running.update_attributes!({state: Container::Locked})
+ c_running.update_attributes!({state: Container::Running,
+ progress: 0.15})
+ reused = Container.find_reusable(common_attrs)
+ assert_not_nil reused
+ assert_equal reused.uuid, c_running.uuid
+ end
+
test "find_reusable method should select complete over running container" do
set_user_from_auth :active
common_attrs = REUSABLE_COMMON_ATTRS.merge({environment: {"var" => "completed_vs_running"}})
commit 6139a7b24e650aadd16857ed4383cfe4363ea45b
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Sep 14 09:39:43 2016 -0300
9623: Changed query strategy for better performance. Also fixed the case where state = complete and several containers were found with different outputs.
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index 48d9e5f..1784ae5 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -85,33 +85,27 @@ class Container < ArvadosModel
where('output_path = ?', attrs[:output_path]).
where('container_image = ?', attrs[:container_image]).
where('mounts = ?', self.deep_sort_hash(attrs[:mounts]).to_yaml).
- where('runtime_constraints = ?', self.deep_sort_hash(attrs[:runtime_constraints]).to_yaml).
- where('state in (?)', [Queued, Locked, Running, Complete]).
- reject {|c| c.state == Complete and
- (c.exit_code != 0 or c.output.nil? or c.log.nil?)}
- if candidates.empty?
- nil
- elsif candidates.count == 1
- candidates.first
- else
- # Multiple candidates found, search for the best one:
- # The most recent completed container
- winner = candidates.select {|c| c.state == Complete}.
- sort_by {|c| c.finished_at}.last
- return winner if not winner.nil?
- # The running container that's most likely to finish sooner.
- winner = candidates.select {|c| c.state == Running}.
- sort {|a, b| [b.progress, a.started_at] <=> [a.progress, b.started_at]}.first
- return winner if not winner.nil?
- # The locked container that's most likely to start sooner.
- winner = candidates.select {|c| c.state == Locked}.
- sort {|a, b| [b.priority, a.created_at] <=> [a.priority, b.created_at]}.first
- return winner if not winner.nil?
- # The queued container that's most likely to start sooner.
- winner = candidates.select {|c| c.state == Queued}.
- sort {|a, b| [b.priority, a.created_at] <=> [a.priority, b.created_at]}.first
- return winner if not winner.nil?
+ where('runtime_constraints = ?', self.deep_sort_hash(attrs[:runtime_constraints]).to_yaml)
+
+ # Check for Completed candidates that only had consistent outputs.
+ completed = candidates.where(state: Complete).where(exit_code: 0)
+ if completed.select("output").group('output').limit(2).length == 1
+ return completed.order('finished_at desc').limit(1).first
+ end
+
+ # First, check for Running candidates and return the most likely to finish sooner,
+ # then for Locked or Queued ones and return the most likely to start first.
+ [
+ [Running, 'progress desc, started_at asc'],
+ [Locked, 'priority desc, created_at asc'],
+ [Queued, 'priority desc, created_at asc']
+ ].each do |candidate_state, ordering_criteria|
+ selected = candidates.where(state: candidate_state).order(ordering_criteria).limit(1).first
+ return selected if not selected.nil?
end
+
+ # No suitable candidate found.
+ nil
end
def lock
commit 93e448118cfb620196967d05e8914fadd68b9d67
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Sep 14 09:38:57 2016 -0300
9623: Fixed progress values to be between 0.0 and 1.0
diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 9b54679..fec3caa 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -151,13 +151,13 @@ class ContainerTest < ActiveSupport::TestCase
set_user_from_auth :dispatch1
c_slower.update_attributes!({state: Container::Locked})
c_slower.update_attributes!({state: Container::Running,
- progress: 10.0})
+ progress: 0.1})
c_faster_started_first.update_attributes!({state: Container::Locked})
c_faster_started_first.update_attributes!({state: Container::Running,
- progress: 15.0})
+ progress: 0.15})
c_faster_started_second.update_attributes!({state: Container::Locked})
c_faster_started_second.update_attributes!({state: Container::Running,
- progress: 15.0})
+ progress: 0.15})
reused = Container.find_reusable(common_attrs)
assert_not_nil reused
assert_equal reused.uuid, c_faster_started_first.uuid
@@ -195,7 +195,7 @@ class ContainerTest < ActiveSupport::TestCase
output: "zzzzz-4zz18-znfnqtbbv4spc3w"})
c_running.update_attributes!({state: Container::Locked})
c_running.update_attributes!({state: Container::Running,
- progress: 1.5})
+ progress: 0.15})
reused = Container.find_reusable(common_attrs)
assert_not_nil reused
assert_equal reused.uuid, c_completed.uuid
@@ -210,7 +210,7 @@ class ContainerTest < ActiveSupport::TestCase
c_locked.update_attributes!({state: Container::Locked})
c_running.update_attributes!({state: Container::Locked})
c_running.update_attributes!({state: Container::Running,
- progress: 1.5})
+ progress: 0.15})
reused = Container.find_reusable(common_attrs)
assert_not_nil reused
assert_equal reused.uuid, c_running.uuid
commit a47f0ade77ea89419f23b7746f51d49d24576cf3
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Sep 14 07:54:16 2016 -0300
9623: Container fixtures corrections.
diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml
index 254e859..8e1a3ff 100644
--- a/services/api/test/fixtures/containers.yml
+++ b/services/api/test/fixtures/containers.yml
@@ -7,7 +7,6 @@ queued:
updated_at: 2016-01-11 11:11:11.111111111 Z
container_image: test
cwd: test
- output: test
output_path: test
command: ["echo", "hello"]
runtime_constraints:
@@ -24,7 +23,6 @@ running:
started_at: <%= 1.minute.ago.to_s(:db) %>
container_image: test
cwd: test
- output: test
output_path: test
command: ["echo", "hello"]
runtime_constraints:
@@ -42,7 +40,6 @@ running_older:
started_at: <%= 2.minute.ago.to_s(:db) %>
container_image: test
cwd: test
- output: test
output_path: test
command: ["echo", "hello"]
runtime_constraints:
@@ -76,7 +73,6 @@ completed:
finished_at: 2016-01-12 11:12:13.111111111 Z
container_image: test
cwd: test
- output: test
log: ea10d51bcf88862dbcc36eb292017dfd+45
output: zzzzz-4zz18-znfnqtbbv4spc3w
output_path: test
commit 1d0bae2cf9ab3c493dc974cd06262a657eb608fb
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Sep 14 07:50:18 2016 -0300
9623: Moved deep_sort_hash method to ArvadosModel as it's used by both Job and Container classes.
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 51673f7..7d1c30c 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -271,6 +271,18 @@ class ArvadosModel < ActiveRecord::Base
protected
+ def self.deep_sort_hash(x)
+ if x.is_a? Hash
+ x.sort.collect do |k, v|
+ [k, deep_sort_hash(v)]
+ end.to_h
+ elsif x.is_a? Array
+ x.collect { |v| deep_sort_hash(v) }
+ else
+ x
+ end
+ end
+
def ensure_ownership_path_leads_to_user
if new_record? or owner_uuid_changed?
uuid_in_path = {owner_uuid => true, uuid => true}
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index c28a604..48d9e5f 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -150,18 +150,6 @@ class Container < ArvadosModel
protected
- def self.deep_sort_hash(x)
- if x.is_a? Hash
- x.sort.collect do |k, v|
- [k, deep_sort_hash(v)]
- end.to_h
- elsif x.is_a? Array
- x.collect { |v| deep_sort_hash(v) }
- else
- x
- end
- end
-
def fill_field_defaults
self.state ||= Queued
self.environment ||= {}
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index e7d1b39..30ca7f8 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -282,18 +282,6 @@ class Job < ArvadosModel
Digest::MD5.hexdigest(Oj.dump(deep_sort_hash(h)))
end
- def self.deep_sort_hash x
- if x.is_a? Hash
- x.sort.collect do |k, v|
- [k, deep_sort_hash(v)]
- end.to_h
- elsif x.is_a? Array
- x.collect { |v| deep_sort_hash(v) }
- else
- x
- end
- end
-
def foreign_key_attributes
super + %w(output log)
end
commit 62121f9b2f15cd35a6ce5b6e66ccaaae1c1af834
Merge: f137e22 a61c623
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Tue Sep 13 18:30:43 2016 -0300
Merge branch 'master' into 9623-reuse-containers
diff --cc services/api/app/models/container.rb
index 558e8d1,11040ff..c28a604
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@@ -77,57 -76,42 +77,91 @@@ class Container < ArvadosMode
end
end
+ def self.find_reusable(attrs)
+ candidates = Container.
+ where('command = ?', attrs[:command].to_yaml).
+ where('cwd = ?', attrs[:cwd]).
+ where('environment = ?', self.deep_sort_hash(attrs[:environment]).to_yaml).
+ where('output_path = ?', attrs[:output_path]).
+ where('container_image = ?', attrs[:container_image]).
+ where('mounts = ?', self.deep_sort_hash(attrs[:mounts]).to_yaml).
+ where('runtime_constraints = ?', self.deep_sort_hash(attrs[:runtime_constraints]).to_yaml).
+ where('state in (?)', [Queued, Locked, Running, Complete]).
+ reject {|c| c.state == Complete and
+ (c.exit_code != 0 or c.output.nil? or c.log.nil?)}
+ if candidates.empty?
+ nil
+ elsif candidates.count == 1
+ candidates.first
+ else
+ # Multiple candidates found, search for the best one:
+ # The most recent completed container
+ winner = candidates.select {|c| c.state == Complete}.
+ sort_by {|c| c.finished_at}.last
+ return winner if not winner.nil?
+ # The running container that's most likely to finish sooner.
+ winner = candidates.select {|c| c.state == Running}.
+ sort {|a, b| [b.progress, a.started_at] <=> [a.progress, b.started_at]}.first
+ return winner if not winner.nil?
+ # The locked container that's most likely to start sooner.
+ winner = candidates.select {|c| c.state == Locked}.
+ sort {|a, b| [b.priority, a.created_at] <=> [a.priority, b.created_at]}.first
+ return winner if not winner.nil?
+ # The queued container that's most likely to start sooner.
+ winner = candidates.select {|c| c.state == Queued}.
+ sort {|a, b| [b.priority, a.created_at] <=> [a.priority, b.created_at]}.first
+ return winner if not winner.nil?
+ end
+ end
+
+ def lock
+ with_lock do
+ if self.state == Locked
+ raise AlreadyLockedError
+ end
+ self.state = Locked
+ self.save!
+ end
+ end
+
+ def unlock
+ with_lock do
+ if self.state == Queued
+ raise InvalidStateTransitionError
+ end
+ self.state = Queued
+ self.save!
+ end
+ end
+
+ def self.readable_by(*users_list)
+ if users_list.select { |u| u.is_admin }.any?
+ return self
+ end
+ user_uuids = users_list.map { |u| u.uuid }
+ uuid_list = user_uuids + users_list.flat_map { |u| u.groups_i_can(:read) }
+ uuid_list.uniq!
+ permitted = "(SELECT head_uuid FROM links WHERE link_class='permission' AND tail_uuid IN (:uuids))"
+ joins(:container_requests).
+ where("container_requests.uuid IN #{permitted} OR "+
+ "container_requests.owner_uuid IN (:uuids)",
+ uuids: uuid_list)
+ end
+
protected
+ def self.deep_sort_hash(x)
+ if x.is_a? Hash
+ x.sort.collect do |k, v|
+ [k, deep_sort_hash(v)]
+ end.to_h
+ elsif x.is_a? Array
+ x.collect { |v| deep_sort_hash(v) }
+ else
+ x
+ end
+ end
+
def fill_field_defaults
self.state ||= Queued
self.environment ||= {}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list