[ARVADOS] updated: 6599088b45103087b4be743fd51a8330e694e57f
Git user
git at public.curoverse.com
Wed Jan 4 10:20:34 EST 2017
Summary of changes:
.../app/controllers/projects_controller.rb | 10 +-
.../test/controllers/projects_controller_test.rb | 9 +-
backports/deb-fuse/fpm-info.sh | 5 +
backports/deb-libfuse-dev/fpm-info.sh | 5 +
backports/python-llfuse/fpm-info.sh | 10 +-
backports/rpm-fuse-devel/fpm-info.sh | 5 +
backports/rpm-fuse/fpm-info.sh | 5 +
build/package-build-dockerfiles/Makefile | 10 +-
build/package-build-dockerfiles/centos6/Dockerfile | 37 ----
build/package-build-dockerfiles/debian7/Dockerfile | 20 --
build/package-test-dockerfiles/centos6/Dockerfile | 22 --
.../centos6/localrepo.repo | 5 -
build/package-test-dockerfiles/centos7/Dockerfile | 4 +-
build/package-test-dockerfiles/debian7/Dockerfile | 14 --
build/package-test-dockerfiles/debian8/Dockerfile | 7 +-
.../package-test-dockerfiles/ubuntu1204/Dockerfile | 9 +-
.../package-test-dockerfiles/ubuntu1404/Dockerfile | 9 +-
build/package-testing/test-packages-centos6.sh | 1 -
build/run-build-docker-images.sh | 2 +-
build/run-build-packages-all-targets.sh | 2 +-
build/run-build-packages-one-target.sh | 11 +-
build/run-build-packages-sso.sh | 9 +-
build/run-build-packages.sh | 84 +------
build/run-build-test-packages-one-target.sh | 4 +-
build/run-library.sh | 2 +-
build/run-tests.sh | 246 +++++++++++----------
sdk/cli/bin/crunch-job | 2 +-
sdk/cwl/arvados_cwl/__init__.py | 54 ++++-
sdk/cwl/arvados_cwl/arvcontainer.py | 42 ++--
sdk/cwl/arvados_cwl/arvdocker.py | 73 +++---
sdk/cwl/arvados_cwl/arvjob.py | 17 +-
sdk/cwl/arvados_cwl/arvworkflow.py | 37 ++--
sdk/cwl/arvados_cwl/crunch_script.py | 10 +-
sdk/cwl/arvados_cwl/done.py | 29 ++-
sdk/cwl/arvados_cwl/pathmapper.py | 29 +--
sdk/cwl/arvados_cwl/runner.py | 69 +++---
sdk/cwl/setup.py | 5 +-
sdk/cwl/tests/test_container.py | 9 +-
sdk/cwl/tests/test_job.py | 7 +-
sdk/cwl/tests/test_submit.py | 19 +-
sdk/cwl/tests/wf/expect_packed.cwl | 43 ++--
sdk/cwl/tests/wf/scatter2_subwf.cwl | 50 +++--
sdk/go/arvados/collection.go | 4 +-
sdk/go/arvados/resource_list.go | 11 +-
sdk/go/keepclient/keepclient.go | 4 +
sdk/go/keepclient/keepclient_test.go | 31 ++-
.../arvados/v1/collections_controller.rb | 30 ++-
services/api/app/middlewares/rack_socket.rb | 20 +-
services/api/app/models/arvados_model.rb | 2 +-
services/api/app/models/collection.rb | 98 ++++++--
services/api/app/models/container.rb | 2 +-
services/api/config/application.default.yml | 9 +-
services/api/config/routes.rb | 1 +
...61222153434_split_expiry_to_trash_and_delete.rb | 42 ++++
services/api/db/structure.sql | 46 +++-
services/api/lib/create_superuser_token.rb | 15 +-
services/api/lib/eventbus.rb | 1 +
services/api/lib/load_param.rb | 3 +-
services/api/lib/sweep_trashed_collections.rb | 34 +++
services/api/test/fixtures/collections.yml | 41 +++-
.../arvados/v1/collections_controller_test.rb | 65 ++++++
services/api/test/integration/select_test.rb | 7 +
services/api/test/unit/collection_test.rb | 140 +++++++++++-
services/api/test/unit/container_request_test.rb | 7 +
.../api/test/unit/create_superuser_token_test.rb | 21 +-
services/arv-web/arv-web.py | 2 +-
services/crunch-run/crunchrun.go | 4 +-
services/fuse/arvados_fuse/__init__.py | 71 ++++--
services/fuse/tests/test_mount.py | 12 +-
services/keep-balance/balance_run_test.go | 5 +-
services/keep-balance/collection.go | 22 +-
services/keep-balance/keep-balance.service | 2 +-
services/keep-balance/main.go | 7 +-
services/keep-balance/usage.go | 2 +-
services/keepstore/azure_blob_volume.go | 108 ++++++++-
services/keepstore/azure_blob_volume_test.go | 73 +++++-
services/keepstore/pipe_adapters.go | 89 ++++++++
services/keepstore/s3_volume.go | 85 +++----
services/keepstore/stats_ticker.go | 50 +++++
services/keepstore/volume.go | 12 +
services/keepstore/volume_unix.go | 92 +++++---
services/keepstore/volume_unix_test.go | 50 ++++-
tools/arvbox/lib/arvbox/docker/common.sh | 9 +-
tools/arvbox/lib/arvbox/docker/createusers.sh | 4 +-
84 files changed, 1628 insertions(+), 747 deletions(-)
create mode 100644 backports/deb-fuse/fpm-info.sh
create mode 100644 backports/deb-libfuse-dev/fpm-info.sh
create mode 100644 backports/rpm-fuse-devel/fpm-info.sh
create mode 100644 backports/rpm-fuse/fpm-info.sh
delete mode 100644 build/package-build-dockerfiles/centos6/Dockerfile
delete mode 100644 build/package-build-dockerfiles/debian7/Dockerfile
delete mode 100644 build/package-test-dockerfiles/centos6/Dockerfile
delete mode 100644 build/package-test-dockerfiles/centos6/localrepo.repo
delete mode 100644 build/package-test-dockerfiles/debian7/Dockerfile
delete mode 120000 build/package-testing/test-packages-centos6.sh
create mode 100644 services/api/db/migrate/20161222153434_split_expiry_to_trash_and_delete.rb
create mode 100644 services/api/lib/sweep_trashed_collections.rb
create mode 100644 services/keepstore/pipe_adapters.go
create mode 100644 services/keepstore/stats_ticker.go
via 6599088b45103087b4be743fd51a8330e694e57f (commit)
via a6df6abd009da14b6fa0d111a3ae2f41af9208d2 (commit)
via 8fc563f15301baa0a59bfae748e0c8cd2d5eefda (commit)
via ce42f38437ccda92142eda2e49d77226f78bd82d (commit)
via 002042bd4a1eb6992313ef821053c4a38fd3b4a8 (commit)
via e9661d2d3e117fcb9b48848757b6dfe65210ba43 (commit)
via e7f29d3e4baa5c5d857a1a6f396b7c8047c45c82 (commit)
via bee1bf42b29780bcb28fd26edd78d384e1b578ac (commit)
via 3fa0560c1c4892636c8c06c70b0f16190fac59b1 (commit)
via 36352ce56096e4406344c531147236c355c73235 (commit)
via edf15f609d4eeccb2f693844e4df030f868dbd69 (commit)
via 991a32dd18fb9be2862baca9d6b374d09ae8bc38 (commit)
via 3cbe55d440788b0b9b1a9d9e642103929b57e8fd (commit)
via 3121f0dfb9262ccd50d0637c9f7cedf9191f69bf (commit)
via b2da02379210a1d43914b36f412f76e639203a81 (commit)
via 590e8c5e913576f947a7e28e45166770a8bd619e (commit)
via 2aa6adffd7a89601510b39481a071ba03213f6fd (commit)
via 1767cc94ddd427c6610c82e1b27f6a9f6793b39a (commit)
via d0d60bbbe1c60516334e5d56f3c923b79e611b21 (commit)
via 9048749c1419cf5e130a4b5d992a2b9c5bafd9cf (commit)
via b405f0f487f35f62d8362dc06981b83176b77d44 (commit)
via 4c76ae6b65612579e465a2862f3c2bdfea06b43a (commit)
via 8d0b0fc4046a01a1aeaebb177c3b740237728c25 (commit)
via a12864a31d5569c74ed32157d5fe928a1c2563b7 (commit)
via 29246b83f1b1c66eb12ed53c844ef37ee0b77568 (commit)
via 3f556ca1b44b7e01874bd172abbb7cb3df0615db (commit)
via b4d9dfe1e7acb1f45c2cc699020bf9299a0db5c9 (commit)
via b3e5ea60bdecb41fbf954b67ab859dc4542d0c1a (commit)
via b9df977af1ca8d7fe942cbe13956fa19ed88b91d (commit)
via 1fa9990d33e5cde82d5986b382851b8ea0314553 (commit)
via 9c7a9a6e4660b3e7cc6419e5818ffaa3a78b6d19 (commit)
via 7d6cb0405504a536a88aa8467e474f7fd13299a7 (commit)
via d02ba63331267f782e0bc5868ca4aab456e0f2b4 (commit)
via e367f2eea8974cc137b2c74bfbfcf0e5d334f15f (commit)
via 48d810686a66ac2d554d4f5696fda9a61904992c (commit)
via 7fc1b26a40065518dd5cfc8eaf6259fa1288bd92 (commit)
via e99a8f26537ef2356be5fc479bfe8c4f716c8d22 (commit)
via bfe7a62407aa0a6bc078659f259468f9b4105581 (commit)
via cc33861d2b29e1a0bd63f3c9b9322bd86e3fb5e4 (commit)
via 973c9d87c964cd517704b8f0ba487a644f5d635d (commit)
via d1f91106df53f5333acdfc5ec299c852d8ddfb46 (commit)
via 0d6b95b2e39407051a04ae246612ff4e2a2af85e (commit)
via 25c212007d4c6ef14cf730526246bc8876700bb6 (commit)
via 5627b1bf40d759cef1670afa6875005d9ca7e691 (commit)
via 31d8056dbcaf236299729694b3e63882cb42dde2 (commit)
via 42759ae4288f20646ae64bf7d09db26b713d531b (commit)
via 31084ffe95eddee3469674b6f719e00e94de9b5d (commit)
via ea9d53175a0e7670ceb9bbeaf66cd6c89baa09dd (commit)
via a166056578bc02193e1a83f79b2477c9ccd98cc2 (commit)
via b3f306edf87bf1fe406f11b5c99947d73599f84f (commit)
via f82d2a3731310b55420f7dcfe9a7ad28c3b338f1 (commit)
via 4030410ea30f122acb7e67844b1adcd7ed7107fa (commit)
via fabbab6fb71b4733260687cd58a455b16c00948a (commit)
via e821b673b8290ab848b0a071abf6c389f3f906ec (commit)
via 568c7abf660b7a68f70b6ea47ae2e7352233f053 (commit)
via 7f9a68fbef2de18f274437b6161cd7269d87f04e (commit)
via a4f7583ef9d76101415bace9b250df983ba7212d (commit)
via 5af0f99a1dc3d6dbc9066c2649272abf27482ebf (commit)
via 970835342e02cb7a97f228f67602b7f847ab6875 (commit)
via 1f5fee78438f06ec9531939247321dfb30af2b1a (commit)
via 25275859b01a1f8c8bff018878842d194d99c173 (commit)
via b9f7a8693579045f05d142dba8bffd2c5660dfce (commit)
via 997479d1408139e96ecdb42a60b4f727f814f6c9 (commit)
via da2a4d9c13b80066400161cb7638425828de18d3 (commit)
via 1847f3a94d355eed42e0d6ba7b5f929b7af7d143 (commit)
via af967503ee1c446ebed6451462f5ba725bcbbdf7 (commit)
via 1e19bc12a83dacf4a40d1c0d358aae48b6588a58 (commit)
via 0b3b1e9d91f06966cbeecd3d4a11e1fb7e8d8434 (commit)
via 5c7c672a4cd0d8c5bd0e6c5218062214eb5751f3 (commit)
via e675daba8125b20e45e0c121f1fdcac85fb4b360 (commit)
via 91b824a77fe19242de50e838f9f8c4fb30907fa3 (commit)
via 2ecd4749bc9b3f8be8cf41864948108068e187d8 (commit)
via 8d4ec10fc26d93d282845c789cd61da79e4b2836 (commit)
via 692127b79071f419ef39fb4594a4193f8e4a7a37 (commit)
via b27f1106beb0fb321ee123501afb5ccf2354a363 (commit)
via fe060158c380fe92635cae24a12eb43680021169 (commit)
via 6cec281e8653731602ab871cc41ddd21ca8182ab (commit)
via 4ee5a6efb8b01afddddaa8ce5ed7d0de42a287d6 (commit)
via eebb0457f1bb69ec2084cbefcbff8c19e404d556 (commit)
via 0aa01993bc8533d417dc510b1860a4a583e093b4 (commit)
via cdd8dc7bc4cca452e25c5b014e5f2bb592fb31ce (commit)
via f6e68adfecb95e42e984951794c96f54166b7f13 (commit)
via 90633a9dd56502566731991dd4084a57e0366064 (commit)
from 05f94a7837527f79df122e9942eaab166e987b15 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6599088b45103087b4be743fd51a8330e694e57f
Merge: 05f94a7 a6df6ab
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Jan 4 12:19:39 2017 -0300
10223: Merge branch 'master' into 10223-cr-set-output-name
diff --cc sdk/cwl/arvados_cwl/arvcontainer.py
index edf8c26,d6e3098..c1e1a26
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@@ -234,10 -245,9 +245,10 @@@ class RunnerContainer(Runner)
}
container_req["properties"]["template_uuid"] = wfuuid
- command = ["arvados-cwl-runner", "--local", "--api=containers"]
+ command = ["arvados-cwl-runner", "--local", "--api=containers", "--no-log-timestamps"]
if self.output_name:
command.append("--output-name=" + self.output_name)
+ container_req["output_name"] = self.output_name
if self.output_tags:
command.append("--output-tags=" + self.output_tags)
diff --cc sdk/cwl/tests/test_submit.py
index 17b864b,cdeb6df..14ae7b9
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@@ -468,8 -469,9 +469,10 @@@ class TestSubmit(unittest.TestCase)
except:
logging.exception("")
- stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', "--output-name="+output_name, '--enable-reuse', '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
+ stubs.expect_container_spec["command"] = ['arvados-cwl-runner', '--local', '--api=containers', '--no-log-timestamps',
+ "--output-name="+output_name, '--enable-reuse',
+ '/var/lib/cwl/workflow/submit_wf.cwl', '/var/lib/cwl/cwl.input.json']
+ stubs.expect_container_spec["output_name"] = output_name
expect_container = copy.deepcopy(stubs.expect_container_spec)
stubs.api.container_requests().create.assert_called_with(
diff --cc services/api/db/structure.sql
index fb32872,6ef03cf..d4b3713
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@@ -2709,6 -2731,4 +2732,9 @@@ INSERT INTO schema_migrations (version
INSERT INTO schema_migrations (version) VALUES ('20161213172944');
-INSERT INTO schema_migrations (version) VALUES ('20161222153434');
++INSERT INTO schema_migrations (version) VALUES ('20161222153434');
++
+INSERT INTO schema_migrations (version) VALUES ('20161223090712');
+
- INSERT INTO schema_migrations (version) VALUES ('20170102153111');
++INSERT INTO schema_migrations (version) VALUES ('20170102153111');
++
diff --cc services/api/test/unit/container_request_test.rb
index 1f65c01,9e28f7b..78dc3e3
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@@ -501,45 -506,12 +506,47 @@@ class ContainerRequestTest < ActiveSupp
end
cr.reload
+ cr2.reload
assert_equal "Final", cr.state
assert_equal prev_container_uuid, cr.container_uuid
+ assert_not_equal cr2.container_uuid, cr.container_uuid
end
+ test "Set up output collection on finalized container request" do
+ set_user_from_auth :active
+ output_name = 'Test output collection'
+ cr = create_minimal_req!(priority: 1,
+ state: ContainerRequest::Committed,
+ output_name: output_name)
+ cr.reload
+ act_as_system_user do
+ c = Container.find_by_uuid(cr.container_uuid)
+ c.update_attributes!(state: Container::Locked)
+ c.update_attributes!(state: Container::Running)
+ c.update_attributes!(state: Container::Complete,
+ exit_code: 0,
+ output: '1f4b0bc7583c2a7f9102c395f4ffc5e3+45',
+ log: 'fa7aeb5140e2848d39b416daeef4ffc5+45')
+ end
+ cr.reload
+ output_coll = Collection.find_by_uuid(cr.output_uuid)
+ assert_not_nil output_coll
+ assert_equal output_coll.name, output_name
+ # Now ask for a second CR, equal to the first one so it will reuse the container
+ cr2 = create_minimal_req!(priority: 1,
+ state: ContainerRequest::Uncommitted,
+ output_name: output_name)
+ cr2.state = ContainerRequest::Committed
+ cr2.save
+ assert_equal ContainerRequest::Final, cr2.state
+ output_coll_2 = Collection.find_by_uuid(cr2.output_uuid)
+ # Make sure the resulting output collection name include the original name
+ # plus the date
+ assert_not_equal output_coll_2.name, output_coll.name
+ assert output_coll_2.name.include? output_name
+ assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z/, output_coll_2.name
+ end
+
test "Finalize committed request when reusing a finished container" do
set_user_from_auth :active
cr = create_minimal_req!(priority: 1, state: ContainerRequest::Committed)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list