[arvados] updated: 2.5.0-136-ga54c0f726
git repository hosting
git at public.arvados.org
Fri Feb 3 21:39:32 UTC 2023
Summary of changes:
doc/_config.yml | 2 +-
doc/admin/upgrading.html.textile.liquid | 16 +-
doc/api/methods/container_request_lifecycle.svg | 182 ++++
.../methods/container_requests.html.textile.liquid | 57 +-
doc/api/methods/containers.html.textile.liquid | 4 +-
doc/api/methods/groups.html.textile.liquid | 4 +-
doc/api/methods/links.html.textile.liquid | 8 +-
doc/api/methods/users.html.textile.liquid | 2 +-
doc/architecture/dispatchcloud.html.textile.liquid | 2 +-
doc/install/install-postgresql.html.textile.liquid | 8 +-
doc/sdk/python/cookbook.html.textile.liquid | 971 +++++++++++++++------
doc/sdk/python/sdk-python.html.textile.liquid | 88 +-
go.mod | 12 +-
go.sum | 28 +
lib/cloud/azure/azure.go | 4 +
lib/cloud/ec2/ec2.go | 209 ++++-
lib/cloud/ec2/ec2_test.go | 197 +++--
lib/cloud/interfaces.go | 11 +
lib/cloud/loopback/loopback.go | 13 +-
lib/cloud/price.go | 28 +
lib/cloud/price_test.go | 32 +
lib/config/config.default.yml | 50 +-
lib/config/export.go | 3 +-
lib/controller/localdb/login_ldap.go | 21 +-
lib/controller/router/request.go | 12 +
lib/controller/router/request_test.go | 10 +-
lib/crunchrun/background.go | 18 +-
lib/crunchrun/crunchrun.go | 137 ++-
lib/crunchrun/crunchrun_test.go | 149 +++-
lib/dispatchcloud/test/stub_driver.go | 4 +
lib/dispatchcloud/worker/pool.go | 22 +-
lib/dispatchcloud/worker/worker.go | 9 +-
sdk/cwl/arvados_cwl/arvcontainer.py | 2 +
sdk/cwl/arvados_cwl/arvworkflow.py | 2 +-
sdk/cwl/arvados_cwl/util.py | 2 +-
.../collection_per_tool_wrapper.cwl | 2 +-
sdk/cwl/tests/test_submit.py | 11 +-
sdk/cwl/tests/test_util.py | 2 +-
sdk/cwl/tests/wf/expect_upload_wrapper.cwl | 2 +-
sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl | 2 +-
sdk/cwl/tests/wf/submit_wf_wrapper.cwl | 2 +-
sdk/go/arvados/config.go | 50 +-
sdk/go/arvados/config_test.go | 17 +
sdk/go/arvados/keep_service.go | 33 +-
sdk/go/arvados/vocabulary.go | 24 +-
sdk/go/arvados/vocabulary_test.go | 15 +-
sdk/python/setup.py | 2 +-
sdk/python/tests/run_test_server.py | 1 +
.../api/app/controllers/application_controller.rb | 15 +-
.../v1/api_client_authorizations_controller.rb | 8 +-
.../arvados/v1/collections_controller.rb | 10 +-
.../app/controllers/arvados/v1/links_controller.rb | 48 +-
services/api/app/models/container.rb | 27 +-
services/api/app/models/link.rb | 56 ++
services/api/app/models/node.rb | 5 +-
services/api/config/arvados_config.rb | 1 -
.../20221219165512_dedup_permission_links.rb | 42 +
.../api/db/migrate/20221230155924_bigint_id.rb | 37 +
services/api/db/structure.sql | 55 +-
.../functional/arvados/v1/nodes_controller_test.rb | 11 -
services/api/test/integration/permissions_test.rb | 83 ++
services/api/test/unit/container_test.rb | 146 ++++
services/api/test/unit/link_test.rb | 38 +
services/api/test/unit/node_test.rb | 215 -----
services/api/test/unit/permission_test.rb | 16 +-
services/api/test/unit/user_test.rb | 6 +
services/fuse/arvados_fuse/__init__.py | 2 +
services/fuse/tests/test_mount.py | 106 +++
services/keep-balance/balance.go | 86 +-
services/keep-balance/balance_run_test.go | 140 ++-
services/keep-balance/main.go | 2 +
services/keep-balance/server.go | 2 +
72 files changed, 2870 insertions(+), 769 deletions(-)
create mode 100644 doc/api/methods/container_request_lifecycle.svg
create mode 100644 lib/cloud/price.go
create mode 100644 lib/cloud/price_test.go
create mode 100644 services/api/db/migrate/20221219165512_dedup_permission_links.rb
create mode 100644 services/api/db/migrate/20221230155924_bigint_id.rb
delete mode 100644 services/api/test/unit/node_test.rb
via a54c0f72656d883ae8f27d5074e35f60e61dce09 (commit)
via 37eb070f55b5ae0c622fb4bf0a946c9dd49b2752 (commit)
via 74fec3cd8284eae4829dad2c287588d52c621c4b (commit)
via 931c951406ef15491d8b6708417d5df735a07d31 (commit)
via e75f2cd097eb9c8e541576fadce46e09c51c7dab (commit)
via f0cc5e25d9f8b93405994f86f6eb74250622056f (commit)
via aabd5fc179249ccd76e1a4d2a6a204c5b7632c5d (commit)
via fc0445570096282a9f10b2883a6bbd6a30c7aae1 (commit)
via d5c0acee9775de737003dff7ea165a155990fb66 (commit)
via f196aa21e83d75d8fb2e77c9119a650034cb5cac (commit)
via c260a7641679548fbb3623206a9950bb5e0a4dec (commit)
via 6fa46a69150cf4284988a65794b974c1cea6a46b (commit)
via 2c7dcc41f760238c940c94e603ce571034ead6db (commit)
via b4b146dd2d1e8c816def4216951654f60fd8349f (commit)
via 1ebe303758179153f464ce1d5909d7949b738169 (commit)
via e6da4efc43f0b913964eb570c0f52336573abbe8 (commit)
via c3cea0c565585e089f571545fcbf28ee4e703671 (commit)
via 5f41a89121dea9d536ea33391ae2827432db947b (commit)
via 9b17cfdd8676c34c334452a52279ee547392ef33 (commit)
via b8a7f08e9b04868f80731452c31fa7c2ab0c4a42 (commit)
via d2c770ed6eafa87f9b488a0e41ce250af57f1bd1 (commit)
via 45828d11792f00d732b7d8e667db8b45b7a0f8b4 (commit)
via 674630a6f461527f5b26e917814736b444cb4f51 (commit)
via 267c257cb39b1fdaed42c0f2aa74861c8180e852 (commit)
via d9cb58fdfa203eee880dba91a239a1f0edf97422 (commit)
via 60146acf7ae62f4d24f75d22c0bc19537d8d26b3 (commit)
via 7da8a84dd3697b2a69fc9fd0671dbe0a996a993b (commit)
via c11aeb6b640c78feec50ebcacc52718faf5643a7 (commit)
via 1da341c2da87314a7d42a653fcdfb3d51d332ce2 (commit)
via 4d347fd29f4f7767bc7e9e4af49eda877a5f488c (commit)
via 9e674938db7e1a92497ffb0cc97cfb92ea37a885 (commit)
via ddbae2e44ddf1f0cd2209fa77d9ab808632726ce (commit)
via e94156e4f8e96bef62387910003d0f10339d8308 (commit)
via 09e8bd30b84bbeb25ab8e3630c448f6e174ac37f (commit)
via daf69c3eba0f80aa6f18b46bc4fa797a57190fc4 (commit)
via e074cb71b521f11531f3c8059bf752cfec5e02a8 (commit)
via 508a25a9f3677c913b081e79350bc76d25c76698 (commit)
via 0acc3185deb4f2215f36dcbbe66ac24b4c0ecd4a (commit)
via 12cd1df493cc0da3c6ed90b467470b336a36b42b (commit)
via be5771dd0fe41d5108d8fd72e62223b37948cf2d (commit)
via 7d3a6de4ee5afb635e561c75d4f99d1067f030f2 (commit)
via b2a40b550fbe5bcf8ae6f05a1448f46d2f4510db (commit)
via 1ddc21430ede024aa6841e6c49deafe87788c287 (commit)
via c78c9ff6942cfe6fad3f4035bbef0c779c5659fa (commit)
via 24a1cdaa5b5a3a98b47ab6802d00e7c4d2a848df (commit)
via 7fd14a123d0099c5dcfb0646125afa0b643bd6d2 (commit)
via 0775dd58eadfa35c5943bb1603a7eb179909ec4a (commit)
via 2d81dae6704870be0d70e0f25278c4c92a6e7cee (commit)
via daab28c11f17210a29b2cd3cd2b0cd4797d8e013 (commit)
via b5f762fb5f8495a75a2d94f90a8ef5364161584e (commit)
via 80215c92fabf020555726381a223e06a79f6a564 (commit)
via de36befd0c2d9d92a41e1f90f6353229d13b725a (commit)
via dfa06386290a17fb768c359de12719b170f02a72 (commit)
via 6f5e2a33099a0120f4acb2f24430fb74dcfc689a (commit)
via 13010f81771d6301e7f204976a7411f981ef30e5 (commit)
via 4901fc63c6dae531a38c0011c7e00fb743a1274d (commit)
via 9bf05670ae8a7d7132f8000237cdca449518cbc5 (commit)
via 812aac7d7666ef782fd1226ac8046f9528726356 (commit)
via 3502dcb0b4b8243e1e72e55a4177efe1745af9df (commit)
via 6a9efc57d57968d7504327c0569a3acab3925db3 (commit)
via 99a4dc213a58af8eb019ca270b0982286beeb5a2 (commit)
via 36a6bcdb43e4add604bd8ac6409eda6abc248c8c (commit)
via 48697cdb28fca37f1e420855a1bf1af2446184c7 (commit)
via e07a249c5a6683ea4139c331927c27b8d7711f45 (commit)
via c573a0ad8fa7d8fe8f5ccc058ecd14fd852deadc (commit)
via 5339ad0b20a9fe8084c3d75381aeee6f9151447c (commit)
via 5865e0ce5d98d68b71d1e4e146dcbecd35f1b296 (commit)
via c138f58b21edb574b101588f6fc61dce8a98ed3e (commit)
via c4d1131613de217719929ff05a06f19321cb5fdc (commit)
via 52471bfa0ab522d1b8bd7ef54d0a6b657fe0d4f0 (commit)
via 7f359d51c3dca6c04003d56f3f18f86cd0916bae (commit)
via 6c4c662cc5b22883ca1d4f9df9866a8b891a8e8a (commit)
via b896832bf02ded0c9142d758c2866fa4f1ec09e9 (commit)
via 6477db8a1f5ed0b2f81cf743bbea32c681c7c10c (commit)
via 5d6589c1674133460e1bf39d19dbfc6fd075e498 (commit)
via 6345bcd48d437dfd7f6b1d0ed706985f9c08fde1 (commit)
via 5ea5c3a5e509f32d607899c5584a5c811b8e2cec (commit)
via e6fe9c21160d4bbd16b5fac2c95a68bf987a6152 (commit)
via b5dfc8fb22ef85e09e7034b7899f54dfe5adb6cd (commit)
via 032450eaf50691bd981a5abb3d7fc73a75c62881 (commit)
via 239c836b35b385d1ab2384e3b1576e39a5797622 (commit)
via a98114557438e69fc7fc088d5d4c19ac2d4c5274 (commit)
via 91550c635ed37c0a79c17f276823b48433247c8a (commit)
via 7432df3ab18b66c2a5dac1f18c9e8b1d7a388558 (commit)
via 064df2d66faf61f475813863e5c29ca07ad9555a (commit)
via 073cacb8373e770454eb081a1aec39e384681b35 (commit)
via 458436270ce8fb80d421d55e192236c5ac4a225e (commit)
via 0ed815ce9bd5d266808892275d9959546cdd8227 (commit)
via ffd49a86ec939a674e141a4cab8dcb3b6baaf2e6 (commit)
via 2b8061c14002b9c58ccebbfbccd041398eb5201c (commit)
via a0bee12bba25e564b26972dfb572c5a9e13c8de6 (commit)
via 499848b567ebecadbdff1fcbbdb4683dfbf097da (commit)
via 7fd377c676394477f027869dfbdb125ac6ed26f0 (commit)
via b1b3ef2b2b23a63687b93a78ebeb1bbcb23ed5d2 (commit)
via 81a48dd1327179d7977f26f65b3d71458ca0ca9e (commit)
via d6c0418c0266f89d321f94f9fe7388c4974ebccd (commit)
via 82ac02584786a4df10c218c75dbd53b768e8599e (commit)
via 1c9c63e7cb21d97dbeae5568cb64d06884f37613 (commit)
via 37710c707e0f7b0a57a836ff8cc42dd0c5f762ff (commit)
via e219f76f6a8fc11eabb1db7bf0423a0436c40328 (commit)
via 86495b72ebfc305ee2fade8b31af73cfb02b7465 (commit)
via 6742560001ba1db4d51e5bcd5333a0b04d07ca5c (commit)
via 6241cccaeba4413883699c360bde08e0e544a10e (commit)
via 3e44f15742475b79a4bfd07ec0dcaf3ea5df8e79 (commit)
via b7f373541e5b6e7489573040df5b106b786814b5 (commit)
via 676c3f036eb69e9d9a6450d2401617bc279ae0bb (commit)
via 3794eb59917f309eb69dbfebbe09c8b594e712cc (commit)
from 4f3739a4967959db6783408d8aad2137b9ebdab5 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a54c0f72656d883ae8f27d5074e35f60e61dce09
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Feb 3 16:38:50 2023 -0500
19385: Fix tests because wrapper collection now includes original files
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 3703fc621..ceae0dc02 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -544,6 +544,8 @@ class RunnerContainer(Runner):
container_req["properties"]["template_uuid"] = self.embedded_tool.tool["id"][6:33]
else:
main = self.loadingContext.loader.idx["_:main"]
+ if main.get("id") == "_:main":
+ del main["id"]
#print(json.dumps(main, indent=2))
workflowpath = "/var/lib/cwl/workflow.json#main"
container_req["mounts"]["/var/lib/cwl/workflow.json"] = {
diff --git a/sdk/cwl/arvados_cwl/arvworkflow.py b/sdk/cwl/arvados_cwl/arvworkflow.py
index 7b10c15af..2bc823222 100644
--- a/sdk/cwl/arvados_cwl/arvworkflow.py
+++ b/sdk/cwl/arvados_cwl/arvworkflow.py
@@ -329,7 +329,7 @@ def new_upload_workflow(arvRunner, tool, job_order, project_uuid,
update_refs(result, w, tool.doc_loader.expand_url, merged_map, jobmapper, set_block_style, runtimeContext, "", "")
with col.open(w[n+1:], "wt") as f:
- #print(yamlloader.dump(result, stream=sys.stdout))
+ # yamlloader.dump(result, stream=sys.stdout)
yamlloader.dump(result, stream=f)
with col.open(os.path.join("original", w[n+1:]), "wt") as f:
diff --git a/sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl b/sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl
index e43e1af0f..847c80945 100644
--- a/sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl
+++ b/sdk/cwl/tests/collection_per_tool/collection_per_tool_wrapper.cwl
@@ -26,7 +26,7 @@
"in": [],
"label": "collection_per_tool.cwl",
"out": [],
- "run": "keep:473135c3f4af514f85027e9e348cea45+179/collection_per_tool.cwl"
+ "run": "keep:177002db236f41230905621862cc4230+367/collection_per_tool.cwl"
}
]
}
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index bb0485d8b..f82834701 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -70,7 +70,8 @@ def stubs(wfdetails=('submit_wf.cwl', None)):
uuid4.side_effect = ["df80736f-f14d-4b10-b2e3-03aa27f034bb", "df80736f-f14d-4b10-b2e3-03aa27f034b1",
"df80736f-f14d-4b10-b2e3-03aa27f034b2", "df80736f-f14d-4b10-b2e3-03aa27f034b3",
"df80736f-f14d-4b10-b2e3-03aa27f034b4", "df80736f-f14d-4b10-b2e3-03aa27f034b5",
- "df80736f-f14d-4b10-b2e3-03aa27f034b6"]
+ "df80736f-f14d-4b10-b2e3-03aa27f034b6", "df80736f-f14d-4b10-b2e3-03aa27f034b7",
+ "df80736f-f14d-4b10-b2e3-03aa27f034b8", "df80736f-f14d-4b10-b2e3-03aa27f034b9"]
determine_image_id.return_value = None
@@ -403,7 +404,7 @@ class TestSubmit(unittest.TestCase):
"class": "http://arvados.org/cwl#WorkflowRunnerResources"
}
]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:af64a47741bcc401f230eee99c6e80ff+137/wf/submit_wf_no_reuse.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:fa5fbf21deb74f9f239daa3f5bb4b902+292/wf/submit_wf_no_reuse.cwl"
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
@@ -1023,7 +1024,7 @@ class TestSubmit(unittest.TestCase):
# "arv": "http://arvados.org/cwl#",
#}
expect_container["command"] = ["--collection-cache-size=512" if v == "--collection-cache-size=256" else v for v in expect_container["command"]]
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:10f1d17b8c4aad888e1dc2a93a95ceab+145/wf/submit_wf_runner_resources.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:80b60e39456505b91d3989a1f5058b98+308/wf/submit_wf_runner_resources.cwl"
stubs.api.container_requests().create.assert_called_with(
body=JsonDiffMatcher(expect_container))
@@ -1180,7 +1181,7 @@ class TestSubmit(unittest.TestCase):
"out": [
{"id": "#main/step/out"}
],
- "run": "keep:702edbfca88e907b62234556089dbce7+119/secret_wf.cwl"
+ "run": "keep:7628e49da34b93de9f4baf08a6212817+247/secret_wf.cwl"
}
]
}
@@ -1423,7 +1424,7 @@ class TestSubmit(unittest.TestCase):
# "arv": "http://arvados.org/cwl#"
#}
- expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:743a5bcaef0604899e4f4706ac525d83+147/wf/submit_wf_process_properties.cwl"
+ expect_container["mounts"]["/var/lib/cwl/workflow.json"]["content"]["$graph"][0]["steps"][0]["run"] = "keep:df44f9dd4b9467159f210f967e45417f+312/wf/submit_wf_process_properties.cwl"
expect_container["properties"].update({
"baz": "blorp.txt",
diff --git a/sdk/cwl/tests/wf/expect_upload_wrapper.cwl b/sdk/cwl/tests/wf/expect_upload_wrapper.cwl
index a7a30e68a..05599b652 100644
--- a/sdk/cwl/tests/wf/expect_upload_wrapper.cwl
+++ b/sdk/cwl/tests/wf/expect_upload_wrapper.cwl
@@ -79,7 +79,7 @@
],
"label": "submit_wf.cwl",
"out": [],
- "run": "keep:62bd293aa703ddb1897c94711dead399+128/wf/submit_wf.cwl"
+ "run": "keep:5b4cf4181c65ad292ccba5b142d758a4+274/wf/submit_wf.cwl"
}
]
}
diff --git a/sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl b/sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl
index 3edb66d66..63031110c 100644
--- a/sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl
+++ b/sdk/cwl/tests/wf/expect_upload_wrapper_altname.cwl
@@ -79,7 +79,7 @@
],
"label": "testing 123",
"out": [],
- "run": "keep:62bd293aa703ddb1897c94711dead399+128/wf/submit_wf.cwl"
+ "run": "keep:5b4cf4181c65ad292ccba5b142d758a4+274/wf/submit_wf.cwl"
}
]
}
diff --git a/sdk/cwl/tests/wf/submit_wf_wrapper.cwl b/sdk/cwl/tests/wf/submit_wf_wrapper.cwl
index 53df05efc..3e015d65d 100644
--- a/sdk/cwl/tests/wf/submit_wf_wrapper.cwl
+++ b/sdk/cwl/tests/wf/submit_wf_wrapper.cwl
@@ -52,7 +52,7 @@
],
"label": "submit_wf.cwl",
"out": [],
- "run": "keep:62bd293aa703ddb1897c94711dead399+128/wf/submit_wf.cwl"
+ "run": "keep:5b4cf4181c65ad292ccba5b142d758a4+274/wf/submit_wf.cwl"
}
]
}
commit 37eb070f55b5ae0c622fb4bf0a946c9dd49b2752
Merge: 4f3739a49 74fec3cd8
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Feb 3 14:11:09 2023 -0500
Merge branch 'main' into 19385-cwl-fast-pack
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list