[ARVADOS] updated: 3a0d849c08f750dca1d6a40153c0107001769c6d
Git user
git at public.curoverse.com
Mon Apr 24 10:31:08 EDT 2017
Summary of changes:
.../app/assets/javascripts/edit_collection.js | 45 ++++++
.../app/assets/javascripts/selection.js.erb | 7 +
.../app/assets/stylesheets/collections.css.scss | 6 +
apps/workbench/app/helpers/application_helper.rb | 2 +
apps/workbench/app/models/workflow.rb | 8 ++
.../app/views/application/_content.html.erb | 2 +-
.../collections/_extra_tab_line_buttons.html.erb | 3 +
.../app/views/collections/_show_files.html.erb | 14 +-
.../_extra_tab_line_buttons.html.erb | 2 +-
.../app/views/workflows/_show_recent.html.erb | 65 +++++++++
.../test/integration/collection_upload_test.rb | 14 ++
.../workbench/test/integration/collections_test.rb | 67 ++++++++-
apps/workbench/test/integration/work_units_test.rb | 24 ++++
apps/workbench/test/integration_helper.rb | 15 ++
build/run-build-packages.sh | 2 -
build/run-library.sh | 2 +
build/run-tests.sh | 2 +-
doc/user/cwl/cwl-extensions.html.textile.liquid | 17 +++
docker/migrate-docker19/build.sh | 2 +-
docker/migrate-docker19/dnd.sh | 2 +-
docker/migrate-docker19/migrate.sh | 62 +++++++-
sdk/cli/bin/crunch-job | 8 +-
sdk/cli/test/binstub_arv-mount/arv-mount | 1 +
.../test/binstub_clean_fail/{mount => arv-mount} | 0
sdk/cli/test/test_crunch-job.rb | 4 +-
sdk/cwl/arvados_cwl/__init__.py | 69 ++++-----
sdk/cwl/arvados_cwl/arv-cwl-schema.yml | 31 ++++
sdk/cwl/arvados_cwl/arvcontainer.py | 12 +-
sdk/cwl/arvados_cwl/crunch_script.py | 15 +-
sdk/cwl/arvados_cwl/fsaccess.py | 32 +++--
sdk/cwl/arvados_cwl/pathmapper.py | 87 +++++------
sdk/cwl/arvados_cwl/runner.py | 2 +-
sdk/cwl/setup.py | 6 +-
sdk/cwl/tests/arvados-tests.yml | 36 +++++
sdk/cwl/tests/listing-job.yml | 3 +
sdk/cwl/tests/test_container.py | 81 ++++++++++-
sdk/cwl/tests/test_fsaccess.py | 28 ++++
sdk/cwl/tests/test_job.py | 9 +-
sdk/cwl/tests/test_pathmapper.py | 25 ++--
sdk/cwl/tests/test_submit.py | 25 +---
.../cwl/tests/tmp1/tmp2/tmp3}/.gitkeep | 0
sdk/cwl/tests/wf/listing_deep.cwl | 15 ++
sdk/cwl/tests/wf/listing_none.cwl | 15 ++
sdk/cwl/tests/wf/listing_shallow.cwl | 15 ++
sdk/cwl/tests/wf/scatter2.cwl | 1 +
sdk/cwl/tests/wf/scatter2_subwf.cwl | 5 +-
sdk/go/crunchrunner/crunchrunner.go | 66 ++++-----
sdk/go/crunchrunner/crunchrunner_test.go | 51 +++----
sdk/python/arvados/_ranges.py | 6 +-
sdk/python/arvados/arvfile.py | 90 +++++++++---
sdk/python/arvados/commands/get.py | 16 ++-
sdk/python/arvados/commands/migrate19.py | 81 +++++++++--
sdk/python/arvados/commands/put.py | 30 ++--
sdk/python/tests/test_arv_get.py | 50 +++++--
sdk/python/tests/test_arv_put.py | 6 +-
sdk/python/tests/test_arvfile.py | 159 ++++++++++++++++++++-
sdk/python/tests/test_stream.py | 7 +-
...1_add_created_by_job_task_index_to_job_tasks.rb | 5 +
...0170419173712_add_object_owner_index_to_logs.rb | 5 +
...esting_container_index_to_container_requests.rb | 5 +
services/api/db/structure.sql | 29 +++-
services/keep-web/handler.go | 14 ++
services/keep-web/handler_test.go | 31 ++++
63 files changed, 1243 insertions(+), 296 deletions(-)
create mode 100644 apps/workbench/app/assets/javascripts/edit_collection.js
create mode 100644 apps/workbench/app/views/collections/_extra_tab_line_buttons.html.erb
create mode 100644 apps/workbench/app/views/workflows/_show_recent.html.erb
create mode 100755 sdk/cli/test/binstub_arv-mount/arv-mount
rename sdk/cli/test/binstub_clean_fail/{mount => arv-mount} (100%)
create mode 100644 sdk/cwl/tests/listing-job.yml
create mode 100644 sdk/cwl/tests/test_fsaccess.py
copy {services/api/vendor/plugins => sdk/cwl/tests/tmp1/tmp2/tmp3}/.gitkeep (100%)
create mode 100644 sdk/cwl/tests/wf/listing_deep.cwl
create mode 100644 sdk/cwl/tests/wf/listing_none.cwl
create mode 100644 sdk/cwl/tests/wf/listing_shallow.cwl
create mode 100644 services/api/db/migrate/20170419173031_add_created_by_job_task_index_to_job_tasks.rb
create mode 100644 services/api/db/migrate/20170419173712_add_object_owner_index_to_logs.rb
create mode 100644 services/api/db/migrate/20170419175801_add_requesting_container_index_to_container_requests.rb
via 3a0d849c08f750dca1d6a40153c0107001769c6d (commit)
via 2b3b54becbce29efdf4c53787c4aa6d666900ef0 (commit)
via 31d31c010bb6b5170e3962fdd50c6d393cfe6076 (commit)
via b04638275cff9b393e1bc04136d44f361b999cf8 (commit)
via c39ba5193005a4e9f619901f8348f11fada88df0 (commit)
via 2333472a4f517a227278f028bbbc4e72687c0e71 (commit)
via 5180238a10bd15302a1c15b9a428f2fdeeabdf4e (commit)
via bfc9660a8b2467893baf131b20e83e76c41ae438 (commit)
via aed7702a67426dfd9d24b512c90df8e909162179 (commit)
via 9aa83ad7b4de05dd2818885ed34111d4dcf322ea (commit)
via 3c34e713aa343d56c7cea00a9c998b06dbf411d6 (commit)
via 65e339856daf4b5c3a4a810cd3a5f1a8e386dc8c (commit)
via 151df8c3b177e4971bfbdf68c87d89599dbe0812 (commit)
via e7284afa8ccb95994dcd2009015cafc6180e7187 (commit)
via 82c40aa7d30cd8e68e2a1bdc0bd8bf03cdfea029 (commit)
via 38a4b3c43a8a6cff5a00624436b8eaa5cbfbc76b (commit)
via f79536fda9dc40f480383caa69a35663702b2ba4 (commit)
via c14ae0edcb3c386e50f46218184e8dabcbc20a37 (commit)
via 04951581a941697d68cdaf9af6661c3c412f1bce (commit)
via de99c0b2effdd43d3843f15475cee84dbde8add8 (commit)
via e3ac17f8a8aa439e21a8bf56a571f91a671313f7 (commit)
via 244c47436f294638271eef637997fd00f7ca49f5 (commit)
via 67ba19113789346005aa61d4234bc33c8677a85c (commit)
via 9609f9a5a4776671f571f765a179506d26df56da (commit)
via 03188ad6eb14ee3dcd6bdf74198624c9358936c5 (commit)
via 692a66affa483483d67931a6b095b361bfd06d24 (commit)
via 243701b66914b69c9f9a94364e84713c8fcbc1da (commit)
via ea40bcae7a5d247ba8667c8c866339c2a8424464 (commit)
via cf311e8e16ba74467c77b5353afedc29b40a6a41 (commit)
via 20364bc1aed2dcad71b006f4314ba0720d20ae05 (commit)
via 72a377c59d972dd64de6e500c140738ff4b3dea6 (commit)
via fc2eaa20275146369e451da6cf14f4461c68117c (commit)
via bf55ee80e71c6503292a44fb377927f2ed908f9b (commit)
via 9cab6a09cca4153104694f0dd4644c5aa5f54b22 (commit)
via cbcb0fc8c3ebc85bf81ba9d50795d62db75efa6c (commit)
via e507e67ee50852fc44011127eb42f535fe2ae493 (commit)
via d64ed33e94700f8204ec8089c7b235cff918f9f7 (commit)
via 88c26c6b2dd8752ec1ff8196f4af11369ea6adbb (commit)
via 91f1e7009305109db0bfb6405adbd11357745bfc (commit)
via 2001423a6eb7937a689414f3fa62be5b124812c1 (commit)
via 3a88a5d8753593be89373c5b124fc5a8398457a0 (commit)
via defb299deab3be85644a7880cba492a73c2e7e62 (commit)
via 5152e0bf159151eaf04d31b788d57bebf7ab089b (commit)
via a3488ff35d219b943f3f52c07914fddf2009a140 (commit)
via 574dc168c559dd680d26be1306c4a3d8b936c6e8 (commit)
via da32ae2a73012ce55cb89b2de9a4716b2800eee1 (commit)
via 2f06f0e4d5b9ef371463c65dd5cae19f5b385a17 (commit)
via 29d77300205e485be6595d9cd9276c246f3f89b7 (commit)
via af78d389b0ad37858108b69fe5100443dc88eb4a (commit)
via 0035033f6bf6db955b116e2e0cc052bc5c79d80e (commit)
via 49a841a5179307f0c8d84f647a71f44fb2b4b26d (commit)
via bafff6941293d4cc12cda4209f1d7a2b10274ea3 (commit)
via cb47df6cc327a59efaa16170f840c2745162bc86 (commit)
via a734789218122d8ab0d8f766bac4d69c04db91bf (commit)
via 00573cf7d28472bf926e8f610a256cd991879c8b (commit)
via 4431b9c71fc08955a89d1af70de3a8d5174c6e9f (commit)
via f61c7d9445c216c2364734fb1da2bea00cfb581d (commit)
via 9ea73e2a66dcb1311732d43ea052412546c625fe (commit)
via 224098499104939fccdb07b39407937c61983687 (commit)
via d3a82af33386c99c0fd3d6471df7e6696560089c (commit)
via f70c457c73f1bff6314f8c3eb4ad7be0c44a5f1a (commit)
via cb4f8b337c56dd3e7a648dda86031ceba65ee6c4 (commit)
via d72f1d79c0a53056a7cecad6c65fd57a183059b0 (commit)
via 76e3d80bcc1548daba7bca97711684e5d68c1624 (commit)
via 777e716728a7da63ece00df7e5bb8be7f9a2a1a3 (commit)
via 828161a7527c535e5c657cd17b79cd562475fe82 (commit)
via 17f22a2b220c38ebae5247d09f8c96ceec151957 (commit)
via 38141f7261a7a2fa29f9cf3281cc46c1d845c1c3 (commit)
via acb0bd431ced4814b37f703ca63f2b39ad29c5df (commit)
via 433c48edb1523809bb8d43e4fc40ebe1a177e103 (commit)
via 0708d4a0b6db9426b9bb5addc1c592d526bd4ecd (commit)
via 25ab6bebfd55db837aa049e5c0fdb2ce28f98f30 (commit)
via 201ddbd84144a10bbcbc72a3525bb0c4e743570d (commit)
via 692ef5fd300fef4ef1b724016c1600dbcfc8a070 (commit)
via d7d00f5140b7622fdc55a4058be73b89d587b7b2 (commit)
via 7bd9af41d106590856e4d8c317502d1f4f56aefd (commit)
via 5cd99e52e342db19c1a77369a5a8c027ee04feaf (commit)
via 33b9f9bd117931bae54f45566fb77e654dcbe263 (commit)
via 6b0632d7706cbb344331bf5d291cafd541070cbb (commit)
via 820d3244a25aa8b4658ab79cd8368492aeddc739 (commit)
from a501e1d874f01acc318e7963837fbeb58a29f744 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3a0d849c08f750dca1d6a40153c0107001769c6d
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 22 02:29:00 2017 -0400
11308: Fix arvfile append mode: write() changes the file pointer.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index b9faa11..931a3a1 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -1248,10 +1248,9 @@ class ArvadosFileWriter(ArvadosFileReader):
@retry_method
def write(self, data, num_retries=None):
if self.mode[0] == "a":
- self.arvadosfile.writeto(self.size(), data, num_retries)
- else:
- self.arvadosfile.writeto(self._filepos, data, num_retries)
- self._filepos += len(data)
+ self._filepos = self.size()
+ self.arvadosfile.writeto(self._filepos, data, num_retries)
+ self._filepos += len(data)
return len(data)
@_FileLikeObjectBase._before_close
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 297fbfc..a800913 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -216,14 +216,21 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
writer = c.open("count.txt", "ab+")
self.assertEqual(writer.read(20), b"0123456789")
- writer.seek(0, os.SEEK_SET)
+ writer.seek(0, os.SEEK_SET)
writer.write("hello")
- self.assertEqual(writer.read(20), b"0123456789hello")
+ self.assertEqual(writer.read(), b"")
+ writer.seek(-5, os.SEEK_CUR)
+ self.assertEqual(writer.read(3), b"hel")
+ self.assertEqual(writer.read(), b"lo")
writer.seek(0, os.SEEK_SET)
+ self.assertEqual(writer.read(), b"0123456789hello")
+ writer.seek(0)
writer.write("world")
- self.assertEqual(writer.read(20), b"0123456789helloworld")
+ self.assertEqual(writer.read(), b"")
+ writer.seek(0)
+ self.assertEqual(writer.read(), b"0123456789helloworld")
self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
commit 2b3b54becbce29efdf4c53787c4aa6d666900ef0
Author: Tom Clegg <tom at curoverse.com>
Date: Sat Apr 22 02:26:59 2017 -0400
11308: Fix bytes vs. strings from recent merge.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 7ee77fd..ed9d55c 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -732,6 +732,7 @@ class ArvPutUploadJob(object):
state = json.dumps(self._state)
try:
new_cache = tempfile.NamedTemporaryFile(
+ mode='w+',
dir=os.path.dirname(self._cache_filename), delete=False)
self._lock_file(new_cache)
new_cache.write(state)
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index d8c1725..083b8fc 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -286,12 +286,14 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
f.flush()
cwriter = arv_put.ArvPutUploadJob([f.name])
cwriter.start(save_collection=False)
- self.assertEqual(3, cwriter.bytes_written - cwriter.bytes_skipped)
+ self.assertEqual(0, cwriter.bytes_skipped)
+ self.assertEqual(3, cwriter.bytes_written)
# Don't destroy the cache, and start another upload
cwriter_new = arv_put.ArvPutUploadJob([f.name])
cwriter_new.start(save_collection=False)
cwriter_new.destroy_cache()
- self.assertEqual(0, cwriter_new.bytes_written - cwriter_new.bytes_skipped)
+ self.assertEqual(3, cwriter_new.bytes_skipped)
+ self.assertEqual(3, cwriter_new.bytes_written)
def make_progress_tester(self):
progression = []
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 6990aa6..297fbfc 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -79,7 +79,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52",
})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- api_client=api, keep_client=keep) as c:
+ api_client=api, keep_client=keep) as c:
writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 10)
self.assertEqual(b"0123456789", writer.read(12))
@@ -101,18 +101,23 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_truncate2(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate2",
- "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
- "portable_data_hash":"272da898abdf86ddc71994835e3155f8+95"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name": "test_truncate2",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+ "replication_desired": None,
+ }, {
+ "uuid": "zzzzz-4zz18-mockcollection0",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+ "portable_data_hash": "272da898abdf86ddc71994835e3155f8+95",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
- api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 10)
- self.assertEqual("0123456789", writer.read(12))
+ self.assertEqual(b"0123456789", writer.read(12))
# extend file size
writer.truncate(12)
@@ -128,30 +133,35 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
self.assertFalse(c.modified())
def test_truncate3(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789",
- "a925576942e94b2ef57a066101b48876+10": "abcdefghij"})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
- "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
- "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ "a925576942e94b2ef57a066101b48876+10": b"abcdefghij",
+ })
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name": "test_truncate",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "replication_desired": None,
+ }, {
+ "uuid": "zzzzz-4zz18-mockcollection0",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "portable_data_hash": "7fcd0eaac3aad4c31a6a0e756475da92+52",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
- api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 20)
- self.assertEqual("0123456789ab", writer.read(12))
+ self.assertEqual(b"0123456789ab", writer.read(12))
self.assertEqual(12, writer.tell())
writer.truncate(8)
# Make sure reading off the end doesn't break
self.assertEqual(12, writer.tell())
- self.assertEqual("", writer.read(12))
+ self.assertEqual(b"", writer.read(12))
self.assertEqual(writer.size(), 8)
self.assertEqual(2, writer.seek(-10, os.SEEK_CUR))
- self.assertEqual("234567", writer.read(12))
+ self.assertEqual(b"234567", writer.read(12))
self.assertIsNone(c.manifest_locator())
self.assertTrue(c.modified())
@@ -352,10 +362,10 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({}, {})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 0)
- text = "0123456789"
+ text = b"0123456789"
writer.seek(2)
writer.write(text)
self.assertEqual(writer.size(), 12)
@@ -370,7 +380,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({}, {})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 0)
text = "0123456789"
@@ -387,28 +397,28 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({}, {})
for r in [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0], [3, 2, 0, 4, 1]]:
with Collection() as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 0)
for i in r:
w = ("%s" % i) * 10
writer.seek(i*10)
- writer.write(w)
+ writer.write(w.encode())
writer.seek(0)
- self.assertEqual(writer.read(), "00000000001111111111222222222233333333334444444444")
+ self.assertEqual(writer.read(), b"00000000001111111111222222222233333333334444444444")
def test_sparse_write4(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
api = ArvadosFileWriterTestCase.MockApi({}, {})
for r in [[0, 1, 2, 4], [4, 2, 1, 0], [2, 0, 4, 1]]:
with Collection() as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 0)
for i in r:
w = ("%s" % i) * 10
writer.seek(i*10)
- writer.write(w)
+ writer.write(w.encode())
writer.seek(0)
self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444")
commit 31d31c010bb6b5170e3962fdd50c6d393cfe6076
Merge: a501e1d b046382
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Apr 21 16:02:50 2017 -0400
11308: Merge branch 'master' into 11308-python3
Conflicts:
sdk/python/arvados/arvfile.py
sdk/python/arvados/commands/get.py
sdk/python/arvados/commands/migrate19.py
sdk/python/tests/test_arv_get.py
sdk/python/tests/test_arvfile.py
diff --cc sdk/python/arvados/arvfile.py
index c52c772,aad3ce1..b9faa11
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@@ -107,7 -96,10 +107,10 @@@ class ArvadosFileReaderBase(_FileLikeOb
pos += self._filepos
elif whence == os.SEEK_END:
pos += self.size()
- self._filepos = min(max(pos, 0), self.size())
- if pos < 0L:
++ if pos < 0:
+ raise IOError(errno.EINVAL, "Tried to seek to negative file offset.")
+ self._filepos = pos
+ return self._filepos
def tell(self):
return self._filepos
@@@ -180,16 -181,16 +192,16 @@@
data_size += len(s)
if data_size >= sizehint:
break
- return ''.join(data).splitlines(True)
+ return b''.join(data).decode().splitlines(True)
def size(self):
- raise NotImplementedError()
+ raise IOError(errno.ENOSYS, "Not implemented")
def read(self, size, num_retries=None):
- raise NotImplementedError()
+ raise IOError(errno.ENOSYS, "Not implemented")
def readfrom(self, start, size, num_retries=None):
- raise NotImplementedError()
+ raise IOError(errno.ENOSYS, "Not implemented")
class StreamFileReader(ArvadosFileReaderBase):
@@@ -971,14 -997,15 +1010,15 @@@ class ArvadosFile(object)
self.parent._my_block_manager().block_prefetch(lr.locator)
locs.add(lr.locator)
- return ''.join(data)
+ return b''.join(data)
def _repack_writes(self, num_retries):
- """Test if the buffer block has more data than actual segments.
+ """Optimize buffer block by repacking segments in file sequence.
- This happens when a buffered write over-writes a file range written in
- a previous buffered write. Re-pack the buffer block for efficiency
- and to avoid leaking information.
+ When the client makes random writes, they appear in the buffer block in
+ the sequence they were written rather than the sequence they appear in
+ the file. This makes for inefficient, fragmented manifests. Attempt
+ to optimize by repacking writes in file sequence.
"""
segs = self._segments
@@@ -1194,9 -1223,13 +1238,12 @@@ class ArvadosFileWriter(ArvadosFileRead
"""
def __init__(self, arvadosfile, mode, num_retries=None):
- super(ArvadosFileWriter, self).__init__(arvadosfile, num_retries=num_retries)
- self.mode = mode
+ super(ArvadosFileWriter, self).__init__(arvadosfile, mode=mode, num_retries=num_retries)
self.arvadosfile.add_writer(self)
+ def writable(self):
+ return True
+
@_FileLikeObjectBase._before_close
@retry_method
def write(self, data, num_retries=None):
diff --cc sdk/python/arvados/commands/get.py
index 48f076b,3bf9295..888fd39
--- a/sdk/python/arvados/commands/get.py
+++ b/sdk/python/arvados/commands/get.py
@@@ -153,11 -155,11 +159,11 @@@ def main(arguments=None, stdout=sys.std
open_flags |= os.O_EXCL
try:
if args.destination == "-":
- stdout.write(reader.manifest_text().encode())
- stdout.write(reader.manifest_text(strip=args.strip_manifest))
++ stdout.write(reader.manifest_text(strip=args.strip_manifest).encode())
else:
out_fd = os.open(args.destination, open_flags)
with os.fdopen(out_fd, 'wb') as out_file:
- out_file.write(reader.manifest_text().encode())
- out_file.write(reader.manifest_text(strip=args.strip_manifest))
++ out_file.write(reader.manifest_text(strip=args.strip_manifest).encode())
except (IOError, OSError) as error:
logger.error("can't write to '{}': {}".format(args.destination, error))
return 1
diff --cc sdk/python/arvados/commands/migrate19.py
index 724de7b,9ff0b78..91899e6
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@@ -120,11 -131,19 +133,20 @@@ def main(arguments=None)
for img in old_images:
i = uuid_to_collection[img["collection"]]
pdh = i["portable_data_hash"]
- if pdh not in already_migrated and (only_migrate is None or pdh in only_migrate):
+ if pdh not in already_migrated and pdh not in need_migrate and (only_migrate is None or pdh in only_migrate):
need_migrate[pdh] = img
with CollectionReader(i["manifest_text"]) as c:
- if list(c.values())[0].size() > biggest:
- biggest = list(c.values())[0].size()
- if c.values()[0].size() > biggest:
- biggest = c.values()[0].size()
++ size = list(c.values())[0].size()
++ if size > biggest:
++ biggest = size
+ biggest_pdh = pdh
- totalbytes += c.values()[0].size()
++ totalbytes += size
+
+
+ if args.storage_driver == "vfs":
+ will_need = (biggest*20)
+ else:
+ will_need = (biggest*2.5)
if args.print_unmigrated:
only_migrate = set()
@@@ -135,7 -154,18 +157,18 @@@
logger.info("Already migrated %i images", len(already_migrated))
logger.info("Need to migrate %i images", len(need_migrate))
logger.info("Using tempdir %s", tempfile.gettempdir())
- logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest>>20, biggest>>19)
- logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest/(2**20))
- logger.info("Total data to migrate about %i MiB", totalbytes/(2**20))
++ logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest>>20)
++ logger.info("Total data to migrate about %i MiB", totalbytes>>20)
+
+ df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()])
+ ln = df_out.splitlines()[1]
+ filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1)
+ if int(available) <= will_need:
- logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)/(2**20), will_need/(2**20))
++ logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)>>20, will_need>>20)
+ if not args.force:
+ exit(1)
+ else:
+ logger.warn("--force provided, will migrate anyway")
if args.dry_run:
return
diff --cc sdk/python/tests/test_arv_get.py
index 46c1a0e,b7f5e2b..5174647
--- a/sdk/python/tests/test_arv_get.py
+++ b/sdk/python/tests/test_arv_get.py
@@@ -1,6 -1,9 +1,8 @@@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
+from __future__ import absolute_import
+from future.utils import listitems
import io
+ import os
+ import re
import shutil
import tempfile
@@@ -32,15 -35,17 +35,18 @@@ class ArvadosGetTestCase(run_test_serve
'subdir/baz.txt' : 'baz',
}):
c = collection.Collection()
- for path, data in contents.items():
- with c.open(path, 'w') as f:
+ for path, data in listitems(contents):
+ with c.open(path, 'wb') as f:
f.write(data)
c.save_new()
- return (c.manifest_locator(), c.portable_data_hash(), c.manifest_text())
++
+ return (c.manifest_locator(),
+ c.portable_data_hash(),
+ c.manifest_text(strip=strip_manifest))
-
+
def run_get(self, args):
- self.stdout = io.BytesIO()
- self.stderr = io.BytesIO()
+ self.stdout = tutil.BytesIO()
+ self.stderr = tutil.StringIO()
return arv_get.main(args, self.stdout, self.stderr)
def test_version_argument(self):
diff --cc sdk/python/tests/test_arvfile.py
index e0f6596,20b2584..6990aa6
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@@ -99,26 -91,83 +99,88 @@@ class ArvadosFileWriterTestCase(unittes
self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
self.assertFalse(c.modified())
+
+ def test_truncate2(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+ api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate2",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+ "replication_desired":None},
+ {"uuid":"zzzzz-4zz18-mockcollection0",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+ "portable_data_hash":"272da898abdf86ddc71994835e3155f8+95"})
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 10)
+ self.assertEqual("0123456789", writer.read(12))
+
+ # extend file size
+ writer.truncate(12)
+
+ self.assertEqual(writer.size(), 12)
+ writer.seek(0, os.SEEK_SET)
+ self.assertEqual(b"0123456789\x00\x00", writer.read(12))
+
+ self.assertIsNone(c.manifest_locator())
+ self.assertTrue(c.modified())
+ c.save_new("test_truncate2")
+ self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
+ self.assertFalse(c.modified())
+
+ def test_truncate3(self):
+ keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789",
+ "a925576942e94b2ef57a066101b48876+10": "abcdefghij"})
+ api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "replication_desired":None},
+ {"uuid":"zzzzz-4zz18-mockcollection0",
+ "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+ "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
+ with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
+ api_client=api, keep_client=keep) as c:
+ writer = c.open("count.txt", "r+")
+ self.assertEqual(writer.size(), 20)
+ self.assertEqual("0123456789ab", writer.read(12))
+ self.assertEqual(12, writer.tell())
+
+ writer.truncate(8)
+
+ # Make sure reading off the end doesn't break
+ self.assertEqual(12, writer.tell())
+ self.assertEqual("", writer.read(12))
+
+ self.assertEqual(writer.size(), 8)
+ self.assertEqual(2, writer.seek(-10, os.SEEK_CUR))
+ self.assertEqual("234567", writer.read(12))
+
+ self.assertIsNone(c.manifest_locator())
+ self.assertTrue(c.modified())
+ c.save_new("test_truncate")
+ self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
+ self.assertFalse(c.modified())
+
+
+
def test_write_to_end(self):
- keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
- api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
- "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
- "replication_desired":None},
- {"uuid":"zzzzz-4zz18-mockcollection0",
- "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
- "portable_data_hash":"c5c3af76565c8efb6a806546bcf073f3+88"})
+ keep = ArvadosFileWriterTestCase.MockKeep({
+ "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+ })
+ api = ArvadosFileWriterTestCase.MockApi({
+ "name": "test_append",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
+ "replication_desired": None,
+ }, {
+ "uuid": "zzzzz-4zz18-mockcollection0",
+ "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
+ "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88",
+ })
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 10)
- writer.seek(5, os.SEEK_SET)
+ self.assertEqual(5, writer.seek(5, os.SEEK_SET))
- self.assertEqual("56789", writer.read(8))
+ self.assertEqual(b"56789", writer.read(8))
writer.seek(10, os.SEEK_SET)
writer.write("foo")
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list