[ARVADOS] updated: 3a0d849c08f750dca1d6a40153c0107001769c6d

Git user git at public.curoverse.com
Mon Apr 24 10:31:08 EDT 2017


Summary of changes:
 .../app/assets/javascripts/edit_collection.js      |  45 ++++++
 .../app/assets/javascripts/selection.js.erb        |   7 +
 .../app/assets/stylesheets/collections.css.scss    |   6 +
 apps/workbench/app/helpers/application_helper.rb   |   2 +
 apps/workbench/app/models/workflow.rb              |   8 ++
 .../app/views/application/_content.html.erb        |   2 +-
 .../collections/_extra_tab_line_buttons.html.erb   |   3 +
 .../app/views/collections/_show_files.html.erb     |  14 +-
 .../_extra_tab_line_buttons.html.erb               |   2 +-
 .../app/views/workflows/_show_recent.html.erb      |  65 +++++++++
 .../test/integration/collection_upload_test.rb     |  14 ++
 .../workbench/test/integration/collections_test.rb |  67 ++++++++-
 apps/workbench/test/integration/work_units_test.rb |  24 ++++
 apps/workbench/test/integration_helper.rb          |  15 ++
 build/run-build-packages.sh                        |   2 -
 build/run-library.sh                               |   2 +
 build/run-tests.sh                                 |   2 +-
 doc/user/cwl/cwl-extensions.html.textile.liquid    |  17 +++
 docker/migrate-docker19/build.sh                   |   2 +-
 docker/migrate-docker19/dnd.sh                     |   2 +-
 docker/migrate-docker19/migrate.sh                 |  62 +++++++-
 sdk/cli/bin/crunch-job                             |   8 +-
 sdk/cli/test/binstub_arv-mount/arv-mount           |   1 +
 .../test/binstub_clean_fail/{mount => arv-mount}   |   0
 sdk/cli/test/test_crunch-job.rb                    |   4 +-
 sdk/cwl/arvados_cwl/__init__.py                    |  69 ++++-----
 sdk/cwl/arvados_cwl/arv-cwl-schema.yml             |  31 ++++
 sdk/cwl/arvados_cwl/arvcontainer.py                |  12 +-
 sdk/cwl/arvados_cwl/crunch_script.py               |  15 +-
 sdk/cwl/arvados_cwl/fsaccess.py                    |  32 +++--
 sdk/cwl/arvados_cwl/pathmapper.py                  |  87 +++++------
 sdk/cwl/arvados_cwl/runner.py                      |   2 +-
 sdk/cwl/setup.py                                   |   6 +-
 sdk/cwl/tests/arvados-tests.yml                    |  36 +++++
 sdk/cwl/tests/listing-job.yml                      |   3 +
 sdk/cwl/tests/test_container.py                    |  81 ++++++++++-
 sdk/cwl/tests/test_fsaccess.py                     |  28 ++++
 sdk/cwl/tests/test_job.py                          |   9 +-
 sdk/cwl/tests/test_pathmapper.py                   |  25 ++--
 sdk/cwl/tests/test_submit.py                       |  25 +---
 .../cwl/tests/tmp1/tmp2/tmp3}/.gitkeep             |   0
 sdk/cwl/tests/wf/listing_deep.cwl                  |  15 ++
 sdk/cwl/tests/wf/listing_none.cwl                  |  15 ++
 sdk/cwl/tests/wf/listing_shallow.cwl               |  15 ++
 sdk/cwl/tests/wf/scatter2.cwl                      |   1 +
 sdk/cwl/tests/wf/scatter2_subwf.cwl                |   5 +-
 sdk/go/crunchrunner/crunchrunner.go                |  66 ++++-----
 sdk/go/crunchrunner/crunchrunner_test.go           |  51 +++----
 sdk/python/arvados/_ranges.py                      |   6 +-
 sdk/python/arvados/arvfile.py                      |  90 +++++++++---
 sdk/python/arvados/commands/get.py                 |  16 ++-
 sdk/python/arvados/commands/migrate19.py           |  81 +++++++++--
 sdk/python/arvados/commands/put.py                 |  30 ++--
 sdk/python/tests/test_arv_get.py                   |  50 +++++--
 sdk/python/tests/test_arv_put.py                   |   6 +-
 sdk/python/tests/test_arvfile.py                   | 159 ++++++++++++++++++++-
 sdk/python/tests/test_stream.py                    |   7 +-
 ...1_add_created_by_job_task_index_to_job_tasks.rb |   5 +
 ...0170419173712_add_object_owner_index_to_logs.rb |   5 +
 ...esting_container_index_to_container_requests.rb |   5 +
 services/api/db/structure.sql                      |  29 +++-
 services/keep-web/handler.go                       |  14 ++
 services/keep-web/handler_test.go                  |  31 ++++
 63 files changed, 1243 insertions(+), 296 deletions(-)
 create mode 100644 apps/workbench/app/assets/javascripts/edit_collection.js
 create mode 100644 apps/workbench/app/views/collections/_extra_tab_line_buttons.html.erb
 create mode 100644 apps/workbench/app/views/workflows/_show_recent.html.erb
 create mode 100755 sdk/cli/test/binstub_arv-mount/arv-mount
 rename sdk/cli/test/binstub_clean_fail/{mount => arv-mount} (100%)
 create mode 100644 sdk/cwl/tests/listing-job.yml
 create mode 100644 sdk/cwl/tests/test_fsaccess.py
 copy {services/api/vendor/plugins => sdk/cwl/tests/tmp1/tmp2/tmp3}/.gitkeep (100%)
 create mode 100644 sdk/cwl/tests/wf/listing_deep.cwl
 create mode 100644 sdk/cwl/tests/wf/listing_none.cwl
 create mode 100644 sdk/cwl/tests/wf/listing_shallow.cwl
 create mode 100644 services/api/db/migrate/20170419173031_add_created_by_job_task_index_to_job_tasks.rb
 create mode 100644 services/api/db/migrate/20170419173712_add_object_owner_index_to_logs.rb
 create mode 100644 services/api/db/migrate/20170419175801_add_requesting_container_index_to_container_requests.rb

       via  3a0d849c08f750dca1d6a40153c0107001769c6d (commit)
       via  2b3b54becbce29efdf4c53787c4aa6d666900ef0 (commit)
       via  31d31c010bb6b5170e3962fdd50c6d393cfe6076 (commit)
       via  b04638275cff9b393e1bc04136d44f361b999cf8 (commit)
       via  c39ba5193005a4e9f619901f8348f11fada88df0 (commit)
       via  2333472a4f517a227278f028bbbc4e72687c0e71 (commit)
       via  5180238a10bd15302a1c15b9a428f2fdeeabdf4e (commit)
       via  bfc9660a8b2467893baf131b20e83e76c41ae438 (commit)
       via  aed7702a67426dfd9d24b512c90df8e909162179 (commit)
       via  9aa83ad7b4de05dd2818885ed34111d4dcf322ea (commit)
       via  3c34e713aa343d56c7cea00a9c998b06dbf411d6 (commit)
       via  65e339856daf4b5c3a4a810cd3a5f1a8e386dc8c (commit)
       via  151df8c3b177e4971bfbdf68c87d89599dbe0812 (commit)
       via  e7284afa8ccb95994dcd2009015cafc6180e7187 (commit)
       via  82c40aa7d30cd8e68e2a1bdc0bd8bf03cdfea029 (commit)
       via  38a4b3c43a8a6cff5a00624436b8eaa5cbfbc76b (commit)
       via  f79536fda9dc40f480383caa69a35663702b2ba4 (commit)
       via  c14ae0edcb3c386e50f46218184e8dabcbc20a37 (commit)
       via  04951581a941697d68cdaf9af6661c3c412f1bce (commit)
       via  de99c0b2effdd43d3843f15475cee84dbde8add8 (commit)
       via  e3ac17f8a8aa439e21a8bf56a571f91a671313f7 (commit)
       via  244c47436f294638271eef637997fd00f7ca49f5 (commit)
       via  67ba19113789346005aa61d4234bc33c8677a85c (commit)
       via  9609f9a5a4776671f571f765a179506d26df56da (commit)
       via  03188ad6eb14ee3dcd6bdf74198624c9358936c5 (commit)
       via  692a66affa483483d67931a6b095b361bfd06d24 (commit)
       via  243701b66914b69c9f9a94364e84713c8fcbc1da (commit)
       via  ea40bcae7a5d247ba8667c8c866339c2a8424464 (commit)
       via  cf311e8e16ba74467c77b5353afedc29b40a6a41 (commit)
       via  20364bc1aed2dcad71b006f4314ba0720d20ae05 (commit)
       via  72a377c59d972dd64de6e500c140738ff4b3dea6 (commit)
       via  fc2eaa20275146369e451da6cf14f4461c68117c (commit)
       via  bf55ee80e71c6503292a44fb377927f2ed908f9b (commit)
       via  9cab6a09cca4153104694f0dd4644c5aa5f54b22 (commit)
       via  cbcb0fc8c3ebc85bf81ba9d50795d62db75efa6c (commit)
       via  e507e67ee50852fc44011127eb42f535fe2ae493 (commit)
       via  d64ed33e94700f8204ec8089c7b235cff918f9f7 (commit)
       via  88c26c6b2dd8752ec1ff8196f4af11369ea6adbb (commit)
       via  91f1e7009305109db0bfb6405adbd11357745bfc (commit)
       via  2001423a6eb7937a689414f3fa62be5b124812c1 (commit)
       via  3a88a5d8753593be89373c5b124fc5a8398457a0 (commit)
       via  defb299deab3be85644a7880cba492a73c2e7e62 (commit)
       via  5152e0bf159151eaf04d31b788d57bebf7ab089b (commit)
       via  a3488ff35d219b943f3f52c07914fddf2009a140 (commit)
       via  574dc168c559dd680d26be1306c4a3d8b936c6e8 (commit)
       via  da32ae2a73012ce55cb89b2de9a4716b2800eee1 (commit)
       via  2f06f0e4d5b9ef371463c65dd5cae19f5b385a17 (commit)
       via  29d77300205e485be6595d9cd9276c246f3f89b7 (commit)
       via  af78d389b0ad37858108b69fe5100443dc88eb4a (commit)
       via  0035033f6bf6db955b116e2e0cc052bc5c79d80e (commit)
       via  49a841a5179307f0c8d84f647a71f44fb2b4b26d (commit)
       via  bafff6941293d4cc12cda4209f1d7a2b10274ea3 (commit)
       via  cb47df6cc327a59efaa16170f840c2745162bc86 (commit)
       via  a734789218122d8ab0d8f766bac4d69c04db91bf (commit)
       via  00573cf7d28472bf926e8f610a256cd991879c8b (commit)
       via  4431b9c71fc08955a89d1af70de3a8d5174c6e9f (commit)
       via  f61c7d9445c216c2364734fb1da2bea00cfb581d (commit)
       via  9ea73e2a66dcb1311732d43ea052412546c625fe (commit)
       via  224098499104939fccdb07b39407937c61983687 (commit)
       via  d3a82af33386c99c0fd3d6471df7e6696560089c (commit)
       via  f70c457c73f1bff6314f8c3eb4ad7be0c44a5f1a (commit)
       via  cb4f8b337c56dd3e7a648dda86031ceba65ee6c4 (commit)
       via  d72f1d79c0a53056a7cecad6c65fd57a183059b0 (commit)
       via  76e3d80bcc1548daba7bca97711684e5d68c1624 (commit)
       via  777e716728a7da63ece00df7e5bb8be7f9a2a1a3 (commit)
       via  828161a7527c535e5c657cd17b79cd562475fe82 (commit)
       via  17f22a2b220c38ebae5247d09f8c96ceec151957 (commit)
       via  38141f7261a7a2fa29f9cf3281cc46c1d845c1c3 (commit)
       via  acb0bd431ced4814b37f703ca63f2b39ad29c5df (commit)
       via  433c48edb1523809bb8d43e4fc40ebe1a177e103 (commit)
       via  0708d4a0b6db9426b9bb5addc1c592d526bd4ecd (commit)
       via  25ab6bebfd55db837aa049e5c0fdb2ce28f98f30 (commit)
       via  201ddbd84144a10bbcbc72a3525bb0c4e743570d (commit)
       via  692ef5fd300fef4ef1b724016c1600dbcfc8a070 (commit)
       via  d7d00f5140b7622fdc55a4058be73b89d587b7b2 (commit)
       via  7bd9af41d106590856e4d8c317502d1f4f56aefd (commit)
       via  5cd99e52e342db19c1a77369a5a8c027ee04feaf (commit)
       via  33b9f9bd117931bae54f45566fb77e654dcbe263 (commit)
       via  6b0632d7706cbb344331bf5d291cafd541070cbb (commit)
       via  820d3244a25aa8b4658ab79cd8368492aeddc739 (commit)
      from  a501e1d874f01acc318e7963837fbeb58a29f744 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 3a0d849c08f750dca1d6a40153c0107001769c6d
Author: Tom Clegg <tom at curoverse.com>
Date:   Sat Apr 22 02:29:00 2017 -0400

    11308: Fix arvfile append mode: write() changes the file pointer.

diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index b9faa11..931a3a1 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -1248,10 +1248,9 @@ class ArvadosFileWriter(ArvadosFileReader):
     @retry_method
     def write(self, data, num_retries=None):
         if self.mode[0] == "a":
-            self.arvadosfile.writeto(self.size(), data, num_retries)
-        else:
-            self.arvadosfile.writeto(self._filepos, data, num_retries)
-            self._filepos += len(data)
+            self._filepos = self.size()
+        self.arvadosfile.writeto(self._filepos, data, num_retries)
+        self._filepos += len(data)
         return len(data)
 
     @_FileLikeObjectBase._before_close
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 297fbfc..a800913 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -216,14 +216,21 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
         c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
         writer = c.open("count.txt", "ab+")
         self.assertEqual(writer.read(20), b"0123456789")
-        writer.seek(0, os.SEEK_SET)
 
+        writer.seek(0, os.SEEK_SET)
         writer.write("hello")
-        self.assertEqual(writer.read(20), b"0123456789hello")
+        self.assertEqual(writer.read(), b"")
+        writer.seek(-5, os.SEEK_CUR)
+        self.assertEqual(writer.read(3), b"hel")
+        self.assertEqual(writer.read(), b"lo")
         writer.seek(0, os.SEEK_SET)
+        self.assertEqual(writer.read(), b"0123456789hello")
 
+        writer.seek(0)
         writer.write("world")
-        self.assertEqual(writer.read(20), b"0123456789helloworld")
+        self.assertEqual(writer.read(), b"")
+        writer.seek(0)
+        self.assertEqual(writer.read(), b"0123456789helloworld")
 
         self.assertEqual(". 781e5e245d69b566979b86e28d23f2c7+10 fc5e038d38a57032085441e7fe7010b0+10 0:20:count.txt\n", c.portable_manifest_text())
 

commit 2b3b54becbce29efdf4c53787c4aa6d666900ef0
Author: Tom Clegg <tom at curoverse.com>
Date:   Sat Apr 22 02:26:59 2017 -0400

    11308: Fix bytes vs. strings from recent merge.

diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 7ee77fd..ed9d55c 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -732,6 +732,7 @@ class ArvPutUploadJob(object):
             state = json.dumps(self._state)
         try:
             new_cache = tempfile.NamedTemporaryFile(
+                mode='w+',
                 dir=os.path.dirname(self._cache_filename), delete=False)
             self._lock_file(new_cache)
             new_cache.write(state)
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index d8c1725..083b8fc 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -286,12 +286,14 @@ class ArvPutUploadJobTest(run_test_server.TestCaseWithServers,
             f.flush()
             cwriter = arv_put.ArvPutUploadJob([f.name])
             cwriter.start(save_collection=False)
-            self.assertEqual(3, cwriter.bytes_written - cwriter.bytes_skipped)
+            self.assertEqual(0, cwriter.bytes_skipped)
+            self.assertEqual(3, cwriter.bytes_written)
             # Don't destroy the cache, and start another upload
             cwriter_new = arv_put.ArvPutUploadJob([f.name])
             cwriter_new.start(save_collection=False)
             cwriter_new.destroy_cache()
-            self.assertEqual(0, cwriter_new.bytes_written - cwriter_new.bytes_skipped)
+            self.assertEqual(3, cwriter_new.bytes_skipped)
+            self.assertEqual(3, cwriter_new.bytes_written)
 
     def make_progress_tester(self):
         progression = []
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 6990aa6..297fbfc 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -79,7 +79,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52",
         })
         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
-                             api_client=api, keep_client=keep) as c:
+                        api_client=api, keep_client=keep) as c:
             writer = c.open("count.txt", "rb+")
             self.assertEqual(writer.size(), 10)
             self.assertEqual(b"0123456789", writer.read(12))
@@ -101,18 +101,23 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
 
 
     def test_truncate2(self):
-        keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
-        api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate2",
-                                                 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
-                                                 "replication_desired":None},
-                                                {"uuid":"zzzzz-4zz18-mockcollection0",
-                                                 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
-                                                 "portable_data_hash":"272da898abdf86ddc71994835e3155f8+95"})
+        keep = ArvadosFileWriterTestCase.MockKeep({
+            "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+        })
+        api = ArvadosFileWriterTestCase.MockApi({
+            "name": "test_truncate2",
+            "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+            "replication_desired": None,
+        }, {
+            "uuid": "zzzzz-4zz18-mockcollection0",
+            "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+            "portable_data_hash": "272da898abdf86ddc71994835e3155f8+95",
+        })
         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
-                             api_client=api, keep_client=keep) as c:
-            writer = c.open("count.txt", "r+")
+                        api_client=api, keep_client=keep) as c:
+            writer = c.open("count.txt", "rb+")
             self.assertEqual(writer.size(), 10)
-            self.assertEqual("0123456789", writer.read(12))
+            self.assertEqual(b"0123456789", writer.read(12))
 
             # extend file size
             writer.truncate(12)
@@ -128,30 +133,35 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
             self.assertFalse(c.modified())
 
     def test_truncate3(self):
-        keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789",
-                                                   "a925576942e94b2ef57a066101b48876+10": "abcdefghij"})
-        api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
-                                                 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
-                                                 "replication_desired":None},
-                                                {"uuid":"zzzzz-4zz18-mockcollection0",
-                                                 "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
-                                                 "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
+        keep = ArvadosFileWriterTestCase.MockKeep({
+            "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
+            "a925576942e94b2ef57a066101b48876+10": b"abcdefghij",
+        })
+        api = ArvadosFileWriterTestCase.MockApi({
+            "name": "test_truncate",
+            "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+            "replication_desired": None,
+        }, {
+            "uuid": "zzzzz-4zz18-mockcollection0",
+            "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+            "portable_data_hash": "7fcd0eaac3aad4c31a6a0e756475da92+52",
+        })
         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
-                             api_client=api, keep_client=keep) as c:
-            writer = c.open("count.txt", "r+")
+                        api_client=api, keep_client=keep) as c:
+            writer = c.open("count.txt", "rb+")
             self.assertEqual(writer.size(), 20)
-            self.assertEqual("0123456789ab", writer.read(12))
+            self.assertEqual(b"0123456789ab", writer.read(12))
             self.assertEqual(12, writer.tell())
 
             writer.truncate(8)
 
             # Make sure reading off the end doesn't break
             self.assertEqual(12, writer.tell())
-            self.assertEqual("", writer.read(12))
+            self.assertEqual(b"", writer.read(12))
 
             self.assertEqual(writer.size(), 8)
             self.assertEqual(2, writer.seek(-10, os.SEEK_CUR))
-            self.assertEqual("234567", writer.read(12))
+            self.assertEqual(b"234567", writer.read(12))
 
             self.assertIsNone(c.manifest_locator())
             self.assertTrue(c.modified())
@@ -352,10 +362,10 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
         api = ArvadosFileWriterTestCase.MockApi({}, {})
         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
                              api_client=api, keep_client=keep) as c:
-            writer = c.open("count.txt", "r+")
+            writer = c.open("count.txt", "rb+")
             self.assertEqual(writer.size(), 0)
 
-            text = "0123456789"
+            text = b"0123456789"
             writer.seek(2)
             writer.write(text)
             self.assertEqual(writer.size(), 12)
@@ -370,7 +380,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
         api = ArvadosFileWriterTestCase.MockApi({}, {})
         with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
                              api_client=api, keep_client=keep) as c:
-            writer = c.open("count.txt", "r+")
+            writer = c.open("count.txt", "rb+")
             self.assertEqual(writer.size(), 0)
 
             text = "0123456789"
@@ -387,28 +397,28 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
         api = ArvadosFileWriterTestCase.MockApi({}, {})
         for r in [[0, 1, 2, 3, 4], [4, 3, 2, 1, 0], [3, 2, 0, 4, 1]]:
             with Collection() as c:
-                writer = c.open("count.txt", "r+")
+                writer = c.open("count.txt", "rb+")
                 self.assertEqual(writer.size(), 0)
 
                 for i in r:
                     w = ("%s" % i) * 10
                     writer.seek(i*10)
-                    writer.write(w)
+                    writer.write(w.encode())
                 writer.seek(0)
-                self.assertEqual(writer.read(), "00000000001111111111222222222233333333334444444444")
+                self.assertEqual(writer.read(), b"00000000001111111111222222222233333333334444444444")
 
     def test_sparse_write4(self):
         keep = ArvadosFileWriterTestCase.MockKeep({})
         api = ArvadosFileWriterTestCase.MockApi({}, {})
         for r in [[0, 1, 2, 4], [4, 2, 1, 0], [2, 0, 4, 1]]:
             with Collection() as c:
-                writer = c.open("count.txt", "r+")
+                writer = c.open("count.txt", "rb+")
                 self.assertEqual(writer.size(), 0)
 
                 for i in r:
                     w = ("%s" % i) * 10
                     writer.seek(i*10)
-                    writer.write(w)
+                    writer.write(w.encode())
                 writer.seek(0)
                 self.assertEqual(writer.read(), b"000000000011111111112222222222\x00\x00\x00\x00\x00\x00\x00\x00\x00\x004444444444")
 

commit 31d31c010bb6b5170e3962fdd50c6d393cfe6076
Merge: a501e1d b046382
Author: Tom Clegg <tom at curoverse.com>
Date:   Fri Apr 21 16:02:50 2017 -0400

    11308: Merge branch 'master' into 11308-python3
    
    Conflicts:
    	sdk/python/arvados/arvfile.py
    	sdk/python/arvados/commands/get.py
    	sdk/python/arvados/commands/migrate19.py
    	sdk/python/tests/test_arv_get.py
    	sdk/python/tests/test_arvfile.py

diff --cc sdk/python/arvados/arvfile.py
index c52c772,aad3ce1..b9faa11
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@@ -107,7 -96,10 +107,10 @@@ class ArvadosFileReaderBase(_FileLikeOb
              pos += self._filepos
          elif whence == os.SEEK_END:
              pos += self.size()
-         self._filepos = min(max(pos, 0), self.size())
 -        if pos < 0L:
++        if pos < 0:
+             raise IOError(errno.EINVAL, "Tried to seek to negative file offset.")
+         self._filepos = pos
+         return self._filepos
  
      def tell(self):
          return self._filepos
@@@ -180,16 -181,16 +192,16 @@@
              data_size += len(s)
              if data_size >= sizehint:
                  break
 -        return ''.join(data).splitlines(True)
 +        return b''.join(data).decode().splitlines(True)
  
      def size(self):
-         raise NotImplementedError()
+         raise IOError(errno.ENOSYS, "Not implemented")
  
      def read(self, size, num_retries=None):
-         raise NotImplementedError()
+         raise IOError(errno.ENOSYS, "Not implemented")
  
      def readfrom(self, start, size, num_retries=None):
-         raise NotImplementedError()
+         raise IOError(errno.ENOSYS, "Not implemented")
  
  
  class StreamFileReader(ArvadosFileReaderBase):
@@@ -971,14 -997,15 +1010,15 @@@ class ArvadosFile(object)
                  self.parent._my_block_manager().block_prefetch(lr.locator)
                  locs.add(lr.locator)
  
 -        return ''.join(data)
 +        return b''.join(data)
  
      def _repack_writes(self, num_retries):
-         """Test if the buffer block has more data than actual segments.
+         """Optimize buffer block by repacking segments in file sequence.
  
-         This happens when a buffered write over-writes a file range written in
-         a previous buffered write.  Re-pack the buffer block for efficiency
-         and to avoid leaking information.
+         When the client makes random writes, they appear in the buffer block in
+         the sequence they were written rather than the sequence they appear in
+         the file.  This makes for inefficient, fragmented manifests.  Attempt
+         to optimize by repacking writes in file sequence.
  
          """
          segs = self._segments
@@@ -1194,9 -1223,13 +1238,12 @@@ class ArvadosFileWriter(ArvadosFileRead
      """
  
      def __init__(self, arvadosfile, mode, num_retries=None):
 -        super(ArvadosFileWriter, self).__init__(arvadosfile, num_retries=num_retries)
 -        self.mode = mode
 +        super(ArvadosFileWriter, self).__init__(arvadosfile, mode=mode, num_retries=num_retries)
          self.arvadosfile.add_writer(self)
  
+     def writable(self):
+         return True
+ 
      @_FileLikeObjectBase._before_close
      @retry_method
      def write(self, data, num_retries=None):
diff --cc sdk/python/arvados/commands/get.py
index 48f076b,3bf9295..888fd39
--- a/sdk/python/arvados/commands/get.py
+++ b/sdk/python/arvados/commands/get.py
@@@ -153,11 -155,11 +159,11 @@@ def main(arguments=None, stdout=sys.std
                  open_flags |= os.O_EXCL
              try:
                  if args.destination == "-":
-                     stdout.write(reader.manifest_text().encode())
 -                    stdout.write(reader.manifest_text(strip=args.strip_manifest))
++                    stdout.write(reader.manifest_text(strip=args.strip_manifest).encode())
                  else:
                      out_fd = os.open(args.destination, open_flags)
                      with os.fdopen(out_fd, 'wb') as out_file:
-                         out_file.write(reader.manifest_text().encode())
 -                        out_file.write(reader.manifest_text(strip=args.strip_manifest))
++                        out_file.write(reader.manifest_text(strip=args.strip_manifest).encode())
              except (IOError, OSError) as error:
                  logger.error("can't write to '{}': {}".format(args.destination, error))
                  return 1
diff --cc sdk/python/arvados/commands/migrate19.py
index 724de7b,9ff0b78..91899e6
--- a/sdk/python/arvados/commands/migrate19.py
+++ b/sdk/python/arvados/commands/migrate19.py
@@@ -120,11 -131,19 +133,20 @@@ def main(arguments=None)
      for img in old_images:
          i = uuid_to_collection[img["collection"]]
          pdh = i["portable_data_hash"]
-         if pdh not in already_migrated and (only_migrate is None or pdh in only_migrate):
+         if pdh not in already_migrated and pdh not in need_migrate and (only_migrate is None or pdh in only_migrate):
              need_migrate[pdh] = img
              with CollectionReader(i["manifest_text"]) as c:
-                 if list(c.values())[0].size() > biggest:
-                     biggest = list(c.values())[0].size()
 -                if c.values()[0].size() > biggest:
 -                    biggest = c.values()[0].size()
++                size = list(c.values())[0].size()
++                if size > biggest:
++                    biggest = size
+                     biggest_pdh = pdh
 -                totalbytes += c.values()[0].size()
++                totalbytes += size
+ 
+ 
+     if args.storage_driver == "vfs":
+         will_need = (biggest*20)
+     else:
+         will_need = (biggest*2.5)
  
      if args.print_unmigrated:
          only_migrate = set()
@@@ -135,7 -154,18 +157,18 @@@
      logger.info("Already migrated %i images", len(already_migrated))
      logger.info("Need to migrate %i images", len(need_migrate))
      logger.info("Using tempdir %s", tempfile.gettempdir())
-     logger.info("Biggest image is about %i MiB, tempdir needs at least %i MiB free", biggest>>20, biggest>>19)
 -    logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest/(2**20))
 -    logger.info("Total data to migrate about %i MiB", totalbytes/(2**20))
++    logger.info("Biggest image %s is about %i MiB", biggest_pdh, biggest>>20)
++    logger.info("Total data to migrate about %i MiB", totalbytes>>20)
+ 
+     df_out = subprocess.check_output(["df", "-B1", tempfile.gettempdir()])
+     ln = df_out.splitlines()[1]
+     filesystem, blocks, used, available, use_pct, mounted = re.match(r"^([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+) *([^ ]+)", ln).groups(1)
+     if int(available) <= will_need:
 -        logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)/(2**20), will_need/(2**20))
++        logger.warn("Temp filesystem mounted at %s does not have enough space for biggest image (has %i MiB, needs %i MiB)", mounted, int(available)>>20, will_need>>20)
+         if not args.force:
+             exit(1)
+         else:
+             logger.warn("--force provided, will migrate anyway")
  
      if args.dry_run:
          return
diff --cc sdk/python/tests/test_arv_get.py
index 46c1a0e,b7f5e2b..5174647
--- a/sdk/python/tests/test_arv_get.py
+++ b/sdk/python/tests/test_arv_get.py
@@@ -1,6 -1,9 +1,8 @@@
 -#!/usr/bin/env python
 -# -*- coding: utf-8 -*-
 -
 +from __future__ import absolute_import
 +from future.utils import listitems
  import io
+ import os
+ import re
  import shutil
  import tempfile
  
@@@ -32,15 -35,17 +35,18 @@@ class ArvadosGetTestCase(run_test_serve
                                    'subdir/baz.txt' : 'baz',
                                }):
          c = collection.Collection()
 -        for path, data in contents.items():
 -            with c.open(path, 'w') as f:
 +        for path, data in listitems(contents):
 +            with c.open(path, 'wb') as f:
                  f.write(data)
          c.save_new()
-         return (c.manifest_locator(), c.portable_data_hash(), c.manifest_text())
++
+         return (c.manifest_locator(),
+                 c.portable_data_hash(),
+                 c.manifest_text(strip=strip_manifest))
 -    
 +
      def run_get(self, args):
 -        self.stdout = io.BytesIO()
 -        self.stderr = io.BytesIO()
 +        self.stdout = tutil.BytesIO()
 +        self.stderr = tutil.StringIO()
          return arv_get.main(args, self.stdout, self.stderr)
  
      def test_version_argument(self):
diff --cc sdk/python/tests/test_arvfile.py
index e0f6596,20b2584..6990aa6
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@@ -99,26 -91,83 +99,88 @@@ class ArvadosFileWriterTestCase(unittes
              self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
              self.assertFalse(c.modified())
  
+ 
+     def test_truncate2(self):
+         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
+         api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate2",
+                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+                                                  "replication_desired":None},
+                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
+                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 7f614da9329cd3aebf59b91aadc30bf0+67108864 0:12:count.txt\n",
+                                                  "portable_data_hash":"272da898abdf86ddc71994835e3155f8+95"})
+         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
+                              api_client=api, keep_client=keep) as c:
+             writer = c.open("count.txt", "r+")
+             self.assertEqual(writer.size(), 10)
+             self.assertEqual("0123456789", writer.read(12))
+ 
+             # extend file size
+             writer.truncate(12)
+ 
+             self.assertEqual(writer.size(), 12)
+             writer.seek(0, os.SEEK_SET)
+             self.assertEqual(b"0123456789\x00\x00", writer.read(12))
+ 
+             self.assertIsNone(c.manifest_locator())
+             self.assertTrue(c.modified())
+             c.save_new("test_truncate2")
+             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
+             self.assertFalse(c.modified())
+ 
+     def test_truncate3(self):
+         keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789",
+                                                    "a925576942e94b2ef57a066101b48876+10": "abcdefghij"})
+         api = ArvadosFileWriterTestCase.MockApi({"name":"test_truncate",
+                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+                                                  "replication_desired":None},
+                                                 {"uuid":"zzzzz-4zz18-mockcollection0",
+                                                  "manifest_text":". 781e5e245d69b566979b86e28d23f2c7+10 0:8:count.txt\n",
+                                                  "portable_data_hash":"7fcd0eaac3aad4c31a6a0e756475da92+52"})
+         with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 a925576942e94b2ef57a066101b48876+10 0:20:count.txt\n',
+                              api_client=api, keep_client=keep) as c:
+             writer = c.open("count.txt", "r+")
+             self.assertEqual(writer.size(), 20)
+             self.assertEqual("0123456789ab", writer.read(12))
+             self.assertEqual(12, writer.tell())
+ 
+             writer.truncate(8)
+ 
+             # Make sure reading off the end doesn't break
+             self.assertEqual(12, writer.tell())
+             self.assertEqual("", writer.read(12))
+ 
+             self.assertEqual(writer.size(), 8)
+             self.assertEqual(2, writer.seek(-10, os.SEEK_CUR))
+             self.assertEqual("234567", writer.read(12))
+ 
+             self.assertIsNone(c.manifest_locator())
+             self.assertTrue(c.modified())
+             c.save_new("test_truncate")
+             self.assertEqual("zzzzz-4zz18-mockcollection0", c.manifest_locator())
+             self.assertFalse(c.modified())
+ 
+ 
+ 
      def test_write_to_end(self):
 -        keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": "0123456789"})
 -        api = ArvadosFileWriterTestCase.MockApi({"name":"test_append",
 -                                                 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 -                                                 "replication_desired":None},
 -                                                {"uuid":"zzzzz-4zz18-mockcollection0",
 -                                                 "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 -                                                 "portable_data_hash":"c5c3af76565c8efb6a806546bcf073f3+88"})
 +        keep = ArvadosFileWriterTestCase.MockKeep({
 +            "781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
 +        })
 +        api = ArvadosFileWriterTestCase.MockApi({
 +            "name": "test_append",
 +            "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 +            "replication_desired": None,
 +        }, {
 +            "uuid": "zzzzz-4zz18-mockcollection0",
 +            "manifest_text": ". 781e5e245d69b566979b86e28d23f2c7+10 acbd18db4cc2f85cedef654fccc4a4d8+3 0:13:count.txt\n",
 +            "portable_data_hash": "c5c3af76565c8efb6a806546bcf073f3+88",
 +        })
          with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
                               api_client=api, keep_client=keep) as c:
 -            writer = c.open("count.txt", "r+")
 +            writer = c.open("count.txt", "rb+")
              self.assertEqual(writer.size(), 10)
  
-             writer.seek(5, os.SEEK_SET)
+             self.assertEqual(5, writer.seek(5, os.SEEK_SET))
 -            self.assertEqual("56789", writer.read(8))
 +            self.assertEqual(b"56789", writer.read(8))
  
              writer.seek(10, os.SEEK_SET)
              writer.write("foo")

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list