[ARVADOS] updated: 4a8883858028d4068b52710acd9c349108b6fc7c
Git user
git at public.curoverse.com
Wed Apr 5 15:50:15 EDT 2017
Summary of changes:
build/run-tests.sh | 27 ++------
sdk/python/arvados/arvfile.py | 29 +++++----
sdk/python/arvados/collection.py | 34 +++++-----
sdk/python/arvados/commands/_util.py | 2 -
sdk/python/arvados/commands/arv_copy.py | 2 -
sdk/python/arvados/commands/keepdocker.py | 2 -
sdk/python/arvados/commands/ls.py | 2 -
sdk/python/arvados/commands/put.py | 13 ++--
sdk/python/arvados/commands/run.py | 2 -
sdk/python/arvados/commands/ws.py | 2 -
sdk/python/arvados/retry.py | 2 -
sdk/python/tests/arvados_testutil.py | 2 -
sdk/python/tests/keepstub.py | 27 ++++++++
.../tests/performance/performance_profiler.py | 14 ++--
sdk/python/tests/run_test_server.py | 2 -
sdk/python/tests/test_api.py | 2 -
sdk/python/tests/test_arv_copy.py | 3 -
sdk/python/tests/test_arv_keepdocker.py | 3 -
sdk/python/tests/test_arv_ls.py | 3 -
sdk/python/tests/test_arv_normalize.py | 3 -
sdk/python/tests/test_arv_put.py | 11 ++--
sdk/python/tests/test_arv_run.py | 3 -
sdk/python/tests/test_arv_ws.py | 2 -
sdk/python/tests/test_arvfile.py | 59 +++++++++--------
sdk/python/tests/test_benchmark_collections.py | 2 +-
sdk/python/tests/test_collections.py | 74 +++++++++++++++-------
sdk/python/tests/test_errors.py | 2 -
sdk/python/tests/test_keep_client.py | 21 +-----
sdk/python/tests/test_keep_locator.py | 3 -
sdk/python/tests/test_retry.py | 2 -
sdk/python/tests/test_retry_job_helpers.py | 2 -
sdk/python/tests/test_stream.py | 2 -
32 files changed, 162 insertions(+), 197 deletions(-)
mode change 100755 => 100644 sdk/python/arvados/commands/ls.py
via 4a8883858028d4068b52710acd9c349108b6fc7c (commit)
via 56b51a9e60da1531d40685525a563bef91f17876 (commit)
via 249b3abab7da1abd531195ed0cd58760bf10774b (commit)
via f54096601d61d3d3016317c4b80b3e1a701af869 (commit)
via e5d948386bad766d481b17c06bc2c3147c42cde5 (commit)
via 4ef817417f439188bea95a51d86acf21ad4f6d37 (commit)
via 0487aeb518af7f4ba4a7b0ae17d42805dcdc3805 (commit)
via 46f6db136c5820555109963f5382b449aff617ce (commit)
from e8074a3ca07250640c30f49cdefbbbe76a4a7885 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4a8883858028d4068b52710acd9c349108b6fc7c
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 5 15:38:40 2017 -0400
11308: Remove superfluous hashbangs
diff --git a/sdk/python/arvados/commands/_util.py b/sdk/python/arvados/commands/_util.py
index c42ee7a..0755784 100644
--- a/sdk/python/arvados/commands/_util.py
+++ b/sdk/python/arvados/commands/_util.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
import argparse
import errno
import os
diff --git a/sdk/python/arvados/commands/arv_copy.py b/sdk/python/arvados/commands/arv_copy.py
index 0eca6d8..c0d05b3 100755
--- a/sdk/python/arvados/commands/arv_copy.py
+++ b/sdk/python/arvados/commands/arv_copy.py
@@ -1,5 +1,3 @@
-#! /usr/bin/env python
-
# arv-copy [--recursive] [--no-recursive] object-uuid src dst
#
# Copies an object from Arvados instance src to instance dst.
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index 354ae50..22ea476 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from builtins import next
import argparse
import collections
diff --git a/sdk/python/arvados/commands/ls.py b/sdk/python/arvados/commands/ls.py
old mode 100755
new mode 100644
index ca2d6d7..250bda8
--- a/sdk/python/arvados/commands/ls.py
+++ b/sdk/python/arvados/commands/ls.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import print_function
from __future__ import division
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 0c5d776..400d57d 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -1,8 +1,3 @@
-#!/usr/bin/env python
-
-# TODO:
-# --md5sum - display md5 of each file as read from disk
-
from __future__ import division
from builtins import str
from builtins import object
diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index 01a18e5..d8bb26e 100644
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import print_function
from __future__ import absolute_import
from builtins import range
diff --git a/sdk/python/arvados/commands/ws.py b/sdk/python/arvados/commands/ws.py
index cd67b73..aa7dd05 100644
--- a/sdk/python/arvados/commands/ws.py
+++ b/sdk/python/arvados/commands/ws.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import print_function
import sys
import logging
diff --git a/sdk/python/arvados/retry.py b/sdk/python/arvados/retry.py
index 168bd39..2f27243 100644
--- a/sdk/python/arvados/retry.py
+++ b/sdk/python/arvados/retry.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from builtins import range
from builtins import object
import functools
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index c70ecd4..ebf6297 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from future import standard_library
standard_library.install_aliases()
from builtins import str
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 7dcd647..5aff52b 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import print_function
from __future__ import division
from builtins import str
diff --git a/sdk/python/tests/test_api.py b/sdk/python/tests/test_api.py
index 7eefd62..aa4c498 100644
--- a/sdk/python/tests/test_api.py
+++ b/sdk/python/tests/test_api.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import absolute_import
from builtins import str
from builtins import range
diff --git a/sdk/python/tests/test_arv_copy.py b/sdk/python/tests/test_arv_copy.py
index cc207c2..54fed85 100644
--- a/sdk/python/tests/test_arv_copy.py
+++ b/sdk/python/tests/test_arv_copy.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
from __future__ import absolute_import
import os
import sys
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index ff3fbb4..358752a 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
from __future__ import absolute_import
import arvados
import hashlib
diff --git a/sdk/python/tests/test_arv_ls.py b/sdk/python/tests/test_arv_ls.py
index 6bb089b..a7117d3 100644
--- a/sdk/python/tests/test_arv_ls.py
+++ b/sdk/python/tests/test_arv_ls.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
from __future__ import absolute_import
from builtins import str
from builtins import range
diff --git a/sdk/python/tests/test_arv_normalize.py b/sdk/python/tests/test_arv_normalize.py
index b0272f4..4357ee2 100644
--- a/sdk/python/tests/test_arv_normalize.py
+++ b/sdk/python/tests/test_arv_normalize.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
import subprocess
import sys
import tempfile
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 0e2574d..d8c1725 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
from __future__ import absolute_import
from __future__ import division
from future import standard_library
diff --git a/sdk/python/tests/test_arv_run.py b/sdk/python/tests/test_arv_run.py
index 8823837..f2f9640 100644
--- a/sdk/python/tests/test_arv_run.py
+++ b/sdk/python/tests/test_arv_run.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
from __future__ import absolute_import
import os
import sys
diff --git a/sdk/python/tests/test_arv_ws.py b/sdk/python/tests/test_arv_ws.py
index 86a21cc..4f0779d 100644
--- a/sdk/python/tests/test_arv_ws.py
+++ b/sdk/python/tests/test_arv_ws.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import absolute_import
import os
import sys
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index 1998f35..e0f6596 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import absolute_import
from builtins import hex
from builtins import str
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index da44ee2..86215f5 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -1,7 +1,4 @@
from __future__ import absolute_import
-# usage example:
-#
-# ARVADOS_API_TOKEN=abc ARVADOS_API_HOST=arvados.local python -m unittest discover
from builtins import object
import arvados
diff --git a/sdk/python/tests/test_errors.py b/sdk/python/tests/test_errors.py
index dd7b87e..968ce98 100644
--- a/sdk/python/tests/test_errors.py
+++ b/sdk/python/tests/test_errors.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import absolute_import
import traceback
import unittest
diff --git a/sdk/python/tests/test_keep_locator.py b/sdk/python/tests/test_keep_locator.py
index 2a90316..9b9c4b6 100644
--- a/sdk/python/tests/test_keep_locator.py
+++ b/sdk/python/tests/test_keep_locator.py
@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
from builtins import next
from builtins import zip
from builtins import str
diff --git a/sdk/python/tests/test_retry.py b/sdk/python/tests/test_retry.py
index 82725c2..56dc560 100644
--- a/sdk/python/tests/test_retry.py
+++ b/sdk/python/tests/test_retry.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from builtins import zip
from builtins import range
from builtins import object
diff --git a/sdk/python/tests/test_retry_job_helpers.py b/sdk/python/tests/test_retry_job_helpers.py
index 9ad957a..e31a9a0 100644
--- a/sdk/python/tests/test_retry_job_helpers.py
+++ b/sdk/python/tests/test_retry_job_helpers.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import absolute_import
from builtins import object
import mock
diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py
index 37cdbf2..664deb5 100644
--- a/sdk/python/tests/test_stream.py
+++ b/sdk/python/tests/test_stream.py
@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-
from __future__ import absolute_import
from builtins import object
import bz2
commit 56b51a9e60da1531d40685525a563bef91f17876
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 5 15:29:06 2017 -0400
11308: Move stub server setup to keepstub.StubKeepServers.
diff --git a/sdk/python/tests/keepstub.py b/sdk/python/tests/keepstub.py
index 4992196..3c9b2b1 100644
--- a/sdk/python/tests/keepstub.py
+++ b/sdk/python/tests/keepstub.py
@@ -6,13 +6,40 @@ import http.server
import hashlib
import os
import re
+import socket
import socketserver
import sys
+import threading
import time
+from . import arvados_testutil as tutil
+
_debug = os.environ.get('ARVADOS_DEBUG', None)
+class StubKeepServers(tutil.ApiClientMock):
+
+ def setUp(self):
+ super(StubKeepServers, self).setUp()
+ sock = socket.socket()
+ sock.bind(('0.0.0.0', 0))
+ self.port = sock.getsockname()[1]
+ sock.close()
+ self.server = Server(('0.0.0.0', self.port), Handler)
+ self.thread = threading.Thread(target=self.server.serve_forever)
+ self.thread.daemon = True # Exit thread if main proc exits
+ self.thread.start()
+ self.api_client = self.mock_keep_services(
+ count=1,
+ service_host='localhost',
+ service_port=self.port,
+ )
+
+ def tearDown(self):
+ self.server.shutdown()
+ super(StubKeepServers, self).tearDown()
+
+
class Server(socketserver.ThreadingMixIn, http.server.HTTPServer, object):
allow_reuse_address = 1
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index 35070f3..8fdcd72 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -13,7 +13,6 @@ import random
import re
import socket
import sys
-import threading
import time
import unittest
import urllib.parse
@@ -669,7 +668,7 @@ class KeepClientRendezvousTestCase(unittest.TestCase, tutil.ApiClientMock):
self.check_64_zeros_error_order('put', arvados.errors.KeepWriteError)
-class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock):
+class KeepClientTimeout(keepstub.StubKeepServers, unittest.TestCase):
# BANDWIDTH_LOW_LIM must be less than len(DATA) so we can transfer
# 1s worth of data and then trigger bandwidth errors before running
# out of data.
@@ -703,24 +702,6 @@ class KeepClientTimeout(unittest.TestCase, tutil.ApiClientMock):
delta = round(time.time() - self.t0, 3)
self.assertGreaterEqual(delta, self.tmin)
- def setUp(self):
- sock = socket.socket()
- sock.bind(('0.0.0.0', 0))
- self.port = sock.getsockname()[1]
- sock.close()
- self.server = keepstub.Server(('0.0.0.0', self.port), keepstub.Handler)
- self.thread = threading.Thread(target=self.server.serve_forever)
- self.thread.daemon = True # Exit thread if main proc exits
- self.thread.start()
- self.api_client = self.mock_keep_services(
- count=1,
- service_host='localhost',
- service_port=self.port,
- )
-
- def tearDown(self):
- self.server.shutdown()
-
def keepClient(self, timeouts=(0.1, TIMEOUT_TIME, BANDWIDTH_LOW_LIM)):
return arvados.KeepClient(
api_client=self.api_client,
commit 249b3abab7da1abd531195ed0cd58760bf10774b
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 5 15:28:07 2017 -0400
11308: Fix misleading argument name.
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 879a199..da44ee2 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -524,8 +524,8 @@ class CollectionTestMixin(tutil.ApiClientMock):
@tutil.skip_sleep
class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
- def mock_get_collection(self, api_mock, code, body):
- body = self.API_COLLECTIONS.get(body)
+ def mock_get_collection(self, api_mock, code, fixturename):
+ body = self.API_COLLECTIONS.get(fixturename)
self._mock_api_call(api_mock.collections().get, code, body)
def api_client_mock(self, status=200):
commit f54096601d61d3d3016317c4b80b3e1a701af869
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 5 15:12:45 2017 -0400
11308: Fix useless assertion.
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index e0ff7a7..0e2574d 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -765,9 +765,11 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
with open(os.path.join(datadir, "foo"), "w") as f:
f.write("The quick brown fox jumped over the lazy dog")
p = subprocess.Popen([sys.executable, arv_put.__file__, datadir],
- stdout=subprocess.PIPE, env=self.ENVIRON)
- (arvout, arverr) = p.communicate()
- self.assertEqual(arverr, None)
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=self.ENVIRON)
+ (out, err) = p.communicate()
+ self.assertRegex(err.decode(), r'INFO: Collection saved as ')
self.assertEqual(p.returncode, 0)
# The manifest text stored in the API server under the same
commit e5d948386bad766d481b17c06bc2c3147c42cde5
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 5 15:12:18 2017 -0400
11308: Fix exception handling in test case.
diff --git a/sdk/python/tests/performance/performance_profiler.py b/sdk/python/tests/performance/performance_profiler.py
index afa53ae..50eadd1 100644
--- a/sdk/python/tests/performance/performance_profiler.py
+++ b/sdk/python/tests/performance/performance_profiler.py
@@ -37,13 +37,9 @@ def profiled(function):
pr = profile.Profile()
pr.enable()
try:
- ret = function(*args, **kwargs)
- except Exception as e:
- caught = e
- pr.disable()
- ps = pstats.Stats(pr, stream=outfile)
- ps.sort_stats('time').print_stats()
- if caught:
- raise
- return ret
+ return function(*args, **kwargs)
+ finally:
+ pr.disable()
+ ps = pstats.Stats(pr, stream=outfile)
+ ps.sort_stats('time').print_stats()
return profiled_function
commit 4ef817417f439188bea95a51d86acf21ad4f6d37
Author: Tom Clegg <tom at curoverse.com>
Date: Wed Apr 5 15:05:10 2017 -0400
11308: Raise exception on invalid/unsupported open() mode.
diff --git a/sdk/python/arvados/arvfile.py b/sdk/python/arvados/arvfile.py
index 9f95ee0..180b120 100644
--- a/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@ -4,21 +4,22 @@ from future import standard_library
standard_library.install_aliases()
from builtins import range
from builtins import object
-import functools
-import os
-import zlib
import bz2
-from . import config
-import hashlib
-import threading
-import queue
+import collections
import copy
import errno
-import re
+import functools
+import hashlib
import logging
-import collections
+import os
+import queue
+import re
+import sys
+import threading
import uuid
+import zlib
+from . import config
from .errors import KeepWriteError, AssertionError, ArgumentError
from .keep import KeepLocator
from ._normalize_stream import normalize_stream
@@ -82,6 +83,9 @@ class _FileLikeObjectBase(object):
class ArvadosFileReaderBase(_FileLikeObjectBase):
def __init__(self, name, mode, num_retries=None):
super(ArvadosFileReaderBase, self).__init__(name, mode)
+ self._binary = 'b' in mode
+ if sys.version_info >= (3, 0) and not self._binary:
+ raise NotImplementedError("text mode {!r} is not implemented".format(mode))
self._filepos = 0
self.num_retries = num_retries
self._readline_cache = (None, None)
@@ -1136,8 +1140,8 @@ class ArvadosFileReader(ArvadosFileReaderBase):
"""
- def __init__(self, arvadosfile, num_retries=None):
- super(ArvadosFileReader, self).__init__(arvadosfile.name, "r", num_retries=num_retries)
+ def __init__(self, arvadosfile, mode="r", num_retries=None):
+ super(ArvadosFileReader, self).__init__(arvadosfile.name, mode=mode, num_retries=num_retries)
self.arvadosfile = arvadosfile
def size(self):
@@ -1189,8 +1193,7 @@ class ArvadosFileWriter(ArvadosFileReader):
"""
def __init__(self, arvadosfile, mode, num_retries=None):
- super(ArvadosFileWriter, self).__init__(arvadosfile, num_retries=num_retries)
- self.mode = mode
+ super(ArvadosFileWriter, self).__init__(arvadosfile, mode=mode, num_retries=num_retries)
self.arvadosfile.add_writer(self)
@_FileLikeObjectBase._before_close
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 1c68c8e..5b3b8d5 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -621,7 +621,12 @@ class RichCollectionBase(CollectionBase):
:path:
path to a file in the collection
:mode:
- one of "r", "r+", "w", "w+", "a", "a+"
+ a string consisting of "r", "w", or "a", optionally followed
+ by "b" or "t", optionally followed by "+".
+ :"b":
+ binary mode: write() accepts bytes, read() returns bytes.
+ :"t":
+ text mode (default): write() accepts strings, read() returns strings.
:"r":
opens for reading
:"r+":
@@ -633,33 +638,28 @@ class RichCollectionBase(CollectionBase):
the end of the file. Writing does not affect the file pointer for
reading.
"""
- mode = mode.replace("b", "")
- if len(mode) == 0 or mode[0] not in ("r", "w", "a"):
- raise errors.ArgumentError("Bad mode '%s'" % mode)
- create = (mode != "r")
- if create and not self.writable():
- raise IOError(errno.EROFS, "Collection is read only")
+ if not re.search(r'^[rwa][bt]?\+?$', mode):
+ raise errors.ArgumentError("Invalid mode {!r}".format(mode))
- if create:
- arvfile = self.find_or_create(path, FILE)
- else:
+ if mode[0] == 'r' and '+' not in mode:
+ fclass = ArvadosFileReader
arvfile = self.find(path)
+ elif not self.writable():
+ raise IOError(errno.EROFS, "Collection is read only")
+ else:
+ fclass = ArvadosFileWriter
+ arvfile = self.find_or_create(path, FILE)
if arvfile is None:
raise IOError(errno.ENOENT, "File not found", path)
if not isinstance(arvfile, ArvadosFile):
raise IOError(errno.EISDIR, "Is a directory", path)
- if mode[0] == "w":
+ if mode[0] == 'w':
arvfile.truncate(0)
- name = os.path.basename(path)
-
- if mode == "r":
- return ArvadosFileReader(arvfile, num_retries=self.num_retries)
- else:
- return ArvadosFileWriter(arvfile, mode, num_retries=self.num_retries)
+ return fclass(arvfile, mode=mode, num_retries=self.num_retries)
def modified(self):
"""Determine if the collection has been modified since last commited."""
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 2dee1dc..0c5d776 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -558,7 +558,7 @@ class ArvPutUploadJob(object):
self.reporter(self.bytes_written, self.bytes_expected)
def _write_stdin(self, filename):
- output = self._local_collection.open(filename, 'w')
+ output = self._local_collection.open(filename, 'wb')
self._write(sys.stdin, output)
output.close()
@@ -624,17 +624,17 @@ class ArvPutUploadJob(object):
def _upload_files(self):
for source, resume_offset, filename in self._files_to_upload:
- with open(source, 'r') as source_fd:
+ with open(source, 'rb') as source_fd:
with self._state_lock:
self._state['files'][source]['mtime'] = os.path.getmtime(source)
self._state['files'][source]['size'] = os.path.getsize(source)
if resume_offset > 0:
# Start upload where we left off
- output = self._local_collection.open(filename, 'a')
+ output = self._local_collection.open(filename, 'ab')
source_fd.seek(resume_offset)
else:
# Start from scratch
- output = self._local_collection.open(filename, 'w')
+ output = self._local_collection.open(filename, 'wb')
self._write(source_fd, output)
output.close(flush=False)
diff --git a/sdk/python/tests/test_arvfile.py b/sdk/python/tests/test_arvfile.py
index e3ba090..1998f35 100644
--- a/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@ -82,7 +82,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 10)
self.assertEqual(b"0123456789", writer.read(12))
@@ -116,7 +116,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 10)
writer.seek(5, os.SEEK_SET)
@@ -144,7 +144,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789",
})
c = Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', keep_client=keep)
- writer = c.open("count.txt", "a+")
+ writer = c.open("count.txt", "ab+")
self.assertEqual(writer.read(20), b"0123456789")
writer.seek(0, os.SEEK_SET)
@@ -163,7 +163,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(0, os.SEEK_SET)
writer.write("foo")
@@ -174,14 +174,14 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_write_empty(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
with Collection(keep_client=keep) as c:
- writer = c.open("count.txt", "w")
+ writer = c.open("count.txt", "wb")
self.assertEqual(writer.size(), 0)
self.assertEqual(". d41d8cd98f00b204e9800998ecf8427e+0 0:0:count.txt\n", c.portable_manifest_text())
def test_save_manifest_text(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
with Collection(keep_client=keep) as c:
- writer = c.open("count.txt", "w")
+ writer = c.open("count.txt", "wb")
writer.write(b"0123456789")
self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
@@ -192,7 +192,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_get_manifest_text_commits(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
with Collection(keep_client=keep) as c:
- writer = c.open("count.txt", "w")
+ writer = c.open("count.txt", "wb")
writer.write("0123456789")
self.assertEqual('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n', c.portable_manifest_text())
self.assertNotIn('781e5e245d69b566979b86e28d23f2c7+10', keep.blocks)
@@ -204,7 +204,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(3, os.SEEK_SET)
writer.write("foo")
@@ -216,7 +216,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(b"0123456789", writer.readfrom(0, 13))
writer.seek(7, os.SEEK_SET)
writer.write("foo")
@@ -228,7 +228,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt 0:10:count.txt\n',
keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(b"012345678901234", writer.readfrom(0, 15))
writer.seek(7, os.SEEK_SET)
writer.write("foobar")
@@ -240,7 +240,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
keep = ArvadosFileWriterTestCase.MockKeep({"781e5e245d69b566979b86e28d23f2c7+10": b"0123456789"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:4:count.txt 0:4:count.txt 0:4:count.txt',
keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(b"012301230123", writer.readfrom(0, 15))
writer.seek(2, os.SEEK_SET)
writer.write("abcdefg")
@@ -258,7 +258,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"portable_data_hash":"9132ca8e3f671c76103a38f5bc24328c+108"})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
text = "0123456789" * 100
for b in range(0, 100000):
writer.write(text)
@@ -276,7 +276,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
api = ArvadosFileWriterTestCase.MockApi({}, {})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
self.assertEqual(writer.size(), 0)
text = "0123456789"
@@ -291,7 +291,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
keep = ArvadosFileWriterTestCase.MockKeep({})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
for b in range(0, 10):
writer.seek(0, os.SEEK_SET)
writer.write("0123456789")
@@ -310,7 +310,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
for b in range(0, 10):
writer.seek(10, os.SEEK_SET)
writer.write("abcdefghij")
@@ -330,7 +330,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt',
keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
for b in range(0, 10):
writer.seek(5, os.SEEK_SET)
writer.write("abcdefghij")
@@ -355,7 +355,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"portable_data_hash":"217665c6b713e1b78dfba7ebd42344db+156"})
with Collection('. ' + arvados.config.EMPTY_BLOCK_LOCATOR + ' 0:0:count.txt',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "r+")
+ writer = c.open("count.txt", "rb+")
text = b''.join([b"0123456789" for a in range(0, 100)])
for b in range(0, 100000):
writer.write(text)
@@ -381,7 +381,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51",
})
with Collection(api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "w+")
+ writer = c.open("count.txt", "wb+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
self.assertEqual(writer.size(), 8)
@@ -405,7 +405,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"portable_data_hash":"1b02aaa62528d28a5be41651cbb9d7c7+59"})
with Collection(api_client=api, keep_client=keep) as c:
self.assertIsNone(c.api_response())
- writer = c.open("foo/bar/count.txt", "w+")
+ writer = c.open("foo/bar/count.txt", "wb+")
writer.write("01234567")
self.assertFalse(c.committed())
c.save_new("test_create")
@@ -422,7 +422,7 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"portable_data_hash":"7a461a8c58601798f690f8b368ac4423+51"})
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n',
api_client=api, keep_client=keep) as c:
- writer = c.open("count.txt", "w+")
+ writer = c.open("count.txt", "wb+")
self.assertEqual(writer.size(), 0)
writer.write("01234567")
self.assertEqual(writer.size(), 8)
@@ -436,12 +436,12 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
def test_file_not_found(self):
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
- writer = c.open("nocount.txt", "r")
+ writer = c.open("nocount.txt", "rb")
def test_cannot_open_directory(self):
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n') as c:
with self.assertRaises(IOError):
- writer = c.open(".", "r")
+ writer = c.open(".", "rb")
def test_create_multiple(self):
keep = ArvadosFileWriterTestCase.MockKeep({})
@@ -452,8 +452,8 @@ class ArvadosFileWriterTestCase(unittest.TestCase):
"manifest_text":". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:8:count1.txt 8:8:count2.txt\n",
"portable_data_hash":"71e7bb6c00d31fc2b4364199fd97be08+102"})
with Collection(api_client=api, keep_client=keep) as c:
- w1 = c.open("count1.txt", "w")
- w2 = c.open("count2.txt", "w")
+ w1 = c.open("count1.txt", "wb")
+ w2 = c.open("count2.txt", "wb")
w1.write("01234567")
w2.write("abcdefgh")
self.assertEqual(w1.size(), 8)
@@ -505,7 +505,7 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
stream.append(Range(loc, n, len(d)))
n += len(d)
af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
- return ArvadosFileReader(af)
+ return ArvadosFileReader(af, mode="rb")
def test_read_block_crossing_behavior(self):
# read() needs to return all the data requested if possible, even if it
@@ -533,7 +533,7 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
"e8dc4081b13434b45189a720b77b6818+8": b"abcdefgh",
})
with Collection(". 2e9ec317e197819358fbc43afca7d837+8 e8dc4081b13434b45189a720b77b6818+8 0:16:count.txt\n", keep_client=keep) as c:
- r = c.open("count.txt", "r")
+ r = c.open("count.txt", "rb")
self.assertEqual(b"0123", r.read(4))
self.assertIn("2e9ec317e197819358fbc43afca7d837+8", keep.requests)
self.assertIn("e8dc4081b13434b45189a720b77b6818+8", keep.requests)
@@ -547,7 +547,7 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
def test__eq__from_writes(self):
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
with Collection() as c2:
- f = c2.open("count1.txt", "w")
+ f = c2.open("count1.txt", "wb")
f.write("0123456789")
self.assertTrue(c1["count1.txt"] == c2["count1.txt"])
@@ -556,7 +556,7 @@ class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
def test__ne__(self):
with Collection('. 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt') as c1:
with Collection() as c2:
- f = c2.open("count1.txt", "w")
+ f = c2.open("count1.txt", "wb")
f.write("1234567890")
self.assertTrue(c1["count1.txt"] != c2["count1.txt"])
@@ -583,6 +583,7 @@ class ArvadosFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
af = ArvadosFile(col, "test",
stream=stream,
segments=segments)
+ kwargs.setdefault('mode', 'rb')
return ArvadosFileReader(af, **kwargs)
def read_for_test(self, reader, byte_count, **kwargs):
diff --git a/sdk/python/tests/test_benchmark_collections.py b/sdk/python/tests/test_benchmark_collections.py
index 273011e..c6cda71 100644
--- a/sdk/python/tests/test_benchmark_collections.py
+++ b/sdk/python/tests/test_benchmark_collections.py
@@ -47,7 +47,7 @@ class CollectionBenchmark(run_test_server.TestCaseWithServers,
dst = arvados.collection.Collection()
with tutil.mock_keep_responses('x'*self.TEST_BLOCK_SIZE, 200):
for name in self.list_recursive(src):
- with src.open(name) as srcfile, dst.open(name, 'w') as dstfile:
+ with src.open(name, 'rb') as srcfile, dst.open(name, 'wb') as dstfile:
dstfile.write(srcfile.read())
dst.save_new()
diff --git a/sdk/python/tests/test_collections.py b/sdk/python/tests/test_collections.py
index 00c78c8..879a199 100644
--- a/sdk/python/tests/test_collections.py
+++ b/sdk/python/tests/test_collections.py
@@ -10,6 +10,7 @@ import mock
import os
import pprint
import re
+import sys
import tempfile
import unittest
@@ -648,7 +649,7 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
def test_open_collection_file_one_argument(self):
client = self.api_client_mock(200)
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client)
- cfile = reader.open('./foo')
+ cfile = reader.open('./foo', 'rb')
self.check_open_file(cfile, '.', 'foo', 3)
def test_open_deep_file(self):
@@ -657,7 +658,7 @@ class CollectionReaderTestCase(unittest.TestCase, CollectionTestMixin):
self.mock_get_collection(client, 200, coll_name)
reader = arvados.CollectionReader(
self.API_COLLECTIONS[coll_name]['uuid'], api_client=client)
- cfile = reader.open('./subdir2/subdir3/file2_in_subdir3.txt')
+ cfile = reader.open('./subdir2/subdir3/file2_in_subdir3.txt', 'rb')
self.check_open_file(cfile, './subdir2/subdir3', 'file2_in_subdir3.txt',
32)
@@ -806,6 +807,38 @@ class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
self.assertRaises(arvados.errors.AssertionError, writer.open, 'two')
+class CollectionOpenModes(run_test_server.TestCaseWithServers):
+
+ def test_open_binary_modes(self):
+ c = Collection()
+ for mode in ['wb', 'wb+', 'ab', 'ab+']:
+ with c.open('foo', 'wb') as f:
+ f.write(b'foo')
+
+ def test_open_invalid_modes(self):
+ c = Collection()
+ for mode in ['+r', 'aa', '++', 'r+b', 'beer', '', None]:
+ with self.assertRaises(Exception):
+ c.open('foo', mode)
+
+ def test_open_text_modes(self):
+ c = Collection()
+ with c.open('foo', 'wb') as f:
+ f.write('foo')
+ for mode in ['r', 'rt', 'r+', 'rt+', 'w', 'wt', 'a', 'at']:
+ if sys.version_info >= (3, 0):
+ with self.assertRaises(NotImplementedError):
+ c.open('foo', mode)
+ else:
+ with c.open('foo', mode) as f:
+ if mode[0] == 'r' and '+' not in mode:
+ self.assertEqual('foo', f.read(3))
+ else:
+ f.write('bar')
+ f.seek(-3, os.SEEK_CUR)
+ self.assertEqual('bar', f.read(3))
+
+
class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
def test_replication_desired_kept_on_load(self):
@@ -1051,7 +1084,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
('add', './count2.txt', c2["count2.txt"]),
('del', './count1.txt', c1["count1.txt"]),
])
- f = c1.open("count1.txt", "w")
+ f = c1.open("count1.txt", "wb")
f.write(b"zzzzz")
# c1 changed, so it should not be deleted.
@@ -1063,7 +1096,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c2 = Collection('. 5348b82a029fd9e971a811ce1f71360b+43 0:10:count1.txt')
d = c1.diff(c2)
self.assertEqual(d, [('mod', './count1.txt', c1["count1.txt"], c2["count1.txt"])])
- f = c1.open("count1.txt", "w")
+ f = c1.open("count1.txt", "wb")
f.write(b"zzzzz")
# c1 changed, so c2 mod will go to a conflict file
@@ -1080,7 +1113,7 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
('add', './count1.txt', c2["count1.txt"]),
('del', './count2.txt', c1["count2.txt"]),
])
- f = c1.open("count1.txt", "w")
+ f = c1.open("count1.txt", "wb")
f.write(b"zzzzz")
# c1 added count1.txt, so c2 add will go to a conflict file
@@ -1106,22 +1139,22 @@ class NewCollectionTestCase(unittest.TestCase, CollectionTestMixin):
c1 = Collection()
events = []
c1.subscribe(lambda event, collection, name, item: events.append((event, collection, name, item)))
- f = c1.open("foo.txt", "w")
+ f = c1.open("foo.txt", "wb")
self.assertEqual(events[0], (arvados.collection.ADD, c1, "foo.txt", f.arvadosfile))
def test_open_w(self):
c1 = Collection(". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count1.txt\n")
self.assertEqual(c1["count1.txt"].size(), 10)
- c1.open("count1.txt", "w").close()
+ c1.open("count1.txt", "wb").close()
self.assertEqual(c1["count1.txt"].size(), 0)
class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers):
def test_get_manifest_text_only_committed(self):
c = Collection()
- with c.open("count.txt", "w") as f:
+ with c.open("count.txt", "wb") as f:
# One file committed
- with c.open("foo.txt", "w") as foo:
+ with c.open("foo.txt", "wb") as foo:
foo.write(b"foo")
foo.flush() # Force block commit
f.write(b"0123456789")
@@ -1144,14 +1177,14 @@ class NewCollectionTestCaseWithServers(run_test_server.TestCaseWithServers):
def test_only_small_blocks_are_packed_together(self):
c = Collection()
# Write a couple of small files,
- f = c.open("count.txt", "w")
+ f = c.open("count.txt", "wb")
f.write(b"0123456789")
f.close(flush=False)
- foo = c.open("foo.txt", "w")
+ foo = c.open("foo.txt", "wb")
foo.write(b"foo")
foo.close(flush=False)
# Then, write a big file, it shouldn't be packed with the ones above
- big = c.open("bigfile.txt", "w")
+ big = c.open("bigfile.txt", "wb")
big.write(b"x" * 1024 * 1024 * 33) # 33 MB > KEEP_BLOCK_SIZE/2
big.close(flush=False)
self.assertEqual(
@@ -1172,7 +1205,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
self.assertEqual(c.portable_data_hash(), "d41d8cd98f00b204e9800998ecf8427e+0")
self.assertEqual(c.api_response()["portable_data_hash"], "d41d8cd98f00b204e9800998ecf8427e+0" )
- with c.open("count.txt", "w") as f:
+ with c.open("count.txt", "wb") as f:
f.write(b"0123456789")
self.assertEqual(c.portable_manifest_text(), ". 781e5e245d69b566979b86e28d23f2c7+10 0:10:count.txt\n")
@@ -1198,7 +1231,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
c1.save()
c2 = Collection(c1.manifest_locator())
- with c2.open("count.txt", "w") as f:
+ with c2.open("count.txt", "wb") as f:
f.write(b"abcdefg")
diff = c1.diff(c2)
@@ -1226,7 +1259,7 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
c1.save()
c2 = arvados.collection.Collection(c1.manifest_locator())
- with c2.open("count.txt", "w") as f:
+ with c2.open("count.txt", "wb") as f:
f.write(b"abcdefg")
c2.save()
@@ -1240,11 +1273,11 @@ class CollectionCreateUpdateTest(run_test_server.TestCaseWithServers):
c1 = self.create_count_txt()
c1.save()
- with c1.open("count.txt", "w") as f:
+ with c1.open("count.txt", "wb") as f:
f.write(b"XYZ")
c2 = arvados.collection.Collection(c1.manifest_locator())
- with c2.open("count.txt", "w") as f:
+ with c2.open("count.txt", "wb") as f:
f.write(b"abcdefg")
c2.save()
commit 0487aeb518af7f4ba4a7b0ae17d42805dcdc3805
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Apr 4 13:12:23 2017 -0400
11308: Create missing temp dir. Unset proxy env during tests.
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 60eba55..cb4c82b 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -745,6 +745,8 @@ install_apiserver() {
&& test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
&& psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
+ mkdir -p "$WORKSPACE/services/api/tmp/pids"
+
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
&& RAILS_ENV=test bundle exec rake db:setup \
@@ -791,6 +793,8 @@ install_workbench() {
}
do_install apps/workbench workbench
+unset http_proxy https_proxy no_proxy
+
test_doclinkchecker() {
(
set -e
commit 46f6db136c5820555109963f5382b449aff617ce
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Apr 4 13:11:41 2017 -0400
11308: Remove obsolete test setup steps.
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 23cb6ec..60eba55 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -732,33 +732,12 @@ install_apiserver() {
if [ -n "$CONFIGSRC" ]
then
- for f in database.yml application.yml
+ for f in database.yml
do
cp "$CONFIGSRC/$f" config/ || fatal "$f"
done
fi
- # Fill in a random secret_token and blob_signing_key for testing
- SECRET_TOKEN=`echo 'puts rand(2**512).to_s(36)' |ruby`
- BLOB_SIGNING_KEY=`echo 'puts rand(2**512).to_s(36)' |ruby`
-
- sed -i'' -e "s:SECRET_TOKEN:$SECRET_TOKEN:" config/application.yml
- sed -i'' -e "s:BLOB_SIGNING_KEY:$BLOB_SIGNING_KEY:" config/application.yml
-
- # Set up empty git repo (for git tests)
- GITDIR=$(mktemp -d)
- sed -i'' -e "s:/var/cache/git:$GITDIR:" config/application.default.yml
-
- rm -rf $GITDIR
- mkdir -p $GITDIR/test
- cd $GITDIR/test \
- && git init \
- && git config user.email "jenkins at ci.curoverse.com" \
- && git config user.name "Jenkins, CI" \
- && touch tmp \
- && git add tmp \
- && git commit -m 'initial commit'
-
# Clear out any lingering postgresql connections to the test
# database, so that we can drop it. This assumes the current user
# is a postgresql superuser.
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list