[ARVADOS] updated: f88d5e28703e3de0e02a05df6d472effb03fc6a8
Git user
git at public.curoverse.com
Wed Dec 7 15:37:54 EST 2016
Summary of changes:
build/run-build-packages.sh | 6 ++---
sdk/python/tests/arvados_testutil.py | 13 +++++++++++
sdk/python/tests/test_arv_copy.py | 34 +++++++--------------------
sdk/python/tests/test_arv_keepdocker.py | 34 +++++++--------------------
sdk/python/tests/test_arv_ls.py | 35 +++++++---------------------
sdk/python/tests/test_arv_put.py | 32 +++++++------------------
sdk/python/tests/test_arv_run.py | 34 +++++++--------------------
sdk/python/tests/test_arv_ws.py | 33 +++++++-------------------
services/nodemanager/tests/test_arguments.py | 33 +++++++-------------------
services/nodemanager/tests/testutil.py | 17 ++++++++++++--
10 files changed, 92 insertions(+), 179 deletions(-)
via f88d5e28703e3de0e02a05df6d472effb03fc6a8 (commit)
via de78c0abbfdd23deebd1bc6b57a63c745c1cee12 (commit)
from fee8873d0c5eeec1bd838161357679de1a3fe0cb (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f88d5e28703e3de0e02a05df6d472effb03fc6a8
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Dec 7 17:36:43 2016 -0300
10587: Added setuptools dependency to fpm package building on arvados-python-client, arvados-fuse & arvados-node-manager.
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 7da70e7..4e87362 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -450,7 +450,7 @@ package_go_binary tools/keep-exercise keep-exercise \
# 2014-05-15
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/python/build"
-fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --deb-recommends=git
+fpm_build $WORKSPACE/sdk/python "${PYTHON2_PKG_PREFIX}-arvados-python-client" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/python/arvados_python_client.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados Python SDK" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --deb-recommends=git
# cwl-runner
cd $WORKSPACE/packages/$TARGET
@@ -502,12 +502,12 @@ fi
# not omit the python- prefix first.
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/fuse/build"
-fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver"
+fpm_build $WORKSPACE/services/fuse "${PYTHON2_PKG_PREFIX}-arvados-fuse" 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/fuse/arvados_fuse.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Keep FUSE driver" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
# The node manager
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/nodemanager/build"
-fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager"
+fpm_build $WORKSPACE/services/nodemanager arvados-node-manager 'Curoverse, Inc.' 'python' "$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/nodemanager/arvados_node_manager.egg-info/PKG-INFO)" "--url=https://arvados.org" "--description=The Arvados node manager" --depends "${PYTHON2_PKG_PREFIX}-setuptools"
# The Docker image cleaner
cd $WORKSPACE/packages/$TARGET
commit de78c0abbfdd23deebd1bc6b57a63c745c1cee12
Author: Lucas Di Pentima <lucas at curoverse.com>
Date: Wed Dec 7 15:29:23 2016 -0300
10587: Removed use of multiprocessing module on --version tests. Added a common stderr/stdout redirector context manager for code reuse.
diff --git a/sdk/python/tests/arvados_testutil.py b/sdk/python/tests/arvados_testutil.py
index 71c9b17..dae3dd3 100644
--- a/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import arvados
+import contextlib
import errno
import hashlib
import httplib
@@ -11,6 +12,7 @@ import os
import pycurl
import Queue
import shutil
+import sys
import tempfile
import unittest
@@ -50,6 +52,17 @@ def mock_api_responses(api_client, body, codes, headers={}):
def str_keep_locator(s):
return '{}+{}'.format(hashlib.md5(s).hexdigest(), len(s))
+ at contextlib.contextmanager
+def redirected_streams(stdout=None, stderr=None):
+ orig_stdout, sys.stdout = sys.stdout, stdout or sys.stdout
+ orig_stderr, sys.stderr = sys.stderr, stderr or sys.stderr
+ try:
+ yield
+ finally:
+ sys.stdout = orig_stdout
+ sys.stderr = orig_stderr
+
+
class FakeCurl:
@classmethod
def make(cls, code, body='', headers={}):
diff --git a/sdk/python/tests/test_arv_copy.py b/sdk/python/tests/test_arv_copy.py
index 4be212c..e291ee0 100644
--- a/sdk/python/tests/test_arv_copy.py
+++ b/sdk/python/tests/test_arv_copy.py
@@ -1,45 +1,29 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import multiprocessing
+import io
import os
import sys
import tempfile
import unittest
import arvados.commands.arv_copy as arv_copy
+import arvados_testutil as tutil
class ArvCopyTestCase(unittest.TestCase):
def run_copy(self, args):
sys.argv = ['arv-copy'] + args
return arv_copy.main()
- def run_copy_process(self, args):
- _, stdout_path = tempfile.mkstemp()
- _, stderr_path = tempfile.mkstemp()
- def wrap():
- def wrapper():
- sys.argv = ['arv-copy'] + args
- sys.stdout = open(stdout_path, 'w')
- sys.stderr = open(stderr_path, 'w')
- arv_copy.main()
- return wrapper
- p = multiprocessing.Process(target=wrap())
- p.start()
- p.join()
- out = open(stdout_path, 'r').read()
- err = open(stderr_path, 'r').read()
- os.unlink(stdout_path)
- os.unlink(stderr_path)
- return p.exitcode, out, err
-
def test_unsupported_arg(self):
with self.assertRaises(SystemExit):
self.run_copy(['-x=unknown'])
def test_version_argument(self):
- exitcode, out, err = self.run_copy_process(['--version'])
- self.assertEqual(0, exitcode)
- self.assertEqual('', out)
- self.assertNotEqual('', err)
- self.assertRegexpMatches(err, "[0-9]+\.[0-9]+\.[0-9]+")
+ err = io.BytesIO()
+ out = io.BytesIO()
+ with tutil.redirected_streams(stdout=out, stderr=err):
+ with self.assertRaises(SystemExit):
+ self.run_copy(['--version'])
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
diff --git a/sdk/python/tests/test_arv_keepdocker.py b/sdk/python/tests/test_arv_keepdocker.py
index 27081af..bb94db5 100644
--- a/sdk/python/tests/test_arv_keepdocker.py
+++ b/sdk/python/tests/test_arv_keepdocker.py
@@ -1,13 +1,14 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import multiprocessing
+import io
import os
import sys
import tempfile
import unittest
import arvados.commands.keepdocker as arv_keepdocker
+import arvados_testutil as tutil
class ArvKeepdockerTestCase(unittest.TestCase):
@@ -15,32 +16,15 @@ class ArvKeepdockerTestCase(unittest.TestCase):
sys.argv = ['arv-keepdocker'] + args
return arv_keepdocker.main()
- def run_arv_keepdocker_process(self, args):
- _, stdout_path = tempfile.mkstemp()
- _, stderr_path = tempfile.mkstemp()
- def wrap():
- def wrapper():
- sys.argv = ['arv-keepdocker'] + args
- sys.stdout = open(stdout_path, 'w')
- sys.stderr = open(stderr_path, 'w')
- arv_keepdocker.main()
- return wrapper
- p = multiprocessing.Process(target=wrap())
- p.start()
- p.join()
- out = open(stdout_path, 'r').read()
- err = open(stderr_path, 'r').read()
- os.unlink(stdout_path)
- os.unlink(stderr_path)
- return p.exitcode, out, err
-
def test_unsupported_arg(self):
with self.assertRaises(SystemExit):
self.run_arv_keepdocker(['-x=unknown'])
def test_version_argument(self):
- exitcode, out, err = self.run_arv_keepdocker_process(['--version'])
- self.assertEqual(0, exitcode)
- self.assertEqual('', out)
- self.assertNotEqual('', err)
- self.assertRegexpMatches(err, "[0-9]+\.[0-9]+\.[0-9]+")
+ err = io.BytesIO()
+ out = io.BytesIO()
+ with tutil.redirected_streams(stdout=out, stderr=err):
+ with self.assertRaises(SystemExit):
+ self.run_arv_keepdocker(['--version'])
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
diff --git a/sdk/python/tests/test_arv_ls.py b/sdk/python/tests/test_arv_ls.py
index ed03c12..5064f07 100644
--- a/sdk/python/tests/test_arv_ls.py
+++ b/sdk/python/tests/test_arv_ls.py
@@ -7,13 +7,12 @@ import random
import sys
import mock
import tempfile
-import multiprocessing
import arvados.errors as arv_error
import arvados.commands.ls as arv_ls
import run_test_server
-from arvados_testutil import str_keep_locator
+from arvados_testutil import str_keep_locator, redirected_streams
class ArvLsTestCase(run_test_server.TestCaseWithServers):
FAKE_UUID = 'zzzzz-4zz18-12345abcde12345'
@@ -41,25 +40,6 @@ class ArvLsTestCase(run_test_server.TestCaseWithServers):
self.stderr = io.BytesIO()
return arv_ls.main(args, self.stdout, self.stderr, api_client)
- def run_ls_process(self, args=[], api_client=None):
- _, stdout_path = tempfile.mkstemp()
- _, stderr_path = tempfile.mkstemp()
- def wrap():
- def wrapper(*args, **kwargs):
- sys.stdout = open(stdout_path, 'w')
- sys.stderr = open(stderr_path, 'w')
- arv_ls.main(*args, **kwargs)
- return wrapper
- p = multiprocessing.Process(target=wrap(),
- args=(args, sys.stdout, sys.stderr, api_client))
- p.start()
- p.join()
- out = open(stdout_path, 'r').read()
- err = open(stderr_path, 'r').read()
- os.unlink(stdout_path)
- os.unlink(stderr_path)
- return p.exitcode, out, err
-
def test_plain_listing(self):
collection, api_client = self.mock_api_for_manifest(
['. {} 0:3:one.txt 3:4:two.txt'.format(self.random_blocks(5, 2)),
@@ -102,9 +82,10 @@ class ArvLsTestCase(run_test_server.TestCaseWithServers):
self.assertNotEqual('', self.stderr.getvalue())
def test_version_argument(self):
- _, api_client = self.mock_api_for_manifest([''])
- exitcode, out, err = self.run_ls_process(['--version'])
- self.assertEqual(0, exitcode)
- self.assertEqual('', out)
- self.assertNotEqual('', err)
- self.assertRegexpMatches(err, "[0-9]+\.[0-9]+\.[0-9]+")
+ err = io.BytesIO()
+ out = io.BytesIO()
+ with redirected_streams(stdout=out, stderr=err):
+ with self.assertRaises(SystemExit):
+ self.run_ls(['--version'], None)
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 76317b6..f35e4c7 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
import apiclient
+import io
import mock
import os
import pwd
@@ -16,7 +17,6 @@ import yaml
import threading
import hashlib
import random
-import multiprocessing
from cStringIO import StringIO
@@ -397,24 +397,6 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
'098f6bcd4621d373cade4e832627b4f6')),
"did not find file stream in Keep store")
- def run_main_process(self, args):
- _, stdout_path = tempfile.mkstemp()
- _, stderr_path = tempfile.mkstemp()
- def wrap():
- def wrapper(*args, **kwargs):
- sys.stdout = open(stdout_path, 'w')
- sys.stderr = open(stderr_path, 'w')
- arv_put.main(*args, **kwargs)
- return wrapper
- p = multiprocessing.Process(target=wrap(), args=(args, sys.stdout, sys.stderr))
- p.start()
- p.join()
- out = open(stdout_path, 'r').read()
- err = open(stderr_path, 'r').read()
- os.unlink(stdout_path)
- os.unlink(stderr_path)
- return p.exitcode, out, err
-
def setUp(self):
super(ArvadosPutTest, self).setUp()
run_test_server.authorize_with('active')
@@ -428,11 +410,13 @@ class ArvadosPutTest(run_test_server.TestCaseWithServers, ArvadosBaseTestCase):
super(ArvadosPutTest, self).tearDown()
def test_version_argument(self):
- exitcode, out, err = self.run_main_process(['--version'])
- self.assertEqual(0, exitcode)
- self.assertEqual('', out)
- self.assertNotEqual('', err)
- self.assertRegexpMatches(err, "[0-9]+\.[0-9]+\.[0-9]+")
+ err = io.BytesIO()
+ out = io.BytesIO()
+ with tutil.redirected_streams(stdout=out, stderr=err):
+ with self.assertRaises(SystemExit):
+ self.call_main_with_args(['--version'])
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
def test_simple_file_put(self):
self.call_main_on_test_file()
diff --git a/sdk/python/tests/test_arv_run.py b/sdk/python/tests/test_arv_run.py
index 7afe52b..3d04d27 100644
--- a/sdk/python/tests/test_arv_run.py
+++ b/sdk/python/tests/test_arv_run.py
@@ -1,45 +1,29 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
-import multiprocessing
+import io
import os
import sys
import tempfile
import unittest
import arvados.commands.run as arv_run
+import arvados_testutil as tutil
class ArvRunTestCase(unittest.TestCase):
def run_arv_run(self, args):
sys.argv = ['arv-run'] + args
return arv_run.main()
- def run_arv_run_process(self, args):
- _, stdout_path = tempfile.mkstemp()
- _, stderr_path = tempfile.mkstemp()
- def wrap():
- def wrapper():
- sys.argv = ['arv-run'] + args
- sys.stdout = open(stdout_path, 'w')
- sys.stderr = open(stderr_path, 'w')
- arv_run.main()
- return wrapper
- p = multiprocessing.Process(target=wrap())
- p.start()
- p.join()
- out = open(stdout_path, 'r').read()
- err = open(stderr_path, 'r').read()
- os.unlink(stdout_path)
- os.unlink(stderr_path)
- return p.exitcode, out, err
-
def test_unsupported_arg(self):
with self.assertRaises(SystemExit):
self.run_arv_run(['-x=unknown'])
def test_version_argument(self):
- exitcode, out, err = self.run_arv_run_process(['--version'])
- self.assertEqual(0, exitcode)
- self.assertEqual('', out)
- self.assertNotEqual('', err)
- self.assertRegexpMatches(err, "[0-9]+\.[0-9]+\.[0-9]+")
+ err = io.BytesIO()
+ out = io.BytesIO()
+ with tutil.redirected_streams(stdout=out, stderr=err):
+ with self.assertRaises(SystemExit):
+ self.run_arv_run(['--version'])
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
diff --git a/sdk/python/tests/test_arv_ws.py b/sdk/python/tests/test_arv_ws.py
index 17706c1..2a85e04 100644
--- a/sdk/python/tests/test_arv_ws.py
+++ b/sdk/python/tests/test_arv_ws.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-import multiprocessing
+import io
import os
import sys
import tempfile
@@ -8,36 +8,21 @@ import unittest
import arvados.errors as arv_error
import arvados.commands.ws as arv_ws
+import arvados_testutil as tutil
class ArvWsTestCase(unittest.TestCase):
def run_ws(self, args):
return arv_ws.main(args)
- def run_ws_process(self, args=[], api_client=None):
- _, stdout_path = tempfile.mkstemp()
- _, stderr_path = tempfile.mkstemp()
- def wrap():
- def wrapper(*args, **kwargs):
- sys.stdout = open(stdout_path, 'w')
- sys.stderr = open(stderr_path, 'w')
- arv_ws.main(*args, **kwargs)
- return wrapper
- p = multiprocessing.Process(target=wrap(), args=(args,))
- p.start()
- p.join()
- out = open(stdout_path, 'r').read()
- err = open(stderr_path, 'r').read()
- os.unlink(stdout_path)
- os.unlink(stderr_path)
- return p.exitcode, out, err
-
def test_unsupported_arg(self):
with self.assertRaises(SystemExit):
self.run_ws(['-x=unknown'])
def test_version_argument(self):
- exitcode, out, err = self.run_ws_process(['--version'])
- self.assertEqual(0, exitcode)
- self.assertEqual('', out)
- self.assertNotEqual('', err)
- self.assertRegexpMatches(err, "[0-9]+\.[0-9]+\.[0-9]+")
+ err = io.BytesIO()
+ out = io.BytesIO()
+ with tutil.redirected_streams(stdout=out, stderr=err):
+ with self.assertRaises(SystemExit):
+ self.run_ws(['--version'])
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
diff --git a/services/nodemanager/tests/test_arguments.py b/services/nodemanager/tests/test_arguments.py
index e4749ce..f98309a 100644
--- a/services/nodemanager/tests/test_arguments.py
+++ b/services/nodemanager/tests/test_arguments.py
@@ -1,42 +1,27 @@
#!/usr/bin/env python
-import multiprocessing
+import io
import os
import sys
import tempfile
import unittest
import arvnodeman.launcher as nodeman
+from . import testutil
class ArvNodemArgumentsTestCase(unittest.TestCase):
def run_nodeman(self, args):
return nodeman.main(args)
- def run_nodeman_process(self, args=[]):
- _, stdout_path = tempfile.mkstemp()
- _, stderr_path = tempfile.mkstemp()
- def wrap():
- def wrapper(*args, **kwargs):
- sys.stdout = open(stdout_path, 'w')
- sys.stderr = open(stderr_path, 'w')
- nodeman.main(*args, **kwargs)
- return wrapper
- p = multiprocessing.Process(target=wrap(), args=(args,))
- p.start()
- p.join()
- out = open(stdout_path, 'r').read()
- err = open(stderr_path, 'r').read()
- os.unlink(stdout_path)
- os.unlink(stderr_path)
- return p.exitcode, out, err
-
def test_unsupported_arg(self):
with self.assertRaises(SystemExit):
self.run_nodeman(['-x=unknown'])
def test_version_argument(self):
- exitcode, out, err = self.run_nodeman_process(['--version'])
- self.assertEqual(0, exitcode)
- self.assertEqual('', out)
- self.assertNotEqual('', err)
- self.assertRegexpMatches(err, "[0-9]+\.[0-9]+\.[0-9]+")
+ err = io.BytesIO()
+ out = io.BytesIO()
+ with testutil.redirected_streams(stdout=out, stderr=err):
+ with self.assertRaises(SystemExit):
+ self.run_nodeman(['--version'])
+ self.assertEqual(out.getvalue(), '')
+ self.assertRegexpMatches(err.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
diff --git a/services/nodemanager/tests/testutil.py b/services/nodemanager/tests/testutil.py
index 1b6aab3..15337c4 100644
--- a/services/nodemanager/tests/testutil.py
+++ b/services/nodemanager/tests/testutil.py
@@ -2,13 +2,15 @@
from __future__ import absolute_import, print_function
+import contextlib
import datetime
+import mock
+import pykka
+import sys
import threading
import time
import libcloud.common.types as cloud_types
-import mock
-import pykka
from . import pykka_timeout
@@ -55,6 +57,17 @@ def cloud_node_fqdn(node):
def ip_address_mock(last_octet):
return '10.20.30.{}'.format(last_octet)
+ at contextlib.contextmanager
+def redirected_streams(stdout=None, stderr=None):
+ orig_stdout, sys.stdout = sys.stdout, stdout or sys.stdout
+ orig_stderr, sys.stderr = sys.stderr, stderr or sys.stderr
+ try:
+ yield
+ finally:
+ sys.stdout = orig_stdout
+ sys.stderr = orig_stderr
+
+
class MockShutdownTimer(object):
def _set_state(self, is_open, next_opening):
self.window_open = lambda: is_open
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list