[ARVADOS] updated: 20ffc9676259b048a0cd08df8a54063e885b9f3e
git at public.curoverse.com
git at public.curoverse.com
Wed Jun 4 17:25:40 EDT 2014
Summary of changes:
.gitignore | 3 +
.../app/assets/stylesheets/application.css.scss | 3 +
.../app/views/application/_svg_div.html.erb | 2 +-
.../test/integration/pipeline_instances_test.rb | 4 +
doc/api/methods/users.html.textile.liquid | 4 -
...rial-gatk-variantfiltration.html.textile.liquid | 42 +------
doc/user/topics/tutorial-job1.html.textile.liquid | 12 +-
docker/api/secret_token.rb.in | 5 +
docker/build_tools/Makefile | 3 +-
docker/config.yml.example | 4 +
docker/jobs/Dockerfile | 2 +
sdk/cli/bin/arv | 2 -
services/api/Gemfile | 1 -
services/api/Gemfile.lock | 2 -
.../app/controllers/arvados/v1/jobs_controller.rb | 45 --------
.../app/controllers/arvados/v1/users_controller.rb | 37 +-----
services/api/app/models/job.rb | 19 ----
services/api/config/routes.rb | 2 -
services/keep/tools/traffic_test.py | 126 +++++++++++++++++++++
19 files changed, 155 insertions(+), 163 deletions(-)
create mode 100755 services/keep/tools/traffic_test.py
via 20ffc9676259b048a0cd08df8a54063e885b9f3e (commit)
via 30d63b582ed093d235ae4a9efdeda5de1d4e2f24 (commit)
via 06750c636edc78256a11aec392be21d63a0cceeb (commit)
via 7ea7c6bc160097c24e2403d6f98f45a4369b4c7d (commit)
via e20bd70e4deaf55e7f162307a10118c66e2faac5 (commit)
via b01245c1c7bc78b32dbc124527660b0c36b31a58 (commit)
via 436d545670d3637dcd96b429e538b995ee98a49b (commit)
via 37301d86406cc4991c6f149f00a6a0846d1808af (commit)
via fd1e12105ef7efb5e2642c8e20f032f5b15cd974 (commit)
via 3ed7de1ce58d88b95cd47dbb4c53811a8f13a91f (commit)
via 263c5c8d78109908fda64b6b327904f1d93e5c52 (commit)
via 22e76f53dad163dd1d6703d792d94527d70bb334 (commit)
via ae2ecdfac7e8fcab495bfa8f6daa812a70123668 (commit)
via b402e6a66e569167c0e990a4fdfc2843df76afd8 (commit)
via 070e2e8e80cfc88414dd2595cae5bfddce1cd94f (commit)
via ff3686b372cd598ee990acd990af2b04daf91ada (commit)
via 3e3389504c6841e160ac086328693001261e016d (commit)
via b2da714edbcb2fc203d2cbc83a940ab776be73d0 (commit)
via 20c594c3c683a492132c0fc105241689ee116c10 (commit)
from 70dd308c6d20951745d9b8e849131d0b14e7386a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 20ffc9676259b048a0cd08df8a54063e885b9f3e
Author: Tim Pierce <twp at curoverse.com>
Date: Wed Jun 4 17:24:49 2014 -0400
2865: add traffic_test.py for testing Keep performance.
Refs #2865.
diff --git a/services/keep/tools/traffic_test.py b/services/keep/tools/traffic_test.py
new file mode 100755
index 0000000..de97edd
--- /dev/null
+++ b/services/keep/tools/traffic_test.py
@@ -0,0 +1,126 @@
+#! /usr/bin/env python
+
+# traffic_test.py
+#
+# Launch a test Keep and API server and PUT and GET a bunch of blocks.
+# Can be used to simulate client traffic in Keep to evaluate memory usage,
+# error logging, performance, etc.
+#
+# This script is warty and is relatively environment-specific, but the
+# example run described below should execute cleanly.
+#
+# Usage:
+# traffic_test.py start
+# Starts the test servers.
+# traffic_test.py put file1 file2 file3 ....
+# Runs arv-put on each file.
+# traffic_test.py get hash1 hash2 hash3 ....
+# Loops forever issuing GET requests for specified blocks.
+# traffic_test.py stop
+# Stops the test servers.
+#
+# Example:
+#
+# $ ./traffic_test.py start
+# $ ./traffic_test.py put GS00253-DNA_A02_200_37.tsv.bz2 \
+# GS00253-DNA_B01_200_37.tsv.bz2 \
+# GS00253-DNA_B02_200_37.tsv.bz2
+# $ ./traffic_test.py get $(find /tmp/tmp* -type f -printf "%f ")
+# [loops forever]
+# ^C
+# $ ./traffic_test.py stop
+#
+# Multiple "get" runs may be run concurrently to evaluate Keep's handling
+# of additional concurrent clients.
+
+PYSDK_DIR = "../../../sdk/python"
+PYTEST_DIR = PYSDK_DIR + "/tests"
+ARV_PUT_PATH = PYSDK_DIR + "/bin/arv-put"
+ARV_GET_PATH = PYSDK_DIR + "/bin/arv-get"
+SECONDS_BETWEEN_GETS = 1
+
+import argparse
+import httplib2
+import os
+import random
+import subprocess
+import sys
+import time
+
+# for run_test_server.py
+sys.path.insert(0, PYSDK_DIR)
+sys.path.insert(0, PYTEST_DIR)
+import arvados
+import run_test_server
+
+def arv_cmd(*args):
+ p = subprocess.Popen([sys.executable] + list(args),
+ stdout=subprocess.PIPE)
+ (arvout, arverr) = p.communicate()
+ if p.returncode != 0:
+ print "error {} from {} {}: {}".format(
+ p.returncode, sys.executable, args, arverr)
+ sys.exit(p.returncode)
+ return arvout
+
+def start():
+ run_test_server.run()
+ run_test_server.run_keep()
+
+def put(files):
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+ run_test_server.authorize_with('active')
+ for v in ["ARVADOS_API_HOST",
+ "ARVADOS_API_HOST_INSECURE",
+ "ARVADOS_API_TOKEN"]:
+ os.environ[v] = arvados.config.settings()[v]
+
+ if not os.environ.has_key('PYTHONPATH'):
+ os.environ['PYTHONPATH'] = ''
+ os.environ['PYTHONPATH'] = "{}:{}:{}".format(
+ PYSDK_DIR, PYTEST_DIR, os.environ['PYTHONPATH'])
+
+ for c in files:
+ manifest_uuid = arv_cmd(ARV_PUT_PATH, c)
+
+def get(blocks):
+ os.environ["ARVADOS_API_HOST"] = "127.0.0.1:3001"
+
+ run_test_server.authorize_with('active')
+ for v in ["ARVADOS_API_HOST",
+ "ARVADOS_API_HOST_INSECURE",
+ "ARVADOS_API_TOKEN"]:
+ os.environ[v] = arvados.config.settings()[v]
+
+ nqueries = 0
+ while True:
+ b = random.choice(blocks)
+ print "GET /" + b
+ body = arv_cmd(ARV_GET_PATH, b)
+ print "got {} bytes".format(len(body))
+ time.sleep(SECONDS_BETWEEN_GETS)
+ nqueries = nqueries + 1
+
+def stop():
+ run_test_server.stop_keep()
+ run_test_server.stop()
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument('action',
+ type=str,
+ nargs='+',
+ help='''"start", "put", "get", "stop"''')
+ args = parser.parse_args()
+
+ if args.action[0] == 'start':
+ start()
+ elif args.action[0] == 'put':
+ put(args.action[1:])
+ elif args.action[0] == 'get':
+ get(args.action[1:])
+ elif args.action[0] == 'stop':
+ stop()
+ else:
+ print('Unrecognized action "{}"'.format(args.action))
+ print('actions are "start", "put", "get", "stop"')
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list