[ARVADOS] created: 546448eb3ebe16b8806d63cb6cadc064835cbe42

[git at public.curoverse.com]

        at  546448eb3ebe16b8806d63cb6cadc064835cbe42 (commit)


commit 546448eb3ebe16b8806d63cb6cadc064835cbe42
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Mon Sep 15 22:52:09 2014 -0400

    3609: Collect command line arguments, uploads files, builds arv-run submission.  Needs work on event listener.

diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
new file mode 100644
index 0000000..17ed3f6
--- /dev/null
+++ b/sdk/python/arvados/commands/run.py
@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+
+import arvados
+import argparse
+import json
+import re
+import os
+import stat
+import put
+import arvados.events
+import time
+
+arvrun_parser = argparse.ArgumentParser()
+arvrun_parser.add_argument('--dry-run', action="store_true")
+arvrun_parser.add_argument('docker_image')
+arvrun_parser.add_argument('command')
+arvrun_parser.add_argument('args', nargs=argparse.REMAINDER)
+
+needupload_files = []
+
+class ArvFile(object):
+    def __init__(self, prefix, fn):
+        self.prefix = prefix
+        self.fn = fn
+
+def statfile(prefix, fn, pattern):
+    absfn = os.path.abspath(fn)
+    if os.path.exists(absfn):
+        fn = os.path.abspath(fn)
+        st = os.stat(fn)
+        mount = os.path.dirname(fn)+"/.arvados#collection"
+        if os.path.exists(mount):
+            with file(mount, 'r') as f:
+                c = json.load(f)
+            return prefix+"$(file "+c["portable_data_hash"]+"/" + os.path.basename(fn) + ")"
+        else:
+            needupload_files.append(fn)
+        return ArvFile(prefix, fn[1:])
+    else:
+        return prefix+fn
+
+def main(arguments=None):
+    args = arvrun_parser.parse_args(arguments)
+
+    patterns = [re.compile("(--[^=]+=)(.*)"),
+                re.compile("(-[^=]+=)(.*)"),
+                re.compile("(-.)(.+)")]
+
+    commandargs = []
+
+    for a in args.args:
+        if a[0] == '-':
+            matched = False
+            for p in patterns:
+                m = p.match(a)
+                if m:
+                    commandargs.append(statfile(m.group(1), m.group(2), p))
+                    matched = True
+                    break
+            if not matched:
+                commandargs.append(a)
+        else:
+            commandargs.append(statfile('', a, None))
+
+    n = True
+    pathprefix = "/"
+    while n:
+        pathstep = None
+        for c in [c for c in commandargs if isinstance(c, ArvFile)]:
+            if pathstep is None:
+                sp = c.fn.split('/')
+                if len(sp) < 2:
+                    n = False
+                    break
+                pathstep = sp[0] + "/"
+            else:
+                if not c.fn.startswith(pathstep):
+                    n = False
+                    break
+        if n:
+            pathprefix += pathstep
+            for c in [c for c in commandargs if isinstance(c, ArvFile)]:
+                c.fn = c.fn[len(pathstep):]
+
+    os.chdir(pathprefix)
+
+    if args.dry_run:
+        print("cd %s" % pathprefix)
+        print("arv-put \"%s\"" % '" "'.join([c.fn for c in commandargs if isinstance(c, ArvFile)]))
+        pdh = "$(input)"
+    else:
+        pdh = put.main(["--portable-data-hash"]+[c.fn for c in commandargs if isinstance(c, ArvFile)])
+
+    commandargs = [("%s$(file %s/%s)" % (c.prefix, pdh, c.fn)) if isinstance(c, ArvFile) else c for c in commandargs]
+
+    cut = -1
+    i = -1
+    stdio = [None, None]
+    for j in xrange(0, len(commandargs)):
+        c = commandargs[j]
+        if c == '<':
+            stdio[0] = []
+            i = 0
+            cut = j if cut == -1 else cut
+        elif c == '>':
+            stdio[1] = []
+            i = 1
+            cut = j if cut == -1 else cut
+        elif i > -1:
+            stdio[i].append(c)
+
+    commandargs = commandargs[:cut]
+
+    component = {
+        "script": "run-command",
+        "script_version": "bf243e064a7a2ee4e69a87dc3ba46e949a545150",
+        "repository": "arvados",
+        "script_parameters": {
+            "command": [args.command]+commandargs
+        },
+        "runtime_constraints": {
+            "docker_image": args.docker_image
+        }
+    }
+
+    if stdio[0]:
+        component["script_parameters"]["task.stdin"] = stdio[0][0]
+    if stdio[1]:
+        component["script_parameters"]["task.stdout"] = stdio[1][0]
+
+    pipeline = {
+        "name": "",
+        "components": {
+            args.command: component
+        },
+        "state":"RunningOnServer"
+    }
+
+    if args.dry_run:
+        print(json.dumps(pipeline, indent=4))
+    else:
+        api = arvados.api('v1')
+        pi = api.pipeline_instances().create(body=pipeline).execute()
+        def stuff(x):
+            print x
+        arvados.events.subscribe(api, [["object_uuid", "=", pi["uuid"]]], stuff)
+        state = pi["state"]
+        while state == "RunningOnServer":
+            time.sleep(5)
+            state = api.pipeline_instances().get(uuid=pi["uuid"]).execute()["state"]
+
+if __name__ == '__main__':
+    main()
diff --git a/sdk/python/bin/arv-run b/sdk/python/bin/arv-run
new file mode 100755
index 0000000..41f5fd3
--- /dev/null
+++ b/sdk/python/bin/arv-run
@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+
+from arvados.commands.run import main
+main()

-----------------------------------------------------------------------


hooks/post-receive
--