[ARVADOS] updated: 0821f5481edd016a3744bb50d97a9e5b99cd1a0f

Git user git at public.curoverse.com
Fri Sep 30 14:59:44 EDT 2016


Summary of changes:
 sdk/cwl/arvados_cwl/runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

       via  0821f5481edd016a3744bb50d97a9e5b99cd1a0f (commit)
       via  79af4043ed667b8f81613cb8e4bf88b0450ae8a6 (commit)
      from  fd3e91e6cd737554b4ae491a558e52f41bad3d07 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 0821f5481edd016a3744bb50d97a9e5b99cd1a0f
Merge: fd3e91e 79af404
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Fri Sep 30 14:59:32 2016 -0400

    Merge branch '9857-cwl-acceptlist-re' refs #9857

diff --cc sdk/cwl/arvados_cwl/runner.py
index 07f85bb,e824635..a4132ca
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@@ -21,98 -19,8 +21,98 @@@ from .pathmapper import ArvPathMappe
  
  logger = logging.getLogger('arvados.cwl-runner')
  
- cwltool.draft2tool.ACCEPTLIST_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
+ cwltool.draft2tool.ACCEPTLIST_RE = re.compile(r".*")
  
 +def trim_listing(obj):
 +    """Remove 'listing' field from Directory objects that are keep references.
 +
 +    When Directory objects represent Keep references, it redundant and
 +    potentially very expensive to pass fully enumerated Directory objects
 +    between instances of cwl-runner (e.g. a submitting a job, or using the
 +    RunInSingleContainer feature), so delete the 'listing' field when it is
 +    safe to do so.
 +    """
 +
 +    if obj.get("location", "").startswith("keep:") and "listing" in obj:
 +        del obj["listing"]
 +    if obj.get("location", "").startswith("_:"):
 +        del obj["location"]
 +
 +def upload_dependencies(arvrunner, name, document_loader,
 +                        workflowobj, uri, loadref_run):
 +    """Upload the dependencies of the workflowobj document to Keep.
 +
 +    Returns a pathmapper object mapping local paths to keep references.  Also
 +    does an in-place update of references in "workflowobj".
 +
 +    Use scandeps to find $import, $include, $schemas, run, File and Directory
 +    fields that represent external references.
 +
 +    If workflowobj has an "id" field, this will reload the document to ensure
 +    it is scanning the raw document prior to preprocessing.
 +    """
 +
 +    loaded = set()
 +    def loadref(b, u):
 +        joined = urlparse.urljoin(b, u)
 +        defrg, _ = urlparse.urldefrag(joined)
 +        if defrg not in loaded:
 +            loaded.add(defrg)
 +            # Use fetch_text to get raw file (before preprocessing).
 +            text = document_loader.fetch_text(defrg)
 +            if isinstance(text, bytes):
 +                textIO = StringIO(text.decode('utf-8'))
 +            else:
 +                textIO = StringIO(text)
 +            return yaml.safe_load(textIO)
 +        else:
 +            return {}
 +
 +    if loadref_run:
 +        loadref_fields = set(("$import", "run"))
 +    else:
 +        loadref_fields = set(("$import",))
 +
 +    scanobj = workflowobj
 +    if "id" in workflowobj:
 +        # Need raw file content (before preprocessing) to ensure
 +        # that external references in $include and $mixin are captured.
 +        scanobj = loadref("", workflowobj["id"])
 +
 +    sc = scandeps(uri, scanobj,
 +                  loadref_fields,
 +                  set(("$include", "$schemas", "location")),
 +                  loadref)
 +
 +    normalizeFilesDirs(sc)
 +
 +    if "id" in workflowobj:
 +        sc.append({"class": "File", "location": workflowobj["id"]})
 +
 +    mapper = ArvPathMapper(arvrunner, sc, "",
 +                           "keep:%s",
 +                           "keep:%s/%s",
 +                           name=name)
 +
 +    def setloc(p):
 +        if "location" in p and (not p["location"].startswith("_:")) and (not p["location"].startswith("keep:")):
 +            p["location"] = mapper.mapper(p["location"]).resolved
 +    adjustFileObjs(workflowobj, setloc)
 +    adjustDirObjs(workflowobj, setloc)
 +
 +    return mapper
 +
 +
 +def upload_docker(arvrunner, tool):
 +    if isinstance(tool, CommandLineTool):
 +        (docker_req, docker_is_req) = get_feature(tool, "DockerRequirement")
 +        if docker_req:
 +            arv_docker_get_image(arvrunner.api, docker_req, True, arvrunner.project_uuid)
 +    elif isinstance(tool, cwltool.workflow.Workflow):
 +        for s in tool.steps:
 +            upload_docker(arvrunner, s.embedded_tool)
 +
 +
  class Runner(object):
      def __init__(self, runner, tool, job_order, enable_reuse):
          self.arvrunner = runner

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list