[arvados] updated: 2.1.0-2716-g381d8aa15
git repository hosting
git at public.arvados.org
Tue Aug 2 14:41:21 UTC 2022
Summary of changes:
sdk/cwl/arvados_cwl/executor.py | 15 ++++++++++-----
sdk/cwl/arvados_cwl/runner.py | 26 +++++++++++++++-----------
2 files changed, 25 insertions(+), 16 deletions(-)
via 381d8aa1544bebfc39761636eaf4ca07427be783 (commit)
via 2b95ec7a9a6516a7dd2554e2dd8ecf9ddcfbceb7 (commit)
via b8749c42d9ae650f4e2dbeeda2a8cdae9266ff2a (commit)
from 20aa08040bc12e6387455755ce3fadc238c19d76 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 381d8aa1544bebfc39761636eaf4ca07427be783
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Aug 2 10:41:11 2022 -0400
19280: Fix typo
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 934aeb401..b00a30ad5 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -665,7 +665,7 @@ def upload_workflow_deps(arvrunner, tool, runtimeContext):
def upload_tool_deps(deptool):
if "id" in deptool:
discovered_secondaryfiles = {}
- with Perf(metrics, "upload_dependencies %s" % shortname(deptool["id"]):
+ with Perf(metrics, "upload_dependencies %s" % shortname(deptool["id"])):
pm = upload_dependencies(arvrunner,
"%s dependencies" % (shortname(deptool["id"])),
document_loader,
commit 2b95ec7a9a6516a7dd2554e2dd8ecf9ddcfbceb7
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Aug 2 10:39:55 2022 -0400
19280: more perf metrics
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 3e8115ddb..3241fb607 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -565,8 +565,9 @@ The 'jobs' API is no longer supported.
self.project_uuid = runtimeContext.project_uuid
# Upload local file references in the job order.
- job_order = upload_job_order(self, "%s input" % runtimeContext.name,
- updated_tool, job_order, runtimeContext)
+ with Perf(metrics, "upload_job_order"):
+ job_order = upload_job_order(self, "%s input" % runtimeContext.name,
+ updated_tool, job_order, runtimeContext)
# the last clause means: if it is a command line tool, and we
# are going to wait for the result, and always_submit_runner
@@ -588,14 +589,16 @@ The 'jobs' API is no longer supported.
# document with updating disabled because we want to
# submit the document with its original CWL version, not
# the auto-updated one.
- tool = load_tool(updated_tool.tool["id"], loadingContext)
+ with Perf(metrics, "load_tool original"):
+ tool = load_tool(updated_tool.tool["id"], loadingContext)
else:
tool = updated_tool
# Upload direct dependencies of workflow steps, get back mapping of files to keep references.
# Also uploads docker images.
logger.info("Uploading workflow dependencies")
- merged_map = upload_workflow_deps(self, tool, runtimeContext)
+ with Perf(metrics, "upload_workflow_deps"):
+ merged_map = upload_workflow_deps(self, tool, runtimeContext)
# Recreate process object (ArvadosWorkflow or
# ArvadosCommandTool) because tool document may have been
@@ -604,7 +607,8 @@ The 'jobs' API is no longer supported.
loadingContext.loader = tool.doc_loader
loadingContext.avsc_names = tool.doc_schema
loadingContext.metadata = tool.metadata
- tool = load_tool(tool.tool, loadingContext)
+ with Perf(metrics, "load_tool"):
+ tool = load_tool(tool.tool, loadingContext)
if runtimeContext.update_workflow or runtimeContext.create_workflow:
# Create a pipeline template or workflow record and exit.
commit b8749c42d9ae650f4e2dbeeda2a8cdae9266ff2a
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Aug 2 10:30:10 2022 -0400
19280: Add metrics checkpoints
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 9620079ed..3e8115ddb 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -594,6 +594,7 @@ The 'jobs' API is no longer supported.
# Upload direct dependencies of workflow steps, get back mapping of files to keep references.
# Also uploads docker images.
+ logger.info("Uploading workflow dependencies")
merged_map = upload_workflow_deps(self, tool, runtimeContext)
# Recreate process object (ArvadosWorkflow or
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index ed4abbe2e..934aeb401 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -49,8 +49,10 @@ from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, col
from ._version import __version__
from . import done
from . context import ArvRuntimeContext
+from .perf import Perf
logger = logging.getLogger('arvados.cwl-runner')
+metrics = logging.getLogger('arvados.cwl-runner.metrics')
def trim_anonymous_location(obj):
"""Remove 'location' field from File and Directory literals.
@@ -653,7 +655,8 @@ FileUpdates = namedtuple("FileUpdates", ["resolved", "secondaryFiles"])
def upload_workflow_deps(arvrunner, tool, runtimeContext):
# Ensure that Docker images needed by this workflow are available
- upload_docker(arvrunner, tool, runtimeContext)
+ with Perf(metrics, "upload_docker"):
+ upload_docker(arvrunner, tool, runtimeContext)
document_loader = tool.doc_loader
@@ -662,16 +665,17 @@ def upload_workflow_deps(arvrunner, tool, runtimeContext):
def upload_tool_deps(deptool):
if "id" in deptool:
discovered_secondaryfiles = {}
- pm = upload_dependencies(arvrunner,
- "%s dependencies" % (shortname(deptool["id"])),
- document_loader,
- deptool,
- deptool["id"],
- False,
- runtimeContext,
- include_primary=False,
- discovered_secondaryfiles=discovered_secondaryfiles,
- cache=tool_dep_cache)
+ with Perf(metrics, "upload_dependencies %s" % shortname(deptool["id"]):
+ pm = upload_dependencies(arvrunner,
+ "%s dependencies" % (shortname(deptool["id"])),
+ document_loader,
+ deptool,
+ deptool["id"],
+ False,
+ runtimeContext,
+ include_primary=False,
+ discovered_secondaryfiles=discovered_secondaryfiles,
+ cache=tool_dep_cache)
document_loader.idx[deptool["id"]] = deptool
toolmap = {}
for k,v in pm.items():
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list