[ARVADOS] created: 1.3.0-347-g6ab33cb91
Git user
git at public.curoverse.com
Fri Feb 22 16:00:22 EST 2019
at 6ab33cb91de748f4f8ec8e12377f0b0650804db1 (commit)
commit 6ab33cb91de748f4f8ec8e12377f0b0650804db1
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Fri Feb 22 15:59:25 2019 -0500
9865: Executor exception improvement
- Fixed issue where BaseExeptions (SystemExit, KeyboardInterrupt) were being handled incorrectly
- Added a return statement to clarify that the function is meant to return after catching the exception
- Added explicit handling of a potential WorkflowException
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index ce7ea1ad6..f6db61ffc 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -393,9 +393,9 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
for i in self.intermediate_output_collections:
try:
self.api.collections().delete(uuid=i).execute(num_retries=self.num_retries)
- except:
+ except Exception:
logger.warning("Failed to delete intermediate output: %s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
- if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
+ except (KeyboardInterrupt, SystemExit):
break
def check_features(self, obj):
@@ -508,6 +508,7 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
}).execute(num_retries=self.num_retries)
except Exception:
logger.exception("Setting container output")
+ return
elif self.work_api == "jobs" and "TASK_UUID" in os.environ:
self.api.job_tasks().update(uuid=os.environ["TASK_UUID"],
body={
@@ -731,8 +732,11 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
except:
if sys.exc_info()[0] is KeyboardInterrupt or sys.exc_info()[0] is SystemExit:
logger.error("Interrupted, workflow will be cancelled")
- else:
- logger.error("Execution failed:\n%s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
+ elif isinstance(sys.exc_info()[1], WorkflowException):
+ logger.error("Workflow execution failed:\n%s", sys.exc_info()[1], exc_info=(sys.exc_info()[1] if self.debug else False))
+ else:
+ logger.exception("Workflow execution failed")
+
if self.pipeline:
self.api.pipeline_instances().update(uuid=self.pipeline["uuid"],
body={"state": "Failed"}).execute(num_retries=self.num_retries)
commit 0fc8778c47397f7e5de8481e0a3e7716ccc53d21
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Fri Feb 22 15:53:38 2019 -0500
9865: Removes handling of exceptions derived from BaseException
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/fsaccess.py b/sdk/cwl/arvados_cwl/fsaccess.py
index fc7cc42d1..3744b4a93 100644
--- a/sdk/cwl/arvados_cwl/fsaccess.py
+++ b/sdk/cwl/arvados_cwl/fsaccess.py
@@ -240,8 +240,8 @@ class CollectionFetcher(DefaultFetcher):
return True
except arvados.errors.NotFoundError:
return False
- except:
- logger.exception("Got unexpected exception checking if file exists:")
+ except Exception:
+ logger.exception("Got unexpected exception checking if file exists")
return False
return super(CollectionFetcher, self).check_exists(url)
commit e7ce0cd2e144e9139cdbdaaab364803039627922
Author: Eric Biagiotti <ebiagiotti at veritasgenetcs.com>
Date: Fri Feb 22 15:42:45 2019 -0500
9865: Simplifies ACR exception handling
- Ensures stack trace info is included where appropriate
- Adds comments clarifying exclusion of stack traces in some instances
- Removes exceptions from the formatted log string since logger.exception will include it
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti at veritasgenetics.com>
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 52fd4d21a..834ca195f 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -289,10 +289,12 @@ def main(args, stdout, stderr, api_client=None, keep_client=None,
if keep_client is None:
keep_client = arvados.keep.KeepClient(api_client=api_client, num_retries=4)
executor = ArvCwlExecutor(api_client, arvargs, keep_client=keep_client, num_retries=4)
- except Exception as e:
- logger.error(e)
+ except Exception:
+ logger.exception("Error creating the Arvados CWL Executor")
return 1
+ # Note that unless in debug mode, some stack traces related to user
+ # workflow errors may be suppressed. See ArvadosJob.done().
if arvargs.debug:
logger.setLevel(logging.DEBUG)
logging.getLogger('arvados').setLevel(logging.DEBUG)
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index af7c02a8f..5fe29bc8f 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -304,8 +304,8 @@ class ArvadosContainer(JobBase):
logger.info("%s reused container %s", self.arvrunner.label(self), response["container_uuid"])
else:
logger.info("%s %s state is %s", self.arvrunner.label(self), response["uuid"], response["state"])
- except Exception as e:
- logger.error("%s got error %s" % (self.arvrunner.label(self), str(e)))
+ except Exception:
+ logger.exception("%s got an error", self.arvrunner.label(self))
self.output_callback({}, "permanentFail")
def done(self, record):
@@ -353,11 +353,13 @@ class ArvadosContainer(JobBase):
if container["output"]:
outputs = done.done_outputs(self, container, "/tmp", self.outdir, "/keep")
except WorkflowException as e:
+ # Only include a stack trace if in debug mode.
+ # A stack trace may obfuscate more useful output about the workflow.
logger.error("%s unable to collect output from %s:\n%s",
self.arvrunner.label(self), container["output"], e, exc_info=(e if self.arvrunner.debug else False))
processStatus = "permanentFail"
- except Exception as e:
- logger.exception("%s while getting output object: %s", self.arvrunner.label(self), e)
+ except Exception:
+ logger.exception("%s while getting output object:", self.arvrunner.label(self))
processStatus = "permanentFail"
finally:
self.output_callback(outputs, processStatus)
@@ -523,8 +525,8 @@ class RunnerContainer(Runner):
container = self.arvrunner.api.containers().get(
uuid=record["container_uuid"]
).execute(num_retries=self.arvrunner.num_retries)
- except Exception as e:
- logger.exception("%s while getting runner container: %s", self.arvrunner.label(self), e)
+ except Exception:
+ logger.exception("%s while getting runner container", self.arvrunner.label(self))
self.arvrunner.output_callback({}, "permanentFail")
else:
super(RunnerContainer, self).done(container)
diff --git a/sdk/cwl/arvados_cwl/arvdocker.py b/sdk/cwl/arvados_cwl/arvdocker.py
index 84006b47d..a8f56ad1d 100644
--- a/sdk/cwl/arvados_cwl/arvdocker.py
+++ b/sdk/cwl/arvados_cwl/arvdocker.py
@@ -63,6 +63,7 @@ def arv_docker_get_image(api_client, dockerRequirement, pull_image, project_uuid
arvados.commands.put.api_client = api_client
arvados.commands.keepdocker.main(args, stdout=sys.stderr, install_sig_handlers=False, api=api_client)
except SystemExit as e:
+ # If e.code is None or zero, then keepdocker exited normally and we can continue
if e.code:
raise WorkflowException("keepdocker exited with code %s" % e.code)
diff --git a/sdk/cwl/arvados_cwl/arvjob.py b/sdk/cwl/arvados_cwl/arvjob.py
index 69fe7e2a8..086e299a5 100644
--- a/sdk/cwl/arvados_cwl/arvjob.py
+++ b/sdk/cwl/arvados_cwl/arvjob.py
@@ -199,7 +199,7 @@ class ArvadosJob(JobBase):
e)
else:
logger.info("%s %s is %s", self.arvrunner.label(self), response["uuid"], response["state"])
- except Exception as e:
+ except Exception:
logger.exception("%s error" % (self.arvrunner.label(self)))
self.output_callback({}, "permanentFail")
@@ -224,8 +224,8 @@ class ArvadosJob(JobBase):
body={
"components": components
}).execute(num_retries=self.arvrunner.num_retries)
- except Exception as e:
- logger.info("Error adding to components: %s", e)
+ except Exception:
+ logger.exception("Error adding to components")
def done(self, record):
try:
@@ -272,10 +272,12 @@ class ArvadosJob(JobBase):
outputs = done.done(self, record, dirs["tmpdir"],
dirs["outdir"], dirs["keep"])
except WorkflowException as e:
+ # Only include a stack trace if in debug mode.
+ # This is most likely a user workflow error and a stack trace may obfuscate more useful output.
logger.error("%s unable to collect output from %s:\n%s",
self.arvrunner.label(self), record["output"], e, exc_info=(e if self.arvrunner.debug else False))
processStatus = "permanentFail"
- except Exception as e:
+ except Exception:
logger.exception("Got unknown exception while collecting output for job %s:", self.name)
processStatus = "permanentFail"
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 535cfd758..ce7ea1ad6 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -506,8 +506,8 @@ http://doc.arvados.org/install/install-api-server.html#disable_api_methods
body={
'is_trashed': True
}).execute(num_retries=self.num_retries)
- except Exception as e:
- logger.info("Setting container output: %s", e)
+ except Exception:
+ logger.exception("Setting container output")
elif self.work_api == "jobs" and "TASK_UUID" in os.environ:
self.api.job_tasks().update(uuid=os.environ["TASK_UUID"],
body={
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index ad8e903ba..65bce95f7 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -474,8 +474,8 @@ class Runner(Process):
fileobj["location"] = "keep:%s/%s" % (record["output"], path)
adjustFileObjs(outputs, keepify)
adjustDirObjs(outputs, keepify)
- except Exception as e:
- logger.exception("[%s] While getting final output object: %s", self.name, e)
+ except Exception:
+ logger.exception("[%s] While getting final output object", self.name)
self.arvrunner.output_callback({}, "permanentFail")
else:
self.arvrunner.output_callback(outputs, processStatus)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list