[ARVADOS] created: 34a84410d6ce526dc626eb75a431ba7e0992d0c7
git at public.curoverse.com
git at public.curoverse.com
Thu Aug 6 23:56:19 EDT 2015
at 34a84410d6ce526dc626eb75a431ba7e0992d0c7 (commit)
commit 34a84410d6ce526dc626eb75a431ba7e0992d0c7
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 6 23:02:44 2015 -0400
#6600: Added num_retries to task_set_output, current_task, current_job, created _add_task in job_setup class to add retries to execute()
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 1df6470..f75abd9 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -19,6 +19,7 @@ import time
import threading
from .api import api, http_cache
+from .retry import retry_method
from collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
from keep import *
from stream import *
@@ -37,7 +38,8 @@ logger.addHandler(log_handler)
logger.setLevel(logging.DEBUG if config.get('ARVADOS_DEBUG')
else logging.WARNING)
-def task_set_output(self,s):
+ at retry_method
+def task_set_output(self,s,num_retries=5):
api('v1').job_tasks().update(uuid=self['uuid'],
body={
'output':s,
@@ -46,7 +48,8 @@ def task_set_output(self,s):
}).execute()
_current_task = None
-def current_task():
+ at retry_method
+def current_task(num_retries=5):
global _current_task
if _current_task:
return _current_task
@@ -58,7 +61,8 @@ def current_task():
return t
_current_job = None
-def current_job():
+ at retry_method
+def current_job(num_retries=5):
global _current_job
if _current_job:
return _current_job
@@ -82,6 +86,10 @@ class JobTask(object):
print "init jobtask %s %s" % (parameters, runtime_constraints)
class job_setup:
+ @retry_method
+ def _add_task(self, num_retries):
+ return self.num_retries
+
@staticmethod
def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False, api_client=None):
if if_sequence != current_task()['sequence']:
@@ -107,7 +115,7 @@ class job_setup:
'input':task_input
}
}
- api_client.job_tasks().create(body=new_task_attrs).execute()
+ api_client.job_tasks().create(body=new_task_attrs).execute()._add_task(num_retries=5)
if and_end_task:
api_client.job_tasks().update(uuid=current_task()['uuid'],
body={'success':True}
@@ -130,7 +138,7 @@ class job_setup:
'input':task_input
}
}
- api('v1').job_tasks().create(body=new_task_attrs).execute()
+ api('v1').job_tasks().create(body=new_task_attrs).execute()._add_task(num_retries=5)
if and_end_task:
api('v1').job_tasks().update(uuid=current_task()['uuid'],
body={'success':True}
commit 664ca8ac74121ed17d4f8f895ccc0b14be615961
Merge: b923dd3 2ba7fad
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 6 15:36:03 2015 -0400
Merge branch 'master' of https://github.com/curoverse/arvados
closes #6858
commit b923dd329a0c0ed4b5342b0eb6d87cd903ebb460
Merge: db71712 4f4d8ae
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 6 15:35:33 2015 -0400
Merge branch '6858-job-re-run-documentation'
commit 2ba7fad47eff01ab4714ef903f587cb4bbe02607
Merge: db71712 4f4d8ae
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 6 15:21:53 2015 -0400
closes #6858 Merge branch 6858-job-re-run-documentation
commit 4f4d8ae4e69ed5e374990ec56090e7d4b8926b6b
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 6 15:06:34 2015 -0400
Fixed syntax and added notes to re-use the same hashes from previous runs
diff --git a/doc/user/tutorials/running-external-program.html.textile.liquid b/doc/user/tutorials/running-external-program.html.textile.liquid
index 687379b..35d54bd 100644
--- a/doc/user/tutorials/running-external-program.html.textile.liquid
+++ b/doc/user/tutorials/running-external-program.html.textile.liquid
@@ -54,15 +54,15 @@ For more information and examples for writing pipelines, see the "pipeline templ
h2. Re-using your pipeline run
-Arvados allows users to re-use jobs that have the same inputs in order to save computing time and resources. Users are able to change a job downstream without re-computing earlier jobs. This section shows what parameters you need to version control in order to make sure Arvados will not re-compute your jobs.
+Arvados allows users to re-use jobs that have the same inputs in order to save computing time and resources. Users are able to change a job downstream without re-computing earlier jobs. This section shows which version control parameters should be tuned to make sure Arvados will not re-compute your jobs.
Note: Job reuse can only happen if all input collections do not change.
-* @"arvados_sdk_version"@ : The arvados_sdk_version parameter is used to download the specific version of the Arvados sdk into the docker image. The latest version can be found in the "Arvados Python sdk repository":https://arvados.org/projects/arvados/repository/revisions/master/show/sdk/python under *Latest revisions*.
-* @"script_version"@ : The script_version is the commit hash of the git branch that the crunch script resides. This information can be found in your git repository by using the following command:
+* @"arvados_sdk_version"@ : The arvados_sdk_version parameter is used to download the specific version of the Arvados sdk into the docker image. The latest version can be found in the "Arvados Python sdk repository":https://arvados.org/projects/arvados/repository/revisions/master/show/sdk/python under *Latest revisions*. Make sure you set this to the same version as the previous run that you are trying to reuse.
+* @"script_version"@ : The script_version is the commit hash of the git branch that the crunch script resides in. This information can be found in your git repository by using the following command:
<notextile>
<pre><code>~$ <span class="userinput">git rev-parse HEAD</span></code></pre>
</notextile>
-* @"docker_image"@ : This specifies the "Docker":https://www.docker.com/ runtime environment where job's run their scripts. Docker version control is similar to git, you can commit and push changes to your images. In order to version control your docker image on arvados, you must use the docker image hash which is found on the "Collection page":https://cloud.curoverse.com/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address*.
+* @"docker_image"@ : This specifies the "Docker":https://www.docker.com/ runtime environment where jobs run their scripts. Docker version control is similar to git, and you can commit and push changes to your images. You must re-use the docker image hash from the previous run to use the same image. It can be found on the "Collection page":https://cloud.curoverse.com/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address* or the *docker_image_locator* in a job's metadata.
commit f56323e1b8df158bc2fd41798298f6d3f25aa228
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 6 14:22:31 2015 -0400
Added new section 'Re-using your pipeline run' and moved all version control parameters there for ease of reading. Added more explanation of all parameters
diff --git a/doc/user/tutorials/running-external-program.html.textile.liquid b/doc/user/tutorials/running-external-program.html.textile.liquid
index 6f1dae3..687379b 100644
--- a/doc/user/tutorials/running-external-program.html.textile.liquid
+++ b/doc/user/tutorials/running-external-program.html.textile.liquid
@@ -24,7 +24,7 @@ This will open the template record in an interactive text editor (as specified b
* @"repository"@, @"script_version"@, and @"script"@ indicate that we intend to use the external @"run-command"@ tool wrapper that is part of the Arvados. These parameters are described in more detail in "Writing a script":tutorial-firstscript.html.
* @"runtime_constraints"@ describes runtime resource requirements for the component.
** @"docker_image"@ specifies the "Docker":https://www.docker.com/ runtime environment in which to run the job. The Docker image @"bcosc/arv-base-java"@ supplied here has the Java runtime environment, bwa, and samtools installed.
-** @"arvados_sdk_version"@ specifies a version of the Arvados SDK to load alongside the job's script.
+** @"arvados_sdk_version"@ specifies a version of the Arvados SDK to load alongside the job's script. The example uses 'master'. If you would like to use a specific version of the sdk, you can find it in the "Arvados Python sdk repository":https://arvados.org/projects/arvados/repository/revisions/master/show/sdk/python under *Latest revisions*.
* @"script_parameters"@ describes the component parameters.
** @"command"@ is the actual command line to invoke the @bwa@ and then @SortSam at . The notation @$()@ denotes macro substitution commands evaluated by the run-command tool wrapper.
** @"task.stdout"@ indicates that the output of this command should be captured to a file.
@@ -41,16 +41,6 @@ When using @run-command@, the tool should write its output to the current workin
See the "run-command reference":{{site.baseurl}}/user/topics/run-command.html for more information about using @run-command at .
-*Note:* When trying to get job reproducibility without re-computation, you need to set these parameters to their specific hashes. Using a version such as master in @"arvados_sdk_version"@ will grab the latest version hash, which will allow Arvados to re-compute your job if the sdk gets updated.
-* @"arvados_sdk_version"@ : The latest version can be found on the "Arvados Python sdk repository":https://arvados.org/projects/arvados/repository/revisions/master/show/sdk/python under *Latest revisions*.
-* @"script_version"@ : The current version of your script in your git repository can be found by using the following command:
-
-<notextile>
-<pre><code>~$ <span class="userinput">git rev-parse HEAD</span></code></pre>
-</notextile>
-
-* @"docker_image"@ : The docker image hash used is found on the "Collection page":https://cloud.curoverse.com/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address*.
-
h2. Running your pipeline
Your new pipeline template should appear at the top of the Workbench "pipeline templates":{{site.arvados_workbench_host}}/pipeline_templates page. You can run your pipeline "using Workbench":tutorial-pipeline-workbench.html or the "command line.":{{site.baseurl}}/user/topics/running-pipeline-command-line.html
@@ -61,3 +51,18 @@ Test data is available in the "Arvados Tutorial":{{site.arvados_workbench_host}}
* Choose <i class="fa fa-fw fa-archive"></i> "Tutorial sample exome (3229739b505d2b878b62aed09895a55a+142)":{{site.arvados_workbench_host}}/collections/3229739b505d2b878b62aed09895a55a+142 for the "sample" parameter
For more information and examples for writing pipelines, see the "pipeline template reference":{{site.baseurl}}/api/schema/PipelineTemplate.html
+
+h2. Re-using your pipeline run
+
+Arvados allows users to re-use jobs that have the same inputs in order to save computing time and resources. Users are able to change a job downstream without re-computing earlier jobs. This section shows what parameters you need to version control in order to make sure Arvados will not re-compute your jobs.
+
+Note: Job reuse can only happen if all input collections do not change.
+
+* @"arvados_sdk_version"@ : The arvados_sdk_version parameter is used to download the specific version of the Arvados sdk into the docker image. The latest version can be found in the "Arvados Python sdk repository":https://arvados.org/projects/arvados/repository/revisions/master/show/sdk/python under *Latest revisions*.
+* @"script_version"@ : The script_version is the commit hash of the git branch that the crunch script resides. This information can be found in your git repository by using the following command:
+
+<notextile>
+<pre><code>~$ <span class="userinput">git rev-parse HEAD</span></code></pre>
+</notextile>
+
+* @"docker_image"@ : This specifies the "Docker":https://www.docker.com/ runtime environment where job's run their scripts. Docker version control is similar to git, you can commit and push changes to your images. In order to version control your docker image on arvados, you must use the docker image hash which is found on the "Collection page":https://cloud.curoverse.com/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address*.
commit 2f719adbd0eb4ce07fc2c7012bc1ce8df02e790b
Author: Bryan Cosca <bcosc at curoverse.com>
Date: Thu Aug 6 11:54:42 2015 -0400
Added note about job reproducibility and version control with arvados_sdk_version, script_version, and docker_image to Writing a Pipeline template
diff --git a/doc/user/tutorials/running-external-program.html.textile.liquid b/doc/user/tutorials/running-external-program.html.textile.liquid
index 3ed6ea6..6f1dae3 100644
--- a/doc/user/tutorials/running-external-program.html.textile.liquid
+++ b/doc/user/tutorials/running-external-program.html.textile.liquid
@@ -41,6 +41,16 @@ When using @run-command@, the tool should write its output to the current workin
See the "run-command reference":{{site.baseurl}}/user/topics/run-command.html for more information about using @run-command at .
+*Note:* When trying to get job reproducibility without re-computation, you need to set these parameters to their specific hashes. Using a version such as master in @"arvados_sdk_version"@ will grab the latest version hash, which will allow Arvados to re-compute your job if the sdk gets updated.
+* @"arvados_sdk_version"@ : The latest version can be found on the "Arvados Python sdk repository":https://arvados.org/projects/arvados/repository/revisions/master/show/sdk/python under *Latest revisions*.
+* @"script_version"@ : The current version of your script in your git repository can be found by using the following command:
+
+<notextile>
+<pre><code>~$ <span class="userinput">git rev-parse HEAD</span></code></pre>
+</notextile>
+
+* @"docker_image"@ : The docker image hash used is found on the "Collection page":https://cloud.curoverse.com/collections/qr1hi-4zz18-dov6im679g3jr1n as the *Content address*.
+
h2. Running your pipeline
Your new pipeline template should appear at the top of the Workbench "pipeline templates":{{site.arvados_workbench_host}}/pipeline_templates page. You can run your pipeline "using Workbench":tutorial-pipeline-workbench.html or the "command line.":{{site.baseurl}}/user/topics/running-pipeline-command-line.html
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list