[ARVADOS] updated: 6343835f8579a129cfd8cfbaa52836a7cd9a8329
git at public.curoverse.com
git at public.curoverse.com
Mon Oct 6 17:25:12 EDT 2014
Summary of changes:
apps/workbench/Gemfile | 1 +
apps/workbench/Gemfile.lock | 3 +
.../app/assets/javascripts/application.js | 22 ++++
.../app/controllers/actions_controller.rb | 23 +++-
.../app/controllers/application_controller.rb | 7 ++
.../app/controllers/collections_controller.rb | 19 +--
apps/workbench/app/helpers/provenance_helper.rb | 13 +-
.../app/views/application/_content.html.erb | 50 ++++----
.../app/views/application/_content_layout.html.erb | 2 +
.../app/views/application/_loading_modal.html.erb | 12 ++
.../views/application/_title_and_buttons.html.erb | 25 +++-
.../app/views/collections/_show_files.html.erb | 4 +-
apps/workbench/app/views/jobs/show.html.erb | 10 +-
.../pipeline_instances/_show_tab_buttons.html.erb | 9 +-
.../app/views/pipeline_templates/show.html.erb | 4 +-
.../app/views/projects/_show_tab_contents.html.erb | 2 +-
apps/workbench/app/views/projects/show.html.erb | 6 +-
.../test/functional/collections_controller_test.rb | 16 +++
.../workbench/test/integration/collections_test.rb | 60 +++++----
apps/workbench/test/integration/jobs_test.rb | 33 ++---
apps/workbench/test/integration/projects_test.rb | 56 +++++----
apps/workbench/test/test_helper.rb | 9 +-
sdk/cli/bin/crunch-job | 139 +++++++++++++--------
sdk/python/arvados/commands/put.py | 6 +-
services/api/app/models/job.rb | 7 +-
services/api/app/models/user.rb | 24 ++++
.../test/fixtures/api_client_authorizations.yml | 6 +
services/api/test/fixtures/collections.yml | 8 ++
services/api/test/fixtures/groups.yml | 9 ++
services/api/test/fixtures/links.yml | 33 +++++
services/api/test/fixtures/users.yml | 22 ++++
.../functional/arvados/v1/links_controller_test.rb | 18 +++
services/api/test/unit/job_test.rb | 14 +++
33 files changed, 488 insertions(+), 184 deletions(-)
create mode 100644 apps/workbench/app/views/application/_loading_modal.html.erb
via 6343835f8579a129cfd8cfbaa52836a7cd9a8329 (commit)
via 81f6ebc139c29723b62fc0f5e6d0588dfb85897b (commit)
via 1f469018cec0daf492a8a8e397d094ec88f6cd49 (commit)
via 6b19642e4ca1ddd14df276fd64a76e2aa65348fe (commit)
via 3b572b689b2513e193f77d78fb3b58b1fc0a6700 (commit)
via 78ae561901fff56fad51a5aea9c92f3cea6f4bb3 (commit)
via 8a88b30e4b9253cd4f73cd3c620eafc026bb5979 (commit)
via cd69027bc08cb58316b53e8b02ca56f70b5b9eca (commit)
via e3dbac204099a08a68385016c3ce30bbbda67ba1 (commit)
via c6b14938cfb90145dfcf9a8e5ea7264d63ac8035 (commit)
via 7c2d0c899c11de4d52a417e857f4ae8f968ec628 (commit)
via d65963976174bd9c94fdb0b91eeb8a281f01e7b3 (commit)
via 41190e074c3c4504b7627757250a1b188b86b54e (commit)
via 2a34da95f13c361ef32d7bfba477b0de54c38cb0 (commit)
via ec9360b6e787b8e252d7f6ee6269e1241a4beacb (commit)
via 5d0474c43296f2bac56c5b40e1d50ab0022a28b1 (commit)
via bbec60169a83a0022061201758a04b3c6023957a (commit)
via 5c1f5168af4c3d5a30fdcc5a4364da6640c8c5fd (commit)
via 77e3dec8727c1fc00665531a39583c85b2c404bb (commit)
via c76954b90045782599660a069d01efca53d026a5 (commit)
via 07f98aa421dd6b1cbe82817ca9d8540dba42f604 (commit)
via 3e2927123ab9c6c8ab48b50ed65490f809e9e18a (commit)
via fb036b6bb6a63d4cf0e05f7b0fc3bae0ff1ef026 (commit)
via 86b14bac0b6413dd033285a4357cc7edd05a22b3 (commit)
via b344d16c97448505432c0f0e9f9f964bfae07d50 (commit)
via 4096f6dfa7b7f5dfbe230922fa8fe2f9bf4b280d (commit)
via 4b3e5e50e262fc51a158c1aed1942b1ef176e2eb (commit)
via d7d1d0f327ddc42d2d06b836863f51e09793894b (commit)
via e79e5c258fe6ec542c3f2bb3c4230a5de2812b25 (commit)
via 49c507b0bb60e35b951e2f4946c28d7c6729ce88 (commit)
via 136f77bed67bd349fb73053379dcb7bae6a16592 (commit)
via fd7a118e0c88693efdb3662a5bd70b7cdb58eda3 (commit)
via 1cadb15617013ad5882ffaadc434536187618f06 (commit)
via e59d7413748d047575232b16b12e7163dbd9521f (commit)
via 55f0cfccfee8d366514a474d77cfff9befe44715 (commit)
via 0a78f5a0efe0c88fddcd2ee91065f2a8da79055a (commit)
via a800aed41550a1a0c5a9ecee08105149db1b9362 (commit)
via 350f4d5a3ef163b37de2b83bd6fefd37f1a35b8b (commit)
via 1c638636ed50c1eafbf58d4f846b7b1e34c17cad (commit)
via 6a9589b47e553bc3576c3f12492de68962af50d3 (commit)
via e8b1da0f4cf859ba6bc34e4390fbad097d7a00d9 (commit)
via 8f6d2653d4ae4ce84f8e6699d0483bd2a9885f79 (commit)
via 8af19c273f0d9b6a2711bd8a17b53d4b2f61c6c2 (commit)
via 35729f8faeae665e5a706d56cfacf01e76de99a9 (commit)
via f19ce869a14511bb8031309a75fd9ee7acfeee6c (commit)
via 37a349fc026e4559072e5b53c8855be42bf91dcd (commit)
via b1d1c5acf6f237d5e2a02414dd59e92cdb862237 (commit)
via fe8657ef0f8698837e9bdb5e9c1bf26a068efba9 (commit)
via 982b439de0d2113c1e131e53bc6b8151901d77f9 (commit)
via a9a3fe7f94ebf14a5e70a642a596010572544424 (commit)
via f0358d88ba065369e70a89205ca1ad4ea4700cd1 (commit)
via 257004f6162498831d1e47b81514e3311facde3b (commit)
via b7ab1e8793ea06c5b778ce2249b29f4ab1607c58 (commit)
via f35e15efbfe9466d8e08b939e5407e36454a0bc8 (commit)
via 0e01a46b111ee565c236f3be8d8fed894fc35a5e (commit)
via 1779b8427b66ae59340570f37c8f34e9ef2228f4 (commit)
via 9f8d2cb25aa52f269f86eee71aee8bb66c8a0214 (commit)
via c5536580df437cfb2d9e96bbd29e254f57c53869 (commit)
via a77093bdb8cbbba6860bdda67da4d19ad3ac5a0c (commit)
via c203e53c2929c2ddf1b079ec077364f8f4d23c40 (commit)
via 09c2c027beb4ac81a07aa94005d87d92389ddd8a (commit)
from dd2a958d4731695144bb7add675fa14bf590d72a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6343835f8579a129cfd8cfbaa52836a7cd9a8329
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Oct 6 17:21:25 2014 -0400
3775: Set state=Running when creating a Job in local mode.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 1eda2f0..00d8389 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -175,6 +175,7 @@ else
$Job->{'is_locked_by_uuid'} = $User->{'uuid'};
$Job->{'started_at'} = gmtime;
+ $Job->{'state'} = 'Running';
$Job = $arv->{'jobs'}->{'create'}->execute('job' => $Job);
}
commit 81f6ebc139c29723b62fc0f5e6d0588dfb85897b
Merge: dd2a958 1f46901
Author: Tom Clegg <tom at curoverse.com>
Date: Mon Oct 6 17:13:17 2014 -0400
3775: Merge branch 'master' into 3775-fetch-git-repo
Conflicts:
sdk/cli/bin/crunch-job
diff --cc sdk/cli/bin/crunch-job
index 65c81a5,cdf416f..1eda2f0
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@@ -148,35 -148,19 +147,20 @@@ my $Job
my $job_id;
my $dbh;
my $sth;
-if ($job_has_uuid)
+if ($jobspec =~ /^[-a-z\d]+$/)
{
+ # $jobspec is an Arvados UUID, not a JSON job specification
$Job = $arv->{'jobs'}->{'get'}->execute('uuid' => $jobspec);
if (!$force_unlock) {
- # If some other crunch-job process has grabbed this job (or we see
- # other evidence that the job is already underway) we exit
- # EX_TEMPFAIL so crunch-dispatch (our parent process) doesn't
- # mark the job as failed.
- if ($Job->{'is_locked_by_uuid'}) {
- Log(undef, "Job is locked by " . $Job->{'is_locked_by_uuid'});
+ # Claim this job, and make sure nobody else does
+ eval {
+ # lock() sets is_locked_by_uuid and changes state to Running.
+ $arv->{'jobs'}->{'lock'}->execute('uuid' => $Job->{'uuid'})
+ };
+ if ($@) {
+ Log(undef, "Error while locking job, exiting ".EX_TEMPFAIL);
exit EX_TEMPFAIL;
- }
- if ($Job->{'state'} ne 'Queued') {
- Log(undef, "Job state is " . $Job->{'state'} . ", but I can only start queued jobs.");
- exit EX_TEMPFAIL;
- }
- if ($Job->{'success'} ne undef) {
- Log(undef, "Job 'success' flag (" . $Job->{'success'} . ") is not null");
- exit EX_TEMPFAIL;
- }
- if ($Job->{'running'}) {
- Log(undef, "Job 'running' flag is already set");
- exit EX_TEMPFAIL;
- }
- if ($Job->{'started_at'}) {
- Log(undef, "Job 'started_at' time is already set (" . $Job->{'started_at'} . ")");
- exit EX_TEMPFAIL;
- }
+ };
}
}
else
@@@ -1357,6 -1274,73 +1332,73 @@@ sub fhbit
}
+ # Send log output to Keep via arv-put.
+ #
+ # $log_pipe_in and $log_pipe_out are the input and output filehandles to the arv-put pipe.
+ # $log_pipe_pid is the pid of the arv-put subprocess.
+ #
+ # The only functions that should access these variables directly are:
+ #
+ # log_writer_start($logfilename)
+ # Starts an arv-put pipe, reading data on stdin and writing it to
+ # a $logfilename file in an output collection.
+ #
+ # log_writer_send($txt)
+ # Writes $txt to the output log collection.
+ #
+ # log_writer_finish()
+ # Closes the arv-put pipe and returns the output that it produces.
+ #
+ # log_writer_is_active()
+ # Returns a true value if there is currently a live arv-put
+ # process, false otherwise.
+ #
+ my ($log_pipe_in, $log_pipe_out, $log_pipe_pid);
+
+ sub log_writer_start($)
+ {
+ my $logfilename = shift;
+ $log_pipe_pid = open2($log_pipe_out, $log_pipe_in,
+ 'arv-put', '--portable-data-hash',
+ '--retries', '3',
+ '--filename', $logfilename,
+ '-');
+ }
+
+ sub log_writer_send($)
+ {
+ my $txt = shift;
+ print $log_pipe_in $txt;
+ }
+
+ sub log_writer_finish()
+ {
+ return unless $log_pipe_pid;
+
+ close($log_pipe_in);
+ my $arv_put_output;
+
+ my $s = IO::Select->new($log_pipe_out);
+ if ($s->can_read(120)) {
+ sysread($log_pipe_out, $arv_put_output, 1024);
+ chomp($arv_put_output);
+ } else {
+ Log (undef, "timed out reading from 'arv-put'");
+ }
+
+ waitpid($log_pipe_pid, 0);
+ $log_pipe_pid = $log_pipe_in = $log_pipe_out = undef;
+ if ($?) {
- Log("log_writer_finish: arv-put returned error $?")
++ Log("log_writer_finish: arv-put exited ".exit_status_s($?))
+ }
+
+ return $arv_put_output;
+ }
+
+ sub log_writer_is_active() {
+ return $log_pipe_pid;
+ }
+
sub Log # ($jobstep_id, $logmessage)
{
if ($_[1] =~ /\n/) {
@@@ -1390,8 -1374,8 +1432,8 @@@ sub croa
Log (undef, $message);
freeze() if @jobstep_todo;
collate_output() if @jobstep_todo;
- cleanup();
+ cleanup() if $Job;
- save_meta() if $local_logfile;
+ save_meta() if log_writer_is_active();
die;
}
@@@ -1411,18 -1396,10 +1453,10 @@@ sub save_met
my $justcheckpoint = shift; # false if this will be the last meta saved
return if $justcheckpoint; # checkpointing is not relevant post-Warehouse.pm
- $local_logfile->flush;
- my $retry_count = put_retry_count();
- my $cmd = "arv-put --portable-data-hash --retries $retry_count " .
- "--filename ''\Q$keep_logfile\E " . quotemeta($local_logfile->filename);
- my $loglocator = `$cmd`;
- die "system $cmd exited ".exit_status_s($?) if $?;
- chomp($loglocator);
-
- $local_logfile = undef; # the temp file is automatically deleted
+ my $loglocator = log_writer_finish();
Log (undef, "log manifest is $loglocator");
$Job->{'log'} = $loglocator;
- $Job->update_attributes('log', $loglocator) if $job_has_uuid;
+ $Job->update_attributes('log', $loglocator);
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list