[ARVADOS] created: 24c42c71335a426b1838d04698bfbcf4db39e086
git at public.curoverse.com
git at public.curoverse.com
Wed Aug 27 09:07:37 EDT 2014
at 24c42c71335a426b1838d04698bfbcf4db39e086 (commit)
commit 24c42c71335a426b1838d04698bfbcf4db39e086
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Wed Aug 27 09:07:30 2014 -0400
3342: Fix links to log files to use portable_data_hash. Remove spurious puts.
Improve error logging.
diff --git a/apps/workbench/app/helpers/pipeline_components_helper.rb b/apps/workbench/app/helpers/pipeline_components_helper.rb
index de951b3..44c01d6 100644
--- a/apps/workbench/app/helpers/pipeline_components_helper.rb
+++ b/apps/workbench/app/helpers/pipeline_components_helper.rb
@@ -3,7 +3,8 @@ module PipelineComponentsHelper
begin
render(partial: "pipeline_instances/show_components_#{template_suffix}",
locals: locals)
- rescue
+ rescue Exception => e
+ logger.error e.inspect
case fallback
when :json
render(partial: "pipeline_instances/show_components_json")
diff --git a/apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb b/apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb
index 38fe62d..fb756e3 100644
--- a/apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb
@@ -23,7 +23,6 @@
</thead>
<tbody>
<% render_pipeline_jobs.each do |pj| %>
- <% puts pj %>
<tr data-object-uuid="<%= pj[:job].andand[:uuid] %>">
<td>
<%= pj[:name] %>
@@ -43,7 +42,7 @@
</td><td>
<% if current_job.andand[:log] %>
<% fixup = /([a-f0-9]{32}\+\d+)(\+?.*)/.match(current_job[:log])%>
- <% Collection.limit(1).where(uuid: fixup[1]).each do |c| %>
+ <% Collection.limit(1).where(portable_data_hash: fixup[1]).each do |c| %>
<% c.files.first.andand do |file| %>
<%= link_to url_for(controller: 'collections', action: 'show_file', uuid: current_job[:log], file: "#{file[0]}/#{file[1]}", disposition: 'inline', size: file[2]), class: 'btn btn-default btn-xs' do %>
<i class="fa fa-fw fa-info"></i> Show log messages
commit d4c8504da527f1a984ecd1ed0da7147010f38b6e
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Aug 26 17:12:35 2014 -0400
3342: Fixing outputs
diff --git a/apps/workbench/app/helpers/pipeline_instances_helper.rb b/apps/workbench/app/helpers/pipeline_instances_helper.rb
index 396d750..b6df82f 100644
--- a/apps/workbench/app/helpers/pipeline_instances_helper.rb
+++ b/apps/workbench/app/helpers/pipeline_instances_helper.rb
@@ -99,6 +99,7 @@ module PipelineInstancesHelper
pj[:script_parameters] = pj[:job][:script_parameters] || c[:script_parameters]
pj[:script_version] = pj[:job][:script_version] || c[:script_version]
pj[:output] = pj[:job][:output]
+ pj[:output_uuid] = c[:output_uuid]
pj[:finished_at] = (Time.parse(pj[:job][:finished_at]) rescue nil)
ret << pj
end
diff --git a/apps/workbench/app/models/pipeline_instance.rb b/apps/workbench/app/models/pipeline_instance.rb
index df14f4e..9acc849 100644
--- a/apps/workbench/app/models/pipeline_instance.rb
+++ b/apps/workbench/app/models/pipeline_instance.rb
@@ -5,6 +5,15 @@ class PipelineInstance < ArvadosBase
true
end
+ def friendly_link_name
+ pipeline_name = self.name
+ if pipeline_name.nil? or pipeline_name.empty?
+ return PipelineTemplate.where(uuid: self.pipeline_template_uuid).first.name
+ else
+ return pipeline_name
+ end
+ end
+
def content_summary
begin
PipelineTemplate.find(pipeline_template_uuid).name
diff --git a/apps/workbench/app/views/collections/_show_source_summary.html.erb b/apps/workbench/app/views/collections/_show_source_summary.html.erb
index e3f2995..c1336b7 100644
--- a/apps/workbench/app/views/collections/_show_source_summary.html.erb
+++ b/apps/workbench/app/views/collections/_show_source_summary.html.erb
@@ -4,9 +4,12 @@
<% if @output_of.andand.any? %>
<p><i>This collection was the output of:</i><br />
- <%= render_arvados_object_list_start(@output_of, 'Show all jobs',
- jobs_path(filters: [['output', '=', @object.uuid]].to_json)) do |job| %>
- <%= link_to_if_arvados_object(job, friendly_name: true) %><br />
+ <% pipelines = PipelineInstance.filter([["components", "like", "%#{@object.uuid}%"]]).each do |pipeline| %>
+ <% pipeline.components.each do |cname, c| %>
+ <% if c[:output_uuid] == @object.uuid %>
+ <b><%= cname %></b> component of pipeline <b><%= link_to_if_arvados_object(pipeline, friendly_name: true) %></b> finished at <%= c[:job][:finished_at] %>
+ <% end %>
+ <% end %>
<% end %>
</p>
<% end %>
diff --git a/apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb b/apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb
index 36c097e..38fe62d 100644
--- a/apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb
+++ b/apps/workbench/app/views/pipeline_instances/_show_components_running.html.erb
@@ -23,6 +23,7 @@
</thead>
<tbody>
<% render_pipeline_jobs.each do |pj| %>
+ <% puts pj %>
<tr data-object-uuid="<%= pj[:job].andand[:uuid] %>">
<td>
<%= pj[:name] %>
@@ -34,7 +35,7 @@
</td><td>
<%= pj[:progress_bar] %>
</td>
- <% current_job = Job.find(pj[:job][:uuid]) rescue nil %>
+ <% current_job = pj[:job] rescue nil %>
<td>
<% if current_job %>
<%= render partial: 'show_object_button', locals: {object: current_job, size: 'xs', link_text: 'Show job details'} %>
@@ -51,8 +52,10 @@
<% end %>
<% end %>
</td><td>
- <% if current_job.andand[:output] %>
- <%= link_to_if_arvados_object current_job[:output], {thumbnail: true, link_text: raw('<i class="fa fa-fw fa-archive"></i> Show output files')}, {class: 'btn btn-default btn-xs'} %>
+ <% if pj[:output_uuid] %>
+ <%= link_to_if_arvados_object pj[:output_uuid], {thumbnail: true, link_text: raw('<i class="fa fa-fw fa-archive"></i> Show output files')}, {class: 'btn btn-default btn-xs'} %>
+ <% elsif current_job.andand[:output] %>
+ <%= link_to_if_arvados_object current_job[:output], {thumbnail: true, link_text: raw('<i class="fa fa-fw fa-archive"></i> Show output files')}, {class: 'btn btn-default btn-xs'} %>
<% end %>
</td>
</tr>
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index e04df77..c2e0145 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -562,19 +562,30 @@ class WhRunPipelineInstance
# succeeded. (At the top of this loop, I was still
# waiting for it to finish.)
- output_name = c[:output_name] || cname
+ debuglog "names: #{@instance[:name]} #{@template[:name]}", 0
+ if (not @instance[:name].nil?) and (not @instance[:name].empty?)
+ pipeline_name = @instance[:name]
+ else
+ fetch_template(@instance[:pipeline_template_uuid])
+ pipeline_name = @template[:name]
+ end
+ output_name = c[:output_name] || "Output of #{cname} of #{pipeline_name}"
# Create a collection located in the same project as the pipeline with the contents of the output.
portable_data_hash = c[:job][:output]
collections = $arv.collection.list(limit: 1,
- filters: [['portable_data_hash', '=', portable_data_hash]]
+ filters: [['portable_data_hash', '=', portable_data_hash]],
+ select: ["portable_data_hash", "manifest_text"]
)[:items]
if collections.any?
- newlink = $arv.collection.create collection: {
+ newcollection = {
owner_uuid: owner_uuid,
- name: "#{output_name} #{portable_data_hash[0..7]}",
+ name: "#{output_name} at #{c[:job][:finished_at]}",
portable_data_hash: collections.first[:portable_data_hash],
manifest_text: collections.first[:manifest_text]
}
+ debuglog "Creating collection #{newcollection}", 0
+ newcollection_actual = $arv.collection.create collection: newcollection
+ c[:output_uuid] = newcollection_actual[:uuid]
else
debuglog "Could not find a collection with portable data hash #{portable_data_hash}", 0
end
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 638353d..01786ab 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -868,21 +868,10 @@ else {
}
my $stripped_manifest_text = join("", @stripped_manifest_lines);
my $output = $arv->{'collections'}->{'create'}->execute('collection' => {
- 'uuid' => md5_hex($stripped_manifest_text),
'manifest_text' => $orig_manifest_text,
});
- Log(undef, "output " . $output->{uuid});
- $Job->update_attributes('output' => $output->{uuid}) if $job_has_uuid;
- if ($Job->{'output_is_persistent'}) {
- $arv->{'links'}->{'create'}->execute('link' => {
- 'tail_kind' => 'arvados#user',
- 'tail_uuid' => $User->{'uuid'},
- 'head_kind' => 'arvados#collection',
- 'head_uuid' => $Job->{'output'},
- 'link_class' => 'resources',
- 'name' => 'wants',
- });
- }
+ Log(undef, "output " . $output->{portable_data_hash});
+ $Job->update_attributes('output' => $output->{portable_data_hash}) if $job_has_uuid;
};
if ($@) {
Log (undef, "Failed to register output manifest: $@");
diff --git a/services/api/Gemfile b/services/api/Gemfile
index 20d5152..c727a3f 100644
--- a/services/api/Gemfile
+++ b/services/api/Gemfile
@@ -71,7 +71,7 @@ gem 'database_cleaner'
gem 'themes_for_rails'
-gem 'arvados-cli', '>= 0.1.20140708213257'
+gem 'arvados-cli', '>= 0.1.20140826154708'
# pg_power lets us use partial indexes in schema.rb in Rails 3
gem 'pg_power'
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index 0a9b3c8..f039a5a 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -35,13 +35,13 @@ GEM
addressable (2.3.6)
andand (1.3.3)
arel (3.0.3)
- arvados (0.1.20140825141611)
+ arvados (0.1.20140826144746)
activesupport (>= 3.2.13)
andand
google-api-client (~> 0.6.3)
json (>= 1.7.7)
jwt (>= 0.1.5, < 1.0.0)
- arvados-cli (0.1.20140825141611)
+ arvados-cli (0.1.20140826154708)
activesupport (~> 3.2, >= 3.2.13)
andand (~> 1.3, >= 1.3.3)
arvados (~> 0.1.0)
@@ -223,7 +223,7 @@ PLATFORMS
DEPENDENCIES
acts_as_api
andand
- arvados-cli (>= 0.1.20140708213257)
+ arvados-cli (>= 0.1.20140826154708)
coffee-rails (~> 3.2.0)
database_cleaner
factory_girl_rails
commit 463aed42f5c21b7e347ad13f1ee110feaa7b85a0
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Aug 26 15:47:08 2014 -0400
3342: Removing remaining traces of output_is_persistent. Added code in
arv-run-pipeline-instance to create collections owned by appropriate project.
diff --git a/apps/workbench/app/controllers/pipeline_instances_controller.rb b/apps/workbench/app/controllers/pipeline_instances_controller.rb
index f1e3634..9b31912 100644
--- a/apps/workbench/app/controllers/pipeline_instances_controller.rb
+++ b/apps/workbench/app/controllers/pipeline_instances_controller.rb
@@ -54,12 +54,6 @@ class PipelineInstancesController < ApplicationController
p.components.each do |k, v|
j = v[:job] || next
- # The graph is interested in whether the component is
- # indicated as persistent, more than whether the job
- # satisfying it (which could have been reused, or someone
- # else's) is.
- j[:output_is_persistent] = v[:output_is_persistent]
-
uuid = j[:uuid].intern
provenance[uuid] = j
pips[uuid] = 0 unless pips[uuid] != nil
diff --git a/apps/workbench/app/helpers/provenance_helper.rb b/apps/workbench/app/helpers/provenance_helper.rb
index 1cdf53a..e4ef470 100644
--- a/apps/workbench/app/helpers/provenance_helper.rb
+++ b/apps/workbench/app/helpers/provenance_helper.rb
@@ -189,11 +189,6 @@ module ProvenanceHelper
@pdata.each do |k, job|
if job[:output] == uuid.to_s
extra = { label: 'output' }
- if job[:output_is_persistent]
- extra[:label] += ' (persistent)'
- @node_extra[uuid] ||= {}
- @node_extra[uuid][:penwidth] = 4
- end
gr += edge(uuid, job_uuid(job), extra)
gr += generate_provenance_edges(job[:uuid])
end
diff --git a/apps/workbench/app/views/jobs/_show_status.html.erb b/apps/workbench/app/views/jobs/_show_status.html.erb
index 6eae8db..ede168d 100644
--- a/apps/workbench/app/views/jobs/_show_status.html.erb
+++ b/apps/workbench/app/views/jobs/_show_status.html.erb
@@ -7,7 +7,7 @@
<% else %>
Re-run job using script version:
<%= form_tag '/jobs', style: "display:inline; padding-left: 1em" do |f| %>
- <% [:script, :script_version, :repository, :output_is_persistent, :supplied_script_version, :nondeterministic].each do |d| %>
+ <% [:script, :script_version, :repository, :supplied_script_version, :nondeterministic].each do |d| %>
<%= hidden_field :job, d, :value => @object[d] %>
<% end %>
<% [:script_parameters, :runtime_constraints].each do |d| %>
@@ -17,7 +17,7 @@
<% end %>
<% if @object.respond_to? :supplied_script_version and !@object.supplied_script_version.nil? and !@object.supplied_script_version.empty? and @object.script_version != @object.supplied_script_version%>
<%= form_tag '/jobs', style: "display:inline" do |f| %>
- <% [:script, :repository, :output_is_persistent, :supplied_script_version, :nondeterministic].each do |d| %>
+ <% [:script, :repository, :supplied_script_version, :nondeterministic].each do |d| %>
<%= hidden_field :job, d, :value => @object[d] %>
<% end %>
<%= hidden_field :job, :script_version, :value => @object[:supplied_script_version] %>
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 980ce83..e04df77 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -518,7 +518,6 @@ class WhRunPipelineInstance
:script_version => c[:script_version],
:repository => c[:repository],
:nondeterministic => c[:nondeterministic],
- :output_is_persistent => c[:output_is_persistent] || false,
:runtime_constraints => c[:runtime_constraints],
:owner_uuid => owner_uuid,
}, {
@@ -562,38 +561,22 @@ class WhRunPipelineInstance
# This is my first time discovering that the job
# succeeded. (At the top of this loop, I was still
# waiting for it to finish.)
- if c[:output_is_persistent]
- # I need to make sure a resources/wants link is in
- # place to protect the output from garbage
- # collection. (Normally Crunch does this for me, but
- # here I might be reusing the output of someone else's
- # job and I need to make sure it's understood that the
- # output is valuable to me, too.)
- wanted = c[:job][:output]
- debuglog "checking for existing persistence link for #{wanted}"
- @my_user_uuid ||= $arv.user.current[:uuid]
- links = $arv.link.list(limit: 1,
- filters:
- [%w(link_class = resources),
- %w(name = wants),
- %w(tail_uuid =) + [@my_user_uuid],
- %w(head_uuid =) + [wanted]
- ])[:items]
- if links.any?
- debuglog "link already exists, uuid #{links.first[:uuid]}"
- else
- newlink = $arv.link.create link: \
- {
- link_class: 'resources',
- name: 'wants',
- tail_kind: 'arvados#user',
- tail_uuid: @my_user_uuid,
- head_kind: 'arvados#collection',
- head_uuid: wanted,
- owner_uuid: owner_uuid
- }
- debuglog "added link, uuid #{newlink[:uuid]}"
- end
+
+ output_name = c[:output_name] || cname
+ # Create a collection located in the same project as the pipeline with the contents of the output.
+ portable_data_hash = c[:job][:output]
+ collections = $arv.collection.list(limit: 1,
+ filters: [['portable_data_hash', '=', portable_data_hash]]
+ )[:items]
+ if collections.any?
+ newlink = $arv.collection.create collection: {
+ owner_uuid: owner_uuid,
+ name: "#{output_name} #{portable_data_hash[0..7]}",
+ portable_data_hash: collections.first[:portable_data_hash],
+ manifest_text: collections.first[:manifest_text]
+ }
+ else
+ debuglog "Could not find a collection with portable data hash #{portable_data_hash}", 0
end
end
elsif c[:job][:running] ||
commit 12d2dafabb0d74f7ef9c9b929a5e8fa6b2655190
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Aug 26 14:07:42 2014 -0400
3342: Migration to remove obsolete output_is_persistent column from jobs.
diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index 6ec42a1..9fc58a6 100644
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -28,7 +28,6 @@ class Job < ArvadosModel
t.add :started_at
t.add :finished_at
t.add :output
- t.add :output_is_persistent
t.add :success
t.add :running
t.add :is_locked_by_uuid
diff --git a/services/api/db/migrate/20140826180337_remove_output_is_persistent_column.rb b/services/api/db/migrate/20140826180337_remove_output_is_persistent_column.rb
new file mode 100644
index 0000000..f2744ad
--- /dev/null
+++ b/services/api/db/migrate/20140826180337_remove_output_is_persistent_column.rb
@@ -0,0 +1,9 @@
+class RemoveOutputIsPersistentColumn < ActiveRecord::Migration
+ def up
+ remove_column :jobs, :output_is_persistent
+ end
+
+ def down
+ add_column :jobs, :output_is_persistent, :boolean, null: false, default: false
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index af4bf6a..9bcb3fb 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -427,7 +427,6 @@ CREATE TABLE jobs (
runtime_constraints text,
nondeterministic boolean,
repository character varying(255),
- output_is_persistent boolean DEFAULT false NOT NULL,
supplied_script_version character varying(255),
docker_image_locator character varying(255),
name character varying(255),
@@ -2024,4 +2023,6 @@ INSERT INTO schema_migrations (version) VALUES ('20140815171049');
INSERT INTO schema_migrations (version) VALUES ('20140817035914');
-INSERT INTO schema_migrations (version) VALUES ('20140818125735');
\ No newline at end of file
+INSERT INTO schema_migrations (version) VALUES ('20140818125735');
+
+INSERT INTO schema_migrations (version) VALUES ('20140826180337');
\ No newline at end of file
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list