[ARVADOS] created: 8298d7a1b1adf8d4c66e60347984762d60e3c665
git at public.curoverse.com
git at public.curoverse.com
Tue Jun 24 09:16:43 EDT 2014
at 8298d7a1b1adf8d4c66e60347984762d60e3c665 (commit)
commit 8298d7a1b1adf8d4c66e60347984762d60e3c665
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 24 09:16:25 2014 -0400
Add support for ARVADOS_API_HOST_INSECURE environment variable. Use Net::SSL
instead of IO::Socket::SSL because the latter seems to ignore
PERL_LWP_SSL_VERIFY_HOSTNAME. no issue #
diff --git a/sdk/perl/lib/Arvados.pm b/sdk/perl/lib/Arvados.pm
index d5eca90..414272b 100644
--- a/sdk/perl/lib/Arvados.pm
+++ b/sdk/perl/lib/Arvados.pm
@@ -6,14 +6,14 @@ Arvados -- client library for Arvados services
use Arvados;
$arv = Arvados->new(apiHost => 'arvados.local');
-
+
my $instances = $arv->{'pipeline_instances'}->{'list'}->execute();
print "UUID is ", $instances->{'items'}->[0]->{'uuid'}, "\n";
-
+
$uuid = 'eiv0u-arx5y-2c5ovx43zw90gvh';
$instance = $arv->{'pipeline_instances'}->{'get'}->execute('uuid' => $uuid);
print "ETag is ", $instance->{'etag'}, "\n";
-
+
$instance->{'active'} = 1;
$instance->{'name'} = '';
$instance->save();
@@ -58,15 +58,20 @@ Default C<v1>
=cut
package Arvados;
+
+use Net::SSL (); # From Crypt-SSLeay
+BEGIN {
+ $Net::HTTPS::SSL_SOCKET_CLASS = "Net::SSL"; # Force use of Net::SSL
+}
+
use JSON;
-use Data::Dumper;
-use IO::Socket::SSL;
use Carp;
use Arvados::ResourceAccessor;
use Arvados::ResourceMethod;
use Arvados::ResourceProxy;
use Arvados::ResourceProxyList;
use Arvados::Request;
+use Data::Dumper;
$Arvados::VERSION = 0.1;
@@ -85,12 +90,15 @@ sub build
$config = load_config_file("$ENV{HOME}/.config/arvados/settings.conf");
- $self->{'authToken'} ||=
+ $self->{'authToken'} ||=
$ENV{ARVADOS_API_TOKEN} || $config->{ARVADOS_API_TOKEN};
$self->{'apiHost'} ||=
$ENV{ARVADOS_API_HOST} || $config->{ARVADOS_API_HOST};
+ $self->{'noVerifyHostname'} ||=
+ $ENV{ARVADOS_API_HOST_INSECURE};
+
$self->{'apiProtocolScheme'} ||=
$ENV{ARVADOS_API_PROTOCOL_SCHEME} ||
$config->{ARVADOS_API_PROTOCOL_SCHEME};
@@ -127,7 +135,7 @@ sub new_request
{
my $self = shift;
local $ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'};
- if ($opts{'noVerifyHostname'} || ($host =~ /\.local$/)) {
+ if ($self{'noVerifyHostname'} || ($host =~ /\.local$/)) {
$ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = 0;
}
Arvados::Request->new();
commit 1b08dfa05208e88c86490c8a2ea280c3f65c8d98
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 24 09:12:59 2014 -0400
Now includes runtime_constraints from the component when creating a new job. no issue #
diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 4810768..8e26600 100755
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -469,6 +469,7 @@ class WhRunPipelineInstance
:repository => c[:repository],
:nondeterministic => c[:nondeterministic],
:output_is_persistent => c[:output_is_persistent] || false,
+ :runtime_constraints => c[:runtime_constraints],
:owner_uuid => owner_uuid,
# TODO: Delete the following three attributes when
# supporting pre-20140418 API servers is no longer
@@ -591,7 +592,7 @@ class WhRunPipelineInstance
ended += 1
if c[:job][:success] == true
succeeded += 1
- elsif c[:job][:success] == false
+ elsif c[:job][:success] == false or c[:job][:cancelled_at]
failed += 1
end
end
commit 665812507ba914bd91918fc438faa8036c68181b
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 24 09:11:52 2014 -0400
Only attempt to preload ArvadosBase objects in render_pipeline_component_attribute. no issue #
diff --git a/apps/workbench/app/helpers/application_helper.rb b/apps/workbench/app/helpers/application_helper.rb
index 66267e0..26be289 100644
--- a/apps/workbench/app/helpers/application_helper.rb
+++ b/apps/workbench/app/helpers/application_helper.rb
@@ -306,7 +306,7 @@ module ApplicationHelper
selectables = []
attrtext = attrvalue
- if dataclass and dataclass.is_a? Class
+ if dataclass and dataclass.is_a? ArvadosBase
objects = get_n_objects_of_class dataclass, 10
objects.each do |item|
items << item
@@ -392,7 +392,7 @@ module ApplicationHelper
render opts.merge(partial: "application/#{partial}")
end
end
-
+
def fa_icon_class_for_object object
case object.class.to_s.to_sym
when :User
commit a6611c5dacf93eb127eecb355f6fbdad337e8821
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 24 09:11:08 2014 -0400
Fix get_n_objects_of only accepts ArvadosBase classes, not any class. no issue #
diff --git a/apps/workbench/app/controllers/application_controller.rb b/apps/workbench/app/controllers/application_controller.rb
index 22a5c49..ec2e95a 100644
--- a/apps/workbench/app/controllers/application_controller.rb
+++ b/apps/workbench/app/controllers/application_controller.rb
@@ -735,7 +735,7 @@ class ApplicationController < ActionController::Base
def get_n_objects_of_class dataclass, size
@objects_map_for ||= {}
- raise ArgumentError, 'Argument is not a data class' unless dataclass.is_a? Class
+ raise ArgumentError, 'Argument is not a data class' unless dataclass.is_a? ArvadosBase
raise ArgumentError, 'Argument is not a valid limit size' unless (size && size>0)
# if the objects_map_for has a value for this dataclass, and the
commit be3b0b66d25b94eb450ad1f41d9da0f2bad56fa9
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 24 09:09:42 2014 -0400
Fixed cancel_stale_jobs to authorize as system user before modifying jobs
table. no issue #
diff --git a/services/api/script/cancel_stale_jobs.rb b/services/api/script/cancel_stale_jobs.rb
index dde4cbe..717edd7 100755
--- a/services/api/script/cancel_stale_jobs.rb
+++ b/services/api/script/cancel_stale_jobs.rb
@@ -1,5 +1,6 @@
#!/usr/bin/env ruby
+
if ENV["CRUNCH_DISPATCH_LOCKFILE"]
lockfilename = ENV.delete "CRUNCH_DISPATCH_LOCKFILE"
lockfile = File.open(lockfilename, File::RDWR|File::CREAT, 0644)
@@ -13,25 +14,32 @@ ENV["RAILS_ENV"] = ARGV[0] || ENV["RAILS_ENV"] || "development"
require File.dirname(__FILE__) + '/../config/boot'
require File.dirname(__FILE__) + '/../config/environment'
-def cancel_stale_jobs
- Job.running.each do |jobrecord|
- f = Log.where("object_uuid=?", jobrecord.uuid).limit(1).order("created_at desc").first
- if f
- age = (Time.now - f.created_at)
- if age > 300
- $stderr.puts "dispatch: failing orphan job #{jobrecord.uuid}, last log is #{age} seconds old"
- # job is marked running, but not known to crunch-dispatcher, and
- # hasn't produced any log entries for 5 minutes, so mark it as failed.
- jobrecord.running = false
- jobrecord.cancelled_at ||= Time.now
- jobrecord.finished_at ||= Time.now
- if jobrecord.success.nil?
- jobrecord.success = false
+class CancelJobs
+ include ApplicationHelper
+
+ def cancel_stale_jobs
+ act_as_system_user do
+ Job.running.each do |jobrecord|
+ puts jobrecord
+ f = Log.where("object_uuid=?", jobrecord.uuid).limit(1).order("created_at desc").first
+ if f
+ age = (Time.now - f.created_at)
+ if age > 300
+ $stderr.puts "dispatch: failing orphan job #{jobrecord.uuid}, last log is #{age} seconds old"
+ # job is marked running, but not known to crunch-dispatcher, and
+ # hasn't produced any log entries for 5 minutes, so mark it as failed.
+ jobrecord.running = false
+ jobrecord.cancelled_at ||= Time.now
+ jobrecord.finished_at ||= Time.now
+ if jobrecord.success.nil?
+ jobrecord.success = false
+ end
+ jobrecord.save!
+ end
end
- jobrecord.save!
end
end
end
end
-cancel_stale_jobs
+CancelJobs.new.cancel_stale_jobs
commit 904e7f972f9d37031fe6e1c24a2e0233c63d38e2
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Tue Jun 24 09:05:59 2014 -0400
Adds "--strip" option to arv-normalize to strip off authorization tokens.
Fixed "mismatched collection uuid" in the "create collection from parts of
other collections" feature of workbench. no issue #
diff --git a/apps/workbench/app/controllers/actions_controller.rb b/apps/workbench/app/controllers/actions_controller.rb
index c1e2617..0cb6eed 100644
--- a/apps/workbench/app/controllers/actions_controller.rb
+++ b/apps/workbench/app/controllers/actions_controller.rb
@@ -109,15 +109,24 @@ class ActionsController < ApplicationController
IO.popen(['arv-normalize'], 'w+b') do |io|
io.write combined
io.close_write
- while buf = io.read(2**20)
+ while buf = io.read(2**16)
normalized += buf
end
end
+ normalized_stripped = ''
+ IO.popen(['arv-normalize', '--strip'], 'w+b') do |io|
+ io.write combined
+ io.close_write
+ while buf = io.read(2**16)
+ normalized_stripped += buf
+ end
+ end
+
require 'digest/md5'
d = Digest::MD5.new()
- d << normalized
+ d << normalized_stripped
newuuid = "#{d.hexdigest}+#{normalized.length}"
env = Hash[ENV].
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index e7b2601..8792348 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -143,9 +143,13 @@ class CollectionReader(object):
for f in s.all_files():
yield f
- def manifest_text(self):
+ def manifest_text(self, strip=False):
self._populate()
- return self._manifest_text
+ if strip:
+ m = ''.join([StreamReader(stream).manifest_text(strip=True) for stream in self._streams])
+ return m
+ else:
+ return self._manifest_text
class CollectionWriter(object):
KEEP_BLOCK_SIZE = 2**26
@@ -359,7 +363,7 @@ class CollectionWriter(object):
for x in fields[1:-1] ]
clean += fields[0] + ' ' + ' '.join(locators) + ' ' + fields[-1] + "\n"
return clean
-
+
def manifest_text(self):
self.finish_current_stream()
manifest = ''
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index dc90d8e..ca5a9e5 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -71,7 +71,7 @@ def locators_and_ranges(data_locators, range_start, range_size, debug=False):
block_size = data_locators[i][BLOCKSIZE]
block_start = data_locators[i][OFFSET]
block_end = block_start + block_size
-
+
while i < len(data_locators):
locator, block_size, block_start = data_locators[i]
block_end = block_start + block_size
@@ -169,7 +169,7 @@ class StreamFileReader(object):
dc = bz2.BZ2Decompressor()
return self.decompress(lambda segment: dc.decompress(segment), size)
elif re.search('\.gz$', self._name):
- dc = zlib.decompressobj(16+zlib.MAX_WBITS)
+ dc = zlib.decompressobj(16+zlib.MAX_WBITS)
return self.decompress(lambda segment: dc.decompress(dc.unconsumed_tail + segment), size)
else:
return self.readall(size)
@@ -209,7 +209,7 @@ class StreamReader(object):
self._keep = keep
else:
self._keep = Keep.global_client_object()
-
+
streamoffset = 0L
# parse stream
@@ -264,11 +264,16 @@ class StreamReader(object):
for locator, blocksize, segmentoffset, segmentsize in locators_and_ranges(self._data_locators, start, size):
data += self._keep.get(locator)[segmentoffset:segmentoffset+segmentsize]
return data
-
- def manifest_text(self):
+
+ def manifest_text(self, strip=False):
manifest_text = [self.name().replace(' ', '\\040')]
- manifest_text.extend([d[LOCATOR] for d in self._data_locators])
- manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name().replace(' ', '\\040'))
+ if strip:
+ for d in self._data_locators:
+ m = re.match(r'^[0-9a-f]{32}\+(\d+)*', d[LOCATOR])
+ manifest_text.append(m.group(0))
+ else:
+ manifest_text.extend([d[LOCATOR] for d in self._data_locators])
+ manifest_text.extend([' '.join(["{}:{}:{}".format(seg[LOCATOR], seg[BLOCKSIZE], f.name().replace(' ', '\\040'))
for seg in f.segments])
for f in self._files.values()])
return ' '.join(manifest_text) + '\n'
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index e063f12..d5ea18b 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -30,7 +30,10 @@ def run_command(execargs, **kwargs):
kwargs.setdefault('close_fds', True)
kwargs.setdefault('shell', False)
p = subprocess.Popen(execargs, **kwargs)
- stdoutdata, stderrdata = p.communicate(None)
+ if kwargs['stdout'] == subprocess.PIPE:
+ stdoutdata, stderrdata = p.communicate(None)
+ else:
+ p.wait()
if p.returncode != 0:
raise errors.CommandFailedError(
"run_command %s exit %d:\n%s" %
@@ -162,7 +165,7 @@ def zipball_extract(zipball, path):
break
zip_file.write(buf)
zip_file.close()
-
+
p = subprocess.Popen(["unzip",
"-q", "-o",
"-d", path,
diff --git a/sdk/python/bin/arv-normalize b/sdk/python/bin/arv-normalize
index 755b565..0506381 100755
--- a/sdk/python/bin/arv-normalize
+++ b/sdk/python/bin/arv-normalize
@@ -14,13 +14,14 @@ parser = argparse.ArgumentParser(
description='Read manifest on standard input and put normalized manifest on standard output.')
parser.add_argument('--extract', type=str, help="The file to extract from the input manifest")
+parser.add_argument('--strip', action='store_true', help="Strip authorization tokens")
args = parser.parse_args()
import arvados
r = sys.stdin.read()
-
+
cr = arvados.CollectionReader(r)
if args.extract:
@@ -36,4 +37,4 @@ if args.extract:
if fn in s.files():
sys.stdout.write(s.files()[fn].as_manifest())
else:
- sys.stdout.write(cr.manifest_text())
+ sys.stdout.write(cr.manifest_text(args.strip))
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list