[ARVADOS] updated: 96040562743f6056c2b243db16b5a83244894bd8
git at public.curoverse.com
git at public.curoverse.com
Thu Jun 26 11:32:49 EDT 2014
Summary of changes:
.../app/controllers/actions_controller.rb | 38 ++++++++--------------
sdk/cli/bin/arv | 18 +++++-----
sdk/cli/bin/crunch-job | 11 ++++---
sdk/perl/lib/Arvados.pm | 2 +-
sdk/python/arvados/stream.py | 2 +-
services/api/script/cancel_stale_jobs.rb | 1 -
services/fuse/arvados_fuse/__init__.py | 36 ++++++++------------
7 files changed, 45 insertions(+), 63 deletions(-)
via 96040562743f6056c2b243db16b5a83244894bd8 (commit)
via 1a5430a1bfc73674d62cea916b5ea2823959ac13 (commit)
via e85c2e7fa4ec91f712caac22350260475bc69945 (commit)
via 54e2fdfd72beeae2e7a904ca64fa11d02f5f0052 (commit)
via d00d9a36af61cf38fa20cc56a9a49401521cb6e1 (commit)
via 506289bac3ad672715c66b9dc06b770e9be0ef13 (commit)
via 985d4a785e3df9af520fa171e3483b53eca40088 (commit)
from 8c5ea8b50cb16a893f19256575ca3bcf4bac5be3 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 96040562743f6056c2b243db16b5a83244894bd8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jun 26 11:31:17 2014 -0400
Use logging module instead of "if DEBUG:"
diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 2e7f4c7..6e2bf28 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -16,12 +16,11 @@ import arvados.events
import re
import apiclient
import json
+import logging
from time import time
from llfuse import FUSEError
-DEBUG = False
-
class FreshBase(object):
'''Base class for maintaining fresh/stale state to determine when to update.'''
def __init__(self):
@@ -126,8 +125,7 @@ class Directory(FreshBase):
try:
self.update()
except apiclient.errors.HttpError as e:
- if DEBUG:
- print(e, file=sys.stderr)
+ logging.debug(e)
def __getitem__(self, item):
self.checkupdate()
@@ -200,8 +198,7 @@ class CollectionDirectory(Directory):
self.fresh()
return True
except Exception as detail:
- if DEBUG:
- print("arv-mount %s: error: %s" % (self.collection_locator,detail), file=sys.stderr)
+ logging.debug("arv-mount %s: error: %s" % (self.collection_locator,detail))
return False
class MagicDirectory(Directory):
@@ -229,15 +226,14 @@ class MagicDirectory(Directory):
else:
return False
except Exception as e:
- if DEBUG:
- print('arv-mount exception keep', e, file=sys.stderr)
+ logging.debug('arv-mount exception keep', e)
return False
def __getitem__(self, item):
- if self.__contains__(item):
+ if item in self:
return self._entries[item]
else:
- raise KeyError()
+ raise KeyError("No collection with id " + item)
class TagsDirectory(Directory):
'''A special directory that contains as subdirectories all tags visible to the user.'''
@@ -424,9 +420,8 @@ class Operations(llfuse.Operations):
super(Operations, self).__init__()
if debug:
- global DEBUG
- DEBUG = True
- print("arv-mount debug enabled", file=sys.stderr)
+ logging.setLevel(logging.DEBUG)
+ logging.info("arv-mount debug enabled")
self.inodes = Inodes()
self.uid = uid
@@ -484,8 +479,7 @@ class Operations(llfuse.Operations):
return entry
def lookup(self, parent_inode, name):
- if DEBUG:
- print("arv-mount lookup: parent_inode", parent_inode, "name", name, file=sys.stderr)
+ logging.debug("arv-mount lookup: parent_inode", parent_inode, "name", name)
inode = None
if name == '.':
@@ -521,8 +515,7 @@ class Operations(llfuse.Operations):
return fh
def read(self, fh, off, size):
- if DEBUG:
- print("arv-mount read", fh, off, size, file=sys.stderr)
+ logging.debug("arv-mount read", fh, off, size)
if fh in self._filehandles:
handle = self._filehandles[fh]
else:
@@ -539,8 +532,7 @@ class Operations(llfuse.Operations):
del self._filehandles[fh]
def opendir(self, inode):
- if DEBUG:
- print("arv-mount opendir: inode", inode, file=sys.stderr)
+ logging.debug("arv-mount opendir: inode", inode)
if inode in self.inodes:
p = self.inodes[inode]
@@ -561,16 +553,14 @@ class Operations(llfuse.Operations):
return fh
def readdir(self, fh, off):
- if DEBUG:
- print("arv-mount readdir: fh", fh, "off", off, file=sys.stderr)
+ logging.debug("arv-mount readdir: fh", fh, "off", off)
if fh in self._filehandles:
handle = self._filehandles[fh]
else:
raise llfuse.FUSEError(errno.EBADF)
- if DEBUG:
- print("arv-mount handle.entry", handle.entry, file=sys.stderr)
+ logging.debug("arv-mount handle.entry", handle.entry)
e = off
while e < len(handle.entry):
commit 1a5430a1bfc73674d62cea916b5ea2823959ac13
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jun 26 11:30:38 2014 -0400
Remove spurious 'puts'
diff --git a/services/api/script/cancel_stale_jobs.rb b/services/api/script/cancel_stale_jobs.rb
index 717edd7..4949ec0 100755
--- a/services/api/script/cancel_stale_jobs.rb
+++ b/services/api/script/cancel_stale_jobs.rb
@@ -20,7 +20,6 @@ class CancelJobs
def cancel_stale_jobs
act_as_system_user do
Job.running.each do |jobrecord|
- puts jobrecord
f = Log.where("object_uuid=?", jobrecord.uuid).limit(1).order("created_at desc").first
if f
age = (Time.now - f.created_at)
commit e85c2e7fa4ec91f712caac22350260475bc69945
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jun 26 11:30:14 2014 -0400
Fixed regex for extracting just hash and size part of locator
diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index ca5a9e5..cbd1ba9 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -269,7 +269,7 @@ class StreamReader(object):
manifest_text = [self.name().replace(' ', '\\040')]
if strip:
for d in self._data_locators:
- m = re.match(r'^[0-9a-f]{32}\+(\d+)*', d[LOCATOR])
+ m = re.match(r'^[0-9a-f]{32}\+\d+', d[LOCATOR])
manifest_text.append(m.group(0))
else:
manifest_text.extend([d[LOCATOR] for d in self._data_locators])
commit 54e2fdfd72beeae2e7a904ca64fa11d02f5f0052
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jun 26 11:29:25 2014 -0400
Add -> indirection to $self->{'noVerifyHostname'}
diff --git a/sdk/perl/lib/Arvados.pm b/sdk/perl/lib/Arvados.pm
index 414272b..c47f1da 100644
--- a/sdk/perl/lib/Arvados.pm
+++ b/sdk/perl/lib/Arvados.pm
@@ -135,7 +135,7 @@ sub new_request
{
my $self = shift;
local $ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'};
- if ($self{'noVerifyHostname'} || ($host =~ /\.local$/)) {
+ if ($self->{'noVerifyHostname'} || ($host =~ /\.local$/)) {
$ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = 0;
}
Arvados::Request->new();
commit d00d9a36af61cf38fa20cc56a9a49401521cb6e1
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jun 26 11:28:50 2014 -0400
Use /tmp/crunch-src for docker jobs and $ENV{CRUNCH_SRC} for non-docker. Removed umask.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 05185ca..219c315 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -641,7 +641,6 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
."mkdir -p $ENV{JOB_WORK} $ENV{CRUNCH_TMP} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
."&& chmod og+wrx $ENV{TASK_WORK}"
."&& cd $ENV{CRUNCH_TMP} ";
- umask(077);
if ($build_script)
{
$build_script_to_send = $build_script;
@@ -680,11 +679,15 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
}
}
$command .= "\Q$docker_hash\E ";
+ $command .= "stdbuf -o0 -e0 ";
+ $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
} else {
- $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 "
+ # Non-docker run
+ $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
+ $command .= "stdbuf -o0 -e0 ";
+ $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
}
- $command .= "stdbuf -o0 -e0 ";
- $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
+
my @execargs = ('bash', '-c', $command);
srun (\@srunargs, \@execargs, undef, $build_script_to_send);
exit (111);
commit 506289bac3ad672715c66b9dc06b770e9be0ef13
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jun 26 11:17:50 2014 -0400
Refactored "save temp file rescue behavior into function
diff --git a/sdk/cli/bin/arv b/sdk/cli/bin/arv
index a69ba03..0d9051c 100755
--- a/sdk/cli/bin/arv
+++ b/sdk/cli/bin/arv
@@ -143,6 +143,11 @@ def check_subcommands client, arvados, subcommand, global_opts, remaining_opts
end
end
+def arv_edit_save_tmp tmp
+ FileUtils::cp tmp.path, tmp.path + ".saved"
+ puts "Saved contents to " + tmp.path + ".saved"
+end
+
def arv_edit client, arvados, global_opts, remaining_opts
uuid = remaining_opts.shift
if uuid.nil? or uuid == "-h" or uuid == "--help"
@@ -184,8 +189,6 @@ def arv_edit client, arvados, global_opts, remaining_opts
abort "Could not determine resource type #{m[2]}"
end
- require 'fileutils'
-
api_method = 'arvados.' + rsc + '.get'
result = client.execute(:api_method => eval(api_method),
@@ -258,9 +261,8 @@ def arv_edit client, arvados, global_opts, remaining_opts
yn = $stdin.read 1
end
if yn == 'n' or yn == 'N'
- FileUtils::cp tmp.path, tmp.path + ".saved"
- puts "Saved contents to " + tmp.path + ".saved"
- exit 1
+ arv_edit_save_tmp tmp
+ abort
end
end
else
@@ -283,11 +285,10 @@ def arv_edit client, arvados, global_opts, remaining_opts
authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
})
rescue Exception => e
- FileUtils::cp tmp.path, tmp.path + ".saved"
puts "Error communicating with server, error was #{e}"
puts "Update body was:"
puts dumped
- puts "Saved contents to " + tmp.path + ".saved"
+ arv_edit_save_tmp tmp
abort
end
@@ -298,11 +299,10 @@ def arv_edit client, arvados, global_opts, remaining_opts
end
if result.response.status != 200
- FileUtils::cp tmp.path, tmp.path + ".saved"
puts "Update failed. Server responded #{result.response.status}: #{results['errors']} "
puts "Update body was:"
puts dumped
- puts "Saved contents to " + tmp.path + ".saved"
+ arv_edit_save_tmp tmp
abort
end
else
commit 985d4a785e3df9af520fa171e3483b53eca40088
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Jun 26 11:17:32 2014 -0400
Refactored arv-normalize calls into function.
diff --git a/apps/workbench/app/controllers/actions_controller.rb b/apps/workbench/app/controllers/actions_controller.rb
index 0cb6eed..bd182d4 100644
--- a/apps/workbench/app/controllers/actions_controller.rb
+++ b/apps/workbench/app/controllers/actions_controller.rb
@@ -70,6 +70,17 @@ class ActionsController < ApplicationController
redirect_to @object
end
+ def arv_normalize mt, *opts
+ r = ""
+ IO.popen(['arv-normalize'] + opts, 'w+b') do |io|
+ io.write mt
+ io.close_write
+ while buf = io.read(2**16)
+ r += buf
+ end
+ r
+ end
+
expose_action :combine_selected_files_into_collection do
lst = []
files = []
@@ -93,35 +104,14 @@ class ActionsController < ApplicationController
files.each do |m|
mt = chash[m[1]+m[2]].manifest_text
if m[4]
- IO.popen(['arv-normalize', '--extract', m[4][1..-1]], 'w+b') do |io|
- io.write mt
- io.close_write
- while buf = io.read(2**20)
- combined += buf
- end
- end
+ combined += arv_normalize mt, '--extract', m[4][1..-1]
else
combined += chash[m[1]+m[2]].manifest_text
end
end
- normalized = ''
- IO.popen(['arv-normalize'], 'w+b') do |io|
- io.write combined
- io.close_write
- while buf = io.read(2**16)
- normalized += buf
- end
- end
-
- normalized_stripped = ''
- IO.popen(['arv-normalize', '--strip'], 'w+b') do |io|
- io.write combined
- io.close_write
- while buf = io.read(2**16)
- normalized_stripped += buf
- end
- end
+ normalized = arv_normalize combined
+ normalized_stripped = arv_normalize combined, '--strip'
require 'digest/md5'
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list