[ARVADOS] updated: 96040562743f6056c2b243db16b5a83244894bd8

git at public.curoverse.com git at public.curoverse.com
Thu Jun 26 11:32:49 EDT 2014


Summary of changes:
 .../app/controllers/actions_controller.rb          | 38 ++++++++--------------
 sdk/cli/bin/arv                                    | 18 +++++-----
 sdk/cli/bin/crunch-job                             | 11 ++++---
 sdk/perl/lib/Arvados.pm                            |  2 +-
 sdk/python/arvados/stream.py                       |  2 +-
 services/api/script/cancel_stale_jobs.rb           |  1 -
 services/fuse/arvados_fuse/__init__.py             | 36 ++++++++------------
 7 files changed, 45 insertions(+), 63 deletions(-)

       via  96040562743f6056c2b243db16b5a83244894bd8 (commit)
       via  1a5430a1bfc73674d62cea916b5ea2823959ac13 (commit)
       via  e85c2e7fa4ec91f712caac22350260475bc69945 (commit)
       via  54e2fdfd72beeae2e7a904ca64fa11d02f5f0052 (commit)
       via  d00d9a36af61cf38fa20cc56a9a49401521cb6e1 (commit)
       via  506289bac3ad672715c66b9dc06b770e9be0ef13 (commit)
       via  985d4a785e3df9af520fa171e3483b53eca40088 (commit)
      from  8c5ea8b50cb16a893f19256575ca3bcf4bac5be3 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 96040562743f6056c2b243db16b5a83244894bd8
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 26 11:31:17 2014 -0400

    Use logging module instead of "if DEBUG:"

diff --git a/services/fuse/arvados_fuse/__init__.py b/services/fuse/arvados_fuse/__init__.py
index 2e7f4c7..6e2bf28 100644
--- a/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@ -16,12 +16,11 @@ import arvados.events
 import re
 import apiclient
 import json
+import logging
 
 from time import time
 from llfuse import FUSEError
 
-DEBUG = False
-
 class FreshBase(object):
     '''Base class for maintaining fresh/stale state to determine when to update.'''
     def __init__(self):
@@ -126,8 +125,7 @@ class Directory(FreshBase):
             try:
                 self.update()
             except apiclient.errors.HttpError as e:
-                if DEBUG:
-                    print(e, file=sys.stderr)
+                logging.debug(e)
 
     def __getitem__(self, item):
         self.checkupdate()
@@ -200,8 +198,7 @@ class CollectionDirectory(Directory):
             self.fresh()
             return True
         except Exception as detail:
-            if DEBUG:
-                print("arv-mount %s: error: %s" % (self.collection_locator,detail), file=sys.stderr)
+            logging.debug("arv-mount %s: error: %s" % (self.collection_locator,detail))
             return False
 
 class MagicDirectory(Directory):
@@ -229,15 +226,14 @@ class MagicDirectory(Directory):
             else:
                 return False
         except Exception as e:
-            if DEBUG:
-                print('arv-mount exception keep', e, file=sys.stderr)
+            logging.debug('arv-mount exception keep', e)
             return False
 
     def __getitem__(self, item):
-        if self.__contains__(item):
+        if item in self:
             return self._entries[item]
         else:
-            raise KeyError()
+            raise KeyError("No collection with id " + item)
 
 class TagsDirectory(Directory):
     '''A special directory that contains as subdirectories all tags visible to the user.'''
@@ -424,9 +420,8 @@ class Operations(llfuse.Operations):
         super(Operations, self).__init__()
 
         if debug:
-            global DEBUG
-            DEBUG = True
-            print("arv-mount debug enabled", file=sys.stderr)
+            logging.setLevel(logging.DEBUG)
+            logging.info("arv-mount debug enabled")
 
         self.inodes = Inodes()
         self.uid = uid
@@ -484,8 +479,7 @@ class Operations(llfuse.Operations):
         return entry
 
     def lookup(self, parent_inode, name):
-        if DEBUG:
-            print("arv-mount lookup: parent_inode", parent_inode, "name", name, file=sys.stderr)
+        logging.debug("arv-mount lookup: parent_inode", parent_inode, "name", name)
         inode = None
 
         if name == '.':
@@ -521,8 +515,7 @@ class Operations(llfuse.Operations):
         return fh
 
     def read(self, fh, off, size):
-        if DEBUG:
-            print("arv-mount read", fh, off, size, file=sys.stderr)
+        logging.debug("arv-mount read", fh, off, size)
         if fh in self._filehandles:
             handle = self._filehandles[fh]
         else:
@@ -539,8 +532,7 @@ class Operations(llfuse.Operations):
             del self._filehandles[fh]
 
     def opendir(self, inode):
-        if DEBUG:
-            print("arv-mount opendir: inode", inode, file=sys.stderr)
+        logging.debug("arv-mount opendir: inode", inode)
 
         if inode in self.inodes:
             p = self.inodes[inode]
@@ -561,16 +553,14 @@ class Operations(llfuse.Operations):
         return fh
 
     def readdir(self, fh, off):
-        if DEBUG:
-            print("arv-mount readdir: fh", fh, "off", off, file=sys.stderr)
+        logging.debug("arv-mount readdir: fh", fh, "off", off)
 
         if fh in self._filehandles:
             handle = self._filehandles[fh]
         else:
             raise llfuse.FUSEError(errno.EBADF)
 
-        if DEBUG:
-            print("arv-mount handle.entry", handle.entry, file=sys.stderr)
+        logging.debug("arv-mount handle.entry", handle.entry)
 
         e = off
         while e < len(handle.entry):

commit 1a5430a1bfc73674d62cea916b5ea2823959ac13
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 26 11:30:38 2014 -0400

    Remove spurious 'puts'

diff --git a/services/api/script/cancel_stale_jobs.rb b/services/api/script/cancel_stale_jobs.rb
index 717edd7..4949ec0 100755
--- a/services/api/script/cancel_stale_jobs.rb
+++ b/services/api/script/cancel_stale_jobs.rb
@@ -20,7 +20,6 @@ class CancelJobs
   def cancel_stale_jobs
     act_as_system_user do
       Job.running.each do |jobrecord|
-        puts jobrecord
         f = Log.where("object_uuid=?", jobrecord.uuid).limit(1).order("created_at desc").first
         if f
           age = (Time.now - f.created_at)

commit e85c2e7fa4ec91f712caac22350260475bc69945
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 26 11:30:14 2014 -0400

    Fixed regex for extracting just hash and size part of locator

diff --git a/sdk/python/arvados/stream.py b/sdk/python/arvados/stream.py
index ca5a9e5..cbd1ba9 100644
--- a/sdk/python/arvados/stream.py
+++ b/sdk/python/arvados/stream.py
@@ -269,7 +269,7 @@ class StreamReader(object):
         manifest_text = [self.name().replace(' ', '\\040')]
         if strip:
             for d in self._data_locators:
-                m = re.match(r'^[0-9a-f]{32}\+(\d+)*', d[LOCATOR])
+                m = re.match(r'^[0-9a-f]{32}\+\d+', d[LOCATOR])
                 manifest_text.append(m.group(0))
         else:
             manifest_text.extend([d[LOCATOR] for d in self._data_locators])

commit 54e2fdfd72beeae2e7a904ca64fa11d02f5f0052
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 26 11:29:25 2014 -0400

    Add -> indirection to  $self->{'noVerifyHostname'}

diff --git a/sdk/perl/lib/Arvados.pm b/sdk/perl/lib/Arvados.pm
index 414272b..c47f1da 100644
--- a/sdk/perl/lib/Arvados.pm
+++ b/sdk/perl/lib/Arvados.pm
@@ -135,7 +135,7 @@ sub new_request
 {
     my $self = shift;
     local $ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'};
-    if ($self{'noVerifyHostname'} || ($host =~ /\.local$/)) {
+    if ($self->{'noVerifyHostname'} || ($host =~ /\.local$/)) {
         $ENV{'PERL_LWP_SSL_VERIFY_HOSTNAME'} = 0;
     }
     Arvados::Request->new();

commit d00d9a36af61cf38fa20cc56a9a49401521cb6e1
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 26 11:28:50 2014 -0400

    Use /tmp/crunch-src for docker jobs and $ENV{CRUNCH_SRC} for non-docker.  Removed umask.

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 05185ca..219c315 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -641,7 +641,6 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
         ."mkdir -p $ENV{JOB_WORK} $ENV{CRUNCH_TMP} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
         ."&& chmod og+wrx $ENV{TASK_WORK}"
 	."&& cd $ENV{CRUNCH_TMP} ";
-    umask(077);
     if ($build_script)
     {
       $build_script_to_send = $build_script;
@@ -680,11 +679,15 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
         }
       }
       $command .= "\Q$docker_hash\E ";
+      $command .= "stdbuf -o0 -e0 ";
+      $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
     } else {
-      $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 "
+      # Non-docker run
+      $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
+      $command .= "stdbuf -o0 -e0 ";
+      $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
     }
-    $command .= "stdbuf -o0 -e0 ";
-    $command .= "/tmp/crunch-src/crunch_scripts/" . $Job->{"script"};
+
     my @execargs = ('bash', '-c', $command);
     srun (\@srunargs, \@execargs, undef, $build_script_to_send);
     exit (111);

commit 506289bac3ad672715c66b9dc06b770e9be0ef13
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 26 11:17:50 2014 -0400

    Refactored "save temp file rescue behavior into function

diff --git a/sdk/cli/bin/arv b/sdk/cli/bin/arv
index a69ba03..0d9051c 100755
--- a/sdk/cli/bin/arv
+++ b/sdk/cli/bin/arv
@@ -143,6 +143,11 @@ def check_subcommands client, arvados, subcommand, global_opts, remaining_opts
   end
 end
 
+def arv_edit_save_tmp tmp
+  FileUtils::cp tmp.path, tmp.path + ".saved"
+  puts "Saved contents to " + tmp.path + ".saved"
+end
+
 def arv_edit client, arvados, global_opts, remaining_opts
   uuid = remaining_opts.shift
   if uuid.nil? or uuid == "-h" or uuid == "--help"
@@ -184,8 +189,6 @@ def arv_edit client, arvados, global_opts, remaining_opts
     abort "Could not determine resource type #{m[2]}"
   end
 
-  require 'fileutils'
-
   api_method = 'arvados.' + rsc + '.get'
 
   result = client.execute(:api_method => eval(api_method),
@@ -258,9 +261,8 @@ def arv_edit client, arvados, global_opts, remaining_opts
           yn = $stdin.read 1
         end
         if yn == 'n' or yn == 'N'
-          FileUtils::cp tmp.path, tmp.path + ".saved"
-          puts "Saved contents to " + tmp.path + ".saved"
-          exit 1
+          arv_edit_save_tmp tmp
+          abort
         end
       end
     else
@@ -283,11 +285,10 @@ def arv_edit client, arvados, global_opts, remaining_opts
                                   authorization: 'OAuth2 '+ENV['ARVADOS_API_TOKEN']
                                 })
       rescue Exception => e
-        FileUtils::cp tmp.path, tmp.path + ".saved"
         puts "Error communicating with server, error was #{e}"
         puts "Update body was:"
         puts dumped
-        puts "Saved contents to " + tmp.path + ".saved"
+        arv_edit_save_tmp tmp
         abort
       end
 
@@ -298,11 +299,10 @@ def arv_edit client, arvados, global_opts, remaining_opts
       end
 
       if result.response.status != 200
-        FileUtils::cp tmp.path, tmp.path + ".saved"
         puts "Update failed.  Server responded #{result.response.status}: #{results['errors']} "
         puts "Update body was:"
         puts dumped
-        puts "Saved contents to " + tmp.path + ".saved"
+        arv_edit_save_tmp tmp
         abort
       end
     else

commit 985d4a785e3df9af520fa171e3483b53eca40088
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Thu Jun 26 11:17:32 2014 -0400

    Refactored arv-normalize calls into function.

diff --git a/apps/workbench/app/controllers/actions_controller.rb b/apps/workbench/app/controllers/actions_controller.rb
index 0cb6eed..bd182d4 100644
--- a/apps/workbench/app/controllers/actions_controller.rb
+++ b/apps/workbench/app/controllers/actions_controller.rb
@@ -70,6 +70,17 @@ class ActionsController < ApplicationController
     redirect_to @object
   end
 
+  def arv_normalize mt, *opts
+    r = ""
+    IO.popen(['arv-normalize'] + opts, 'w+b') do |io|
+    io.write mt
+    io.close_write
+    while buf = io.read(2**16)
+      r += buf
+    end
+    r
+  end
+
   expose_action :combine_selected_files_into_collection do
     lst = []
     files = []
@@ -93,35 +104,14 @@ class ActionsController < ApplicationController
     files.each do |m|
       mt = chash[m[1]+m[2]].manifest_text
       if m[4]
-        IO.popen(['arv-normalize', '--extract', m[4][1..-1]], 'w+b') do |io|
-          io.write mt
-          io.close_write
-          while buf = io.read(2**20)
-            combined += buf
-          end
-        end
+        combined += arv_normalize mt, '--extract', m[4][1..-1]
       else
         combined += chash[m[1]+m[2]].manifest_text
       end
     end
 
-    normalized = ''
-    IO.popen(['arv-normalize'], 'w+b') do |io|
-      io.write combined
-      io.close_write
-      while buf = io.read(2**16)
-        normalized += buf
-      end
-    end
-
-    normalized_stripped = ''
-    IO.popen(['arv-normalize', '--strip'], 'w+b') do |io|
-      io.write combined
-      io.close_write
-      while buf = io.read(2**16)
-        normalized_stripped += buf
-      end
-    end
+    normalized = arv_normalize combined
+    normalized_stripped = arv_normalize combined, '--strip'
 
     require 'digest/md5'
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list