[ARVADOS] created: bfcf53c5b735b3872075d6ef1295c7115c112d53
git at public.curoverse.com
git at public.curoverse.com
Tue Jun 10 16:33:32 EDT 2014
at bfcf53c5b735b3872075d6ef1295c7115c112d53 (commit)
commit bfcf53c5b735b3872075d6ef1295c7115c112d53
Author: Brett Smith <brett at curoverse.com>
Date: Tue Jun 10 16:34:01 2014 -0400
2879: Add arv-keepdocker command.
This puts a Docker image in Keep, and makes Arvados links to help find
the Collection by the image's names.
diff --git a/sdk/cli/bin/arv b/sdk/cli/bin/arv
index 31cbeec..b485b7b 100755
--- a/sdk/cli/bin/arv
+++ b/sdk/cli/bin/arv
@@ -42,13 +42,16 @@ when 'keep'
elsif ['less', 'check'].index @sub then
# wh* shims
exec `which wh#{@sub}`.strip, *ARGV
+ elsif @sub == 'docker'
+ exec `which arv-keepdocker`.strip, *ARGV
else
puts "Usage: \n" +
"#{$0} keep ls\n" +
"#{$0} keep get\n" +
"#{$0} keep put\n" +
"#{$0} keep less\n" +
- "#{$0} keep check\n"
+ "#{$0} keep check\n" +
+ "#{$0} keep docker\n"
end
abort
when 'pipeline'
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
new file mode 100644
index 0000000..f4ccb2a
--- /dev/null
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+
+import argparse
+import errno
+import json
+import os
+import subprocess
+import sys
+import tarfile
+import tempfile
+
+from collections import namedtuple
+from stat import *
+
+import arvados
+import arvados.commands._util as arv_cmd
+import arvados.commands.put as arv_put
+
+STAT_CACHE_ERRORS = (IOError, OSError, ValueError)
+
+DockerImage = namedtuple('DockerImage',
+ ['repo', 'tag', 'hash', 'created', 'vsize'])
+_docker_image_list = None
+
+opt_parser = argparse.ArgumentParser(add_help=False)
+opt_parser.add_argument(
+ '-f', '--force', action='store_true', default=False,
+ help="Re-upload the image even if it already exists on the server")
+
+_group = opt_parser.add_mutually_exclusive_group()
+_group.add_argument(
+ '--pull', action='store_true', default=True,
+ help="Pull the latest image from Docker repositories first (default)")
+_group.add_argument(
+ '--no-pull', action='store_false', dest='pull',
+ help="Don't pull images from Docker repositories")
+
+opt_parser.add_argument(
+ 'image',
+ help="Docker image to upload, as a repository name or hash")
+opt_parser.add_argument(
+ 'tag', nargs='?', default='latest',
+ help="Tag of the Docker image to upload (default 'latest')")
+
+arg_parser = argparse.ArgumentParser(
+ description="Upload a Docker image to Arvados",
+ parents=[opt_parser, arv_put.run_opts])
+
+class DockerError(Exception):
+ pass
+
+
+def popen_docker(cmd, *args, **kwargs):
+ manage_stdin = ('stdin' not in kwargs)
+ kwargs.setdefault('stdin', subprocess.PIPE)
+ kwargs.setdefault('stdout', sys.stderr)
+ try:
+ docker_proc = subprocess.Popen(['docker.io'] + cmd, *args, **kwargs)
+ except OSError: # No docker.io in $PATH
+ docker_proc = subprocess.Popen(['docker'] + cmd, *args, **kwargs)
+ if manage_stdin:
+ docker_proc.stdin.close()
+ return docker_proc
+
+def check_docker(proc, description):
+ proc.wait()
+ if proc.returncode != 0:
+ raise DockerError("docker {} returned status code {}".
+ format(description, proc.returncode))
+
+def _get_docker_images():
+ # Yield a DockerImage tuple for each installed image.
+ list_proc = popen_docker(['images', '--no-trunc'], stdout=subprocess.PIPE)
+ next(list_proc.stdout) # Ignore the header line
+ for line in list_proc.stdout:
+ words = line.split()
+ size_index = len(words) - 2
+ repo, tag, imageid = words[:3]
+ ctime = ' '.join(words[3:size_index])
+ vsize = ' '.join(words[size_index:])
+ yield DockerImage(repo, tag, imageid, ctime, vsize)
+ list_proc.stdout.close()
+ check_docker(list_proc, "images")
+
+def docker_images():
+ global _docker_image_list
+ if _docker_image_list is None:
+ _docker_image_list = list(_get_docker_images())
+ return _docker_image_list
+
+def find_image_hash(image_name, image_tag):
+ hash_search = image_name.lower()
+ hash_matches = set()
+ for image in docker_images():
+ if (image.repo == image_name) and (image.tag == image_tag):
+ return image.hash
+ elif image.hash.startswith(hash_search):
+ hash_matches.add(image.hash)
+ if len(hash_matches) == 1:
+ return hash_matches.pop()
+ return None
+
+def stat_cache_name(image_file):
+ return getattr(image_file, 'name', image_file) + '.stat'
+
+def pull_image(image_name, image_tag):
+ check_docker(popen_docker(['pull', '-t', image_tag, image_name]), "pull")
+
+def save_image(image_hash, image_file):
+ # Save the specified Docker image to image_file, then try to save its
+ # stats so we can try to resume after interruption.
+ check_docker(popen_docker(['save', image_hash], stdout=image_file),
+ "save")
+ image_file.flush()
+ try:
+ with open(stat_cache_name(image_file), 'w') as statfile:
+ json.dump(tuple(os.fstat(image_file.fileno())), statfile)
+ except STAT_CACHE_ERRORS:
+ pass # We won't resume from this cache. No big deal.
+
+def prep_image_file(filename):
+ # Return a file object ready to save a Docker image,
+ # and a boolean indicating whether or not we need to actually save the
+ # image (False if a cached save is available).
+ cache_dir = arv_cmd.make_home_conf_dir(
+ os.path.join('.cache', 'arvados', 'docker'), 0o700)
+ if cache_dir is None:
+ image_file = tempfile.NamedTemporaryFile(suffix='.tar')
+ need_save = True
+ else:
+ file_path = os.path.join(cache_dir, filename)
+ try:
+ with open(stat_cache_name(file_path)) as statfile:
+ prev_stat = json.load(statfile)
+ now_stat = os.stat(file_path)
+ need_save = any(prev_stat[field] != now_stat[field]
+ for field in [ST_MTIME, ST_SIZE])
+ except STAT_CACHE_ERRORS + (AttributeError, IndexError):
+ need_save = True # We couldn't compare against old stats
+ image_file = open(file_path, 'w+b' if need_save else 'rb')
+ return image_file, need_save
+
+def make_link(link_class, link_name, **link_attrs):
+ link_attrs.update({'link_class': link_class, 'name': link_name})
+ return arvados.api('v1').links().create(body=link_attrs).execute()
+
+def main(arguments=None):
+ args = arg_parser.parse_args(arguments)
+
+ # Pull the image if requested, unless the image is specified as a hash
+ # that we already have.
+ if args.pull and (find_image_hash(args.image, None) is None):
+ pull_image(args.image, args.tag)
+
+ image_hash = find_image_hash(args.image, args.tag)
+ if image_hash is None:
+ print >>sys.stderr, "arv-keepdocker: No image found."
+ sys.exit(1)
+ elif not args.force:
+ # Abort if this image is already in Arvados.
+ existing_links = arvados.api('v1').links().list(
+ filters=[['link_class', '=', 'docker_image_hash'],
+ ['name', '=', image_hash]]).execute()['items']
+ if existing_links:
+ message = [
+ "arv-keepdocker: Image {} already stored in collection(s):".
+ format(image_hash)]
+ message.extend(link['head_uuid'] for link in existing_links)
+ print >>sys.stderr, "\n".join(message)
+ sys.exit(1)
+
+ # Open a file for the saved image, and write it if needed.
+ outfile_name = '{}.tar'.format(image_hash)
+ image_file, need_save = prep_image_file(outfile_name)
+ if need_save:
+ save_image(image_hash, image_file)
+
+ # Call arv-put with switches we inherited from it
+ # (a.k.a., switches that aren't our own).
+ put_args = opt_parser.parse_known_args(arguments)[1]
+ coll_uuid = arv_put.main(
+ put_args + ['--filename', outfile_name, image_file.name]).strip()
+
+ # Read the image metadata and make Arvados links from it.
+ image_file.seek(0)
+ image_tar = tarfile.open(fileobj=image_file)
+ json_file = image_tar.extractfile(image_tar.getmember(image_hash + '/json'))
+ image_metadata = json.load(json_file)
+ json_file.close()
+ image_tar.close()
+ link_base = {'head_uuid': coll_uuid, 'properties': {}}
+ if 'created' in image_metadata:
+ link_base['properties']['image_timestamp'] = image_metadata['created']
+
+ make_link('docker_image_hash', image_hash, **link_base)
+ if not image_hash.startswith(args.image.lower()):
+ make_link('docker_image_repository', args.image, **link_base)
+ make_link('docker_image_tag', args.tag, **link_base)
+
+ # Clean up.
+ image_file.close()
+ for filename in [stat_cache_name(image_file), image_file.name]:
+ try:
+ os.unlink(filename)
+ except OSError as error:
+ if error.errno != errno.ENOENT:
+ raise
+
+if __name__ == '__main__':
+ main()
diff --git a/sdk/python/bin/arv-keepdocker b/sdk/python/bin/arv-keepdocker
new file mode 100755
index 0000000..20d9d62
--- /dev/null
+++ b/sdk/python/bin/arv-keepdocker
@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+
+from arvados.commands.keepdocker import main
+main()
commit cadb97cbd449f564e8948af61f00a265bf8cea3a
Author: Brett Smith <brett at curoverse.com>
Date: Tue Jun 10 11:09:42 2014 -0400
2879: arv-put main() returns what it prints.
This will enable other tools to reuse the results.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index e4e1b6d..ef34e07 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -328,7 +328,7 @@ def progress_writer(progress_func, outfile=sys.stderr):
def exit_signal_handler(sigcode, frame):
sys.exit(-sigcode)
-def main(arguments=None):
+def main(arguments=None, output_to=sys.stdout):
args = parse_arguments(arguments)
if args.progress:
@@ -346,7 +346,8 @@ def main(arguments=None):
except (IOError, OSError):
pass # Couldn't open cache directory/file. Continue without it.
except ResumeCacheConflict:
- print "arv-put: Another process is already uploading this data."
+ output_to.write(
+ "arv-put: Another process is already uploading this data.\n")
sys.exit(1)
if resume_cache is None:
@@ -382,9 +383,9 @@ def main(arguments=None):
print >>sys.stderr
if args.stream:
- print writer.manifest_text(),
+ output = writer.manifest_text()
elif args.raw:
- print ','.join(writer.data_locators())
+ output = ','.join(writer.data_locators())
else:
# Register the resulting collection in Arvados.
collection = arvados.api().collections().create(
@@ -395,7 +396,11 @@ def main(arguments=None):
).execute()
# Print the locator (uuid) of the new collection.
- print collection['uuid']
+ output = collection['uuid']
+
+ output_to.write(output)
+ if not output.endswith('\n'):
+ output_to.write('\n')
for sigcode, orig_handler in orig_signal_handlers.items():
signal.signal(sigcode, orig_handler)
@@ -403,5 +408,7 @@ def main(arguments=None):
if resume_cache is not None:
resume_cache.destroy()
+ return output
+
if __name__ == '__main__':
main()
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index b7c6ed6..4687b4e 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -12,6 +12,8 @@ import time
import unittest
import yaml
+from cStringIO import StringIO
+
import arvados
import arvados.commands.put as arv_put
@@ -323,9 +325,10 @@ class ArvadosPutReportTest(ArvadosBaseTestCase):
class ArvadosPutTest(ArvadosKeepLocalStoreTestCase):
def call_main_on_test_file(self):
+ self.main_output = StringIO()
with self.make_test_file() as testfile:
path = testfile.name
- arv_put.main(['--stream', '--no-progress', path])
+ arv_put.main(['--stream', '--no-progress', path], self.main_output)
self.assertTrue(
os.path.exists(os.path.join(os.environ['KEEP_LOCAL_STORE'],
'098f6bcd4621d373cade4e832627b4f6')),
commit 7a51a96ad5617f4ca2f4850c08a3808601f62f8c
Author: Brett Smith <brett at curoverse.com>
Date: Mon Jun 9 12:02:01 2014 -0400
2879: Move Python is_hex function to arvados.util.
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 4c2d474..82c04ea 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -25,10 +25,10 @@ global_client_object = None
from api import *
import config
import arvados.errors
+import arvados.util
class KeepLocator(object):
EPOCH_DATETIME = datetime.datetime.utcfromtimestamp(0)
- HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
def __init__(self, locator_str):
self.size = None
@@ -53,13 +53,6 @@ class KeepLocator(object):
self.permission_hint()]
if s is not None)
- def _is_hex_length(self, s, *size_spec):
- if len(size_spec) == 1:
- good_len = (len(s) == size_spec[0])
- else:
- good_len = (size_spec[0] <= len(s) <= size_spec[1])
- return good_len and self.HEX_RE.match(s)
-
def _make_hex_prop(name, length):
# Build and return a new property with the given name that
# must be a hex string of the given length.
@@ -67,7 +60,7 @@ class KeepLocator(object):
def getter(self):
return getattr(self, data_name)
def setter(self, hex_str):
- if not self._is_hex_length(hex_str, length):
+ if not arvados.util.is_hex(hex_str, length):
raise ValueError("{} must be a {}-digit hex string: {}".
format(name, length, hex_str))
setattr(self, data_name, hex_str)
@@ -82,7 +75,7 @@ class KeepLocator(object):
@perm_expiry.setter
def perm_expiry(self, value):
- if not self._is_hex_length(value, 1, 8):
+ if not arvados.util.is_hex(value, 1, 8):
raise ValueError(
"permission timestamp must be a hex Unix timestamp: {}".
format(value))
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 7148b92..e063f12 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -7,6 +7,8 @@ import errno
import sys
from arvados.collection import *
+HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
+
def clear_tmpdir(path=None):
"""
Ensure the given directory (or TASK_TMPDIR if none given)
@@ -306,3 +308,25 @@ def listdir_recursive(dirname, base=None):
else:
allfiles += [ent_base]
return allfiles
+
+def is_hex(s, *length_args):
+ """is_hex(s[, length[, max_length]]) -> boolean
+
+ Return True if s is a string of hexadecimal digits.
+ If one length argument is given, the string must contain exactly
+ that number of digits.
+ If two length arguments are given, the string must contain a number of
+ digits between those two lengths, inclusive.
+ Return False otherwise.
+ """
+ num_length_args = len(length_args)
+ if num_length_args > 2:
+ raise ArgumentError("is_hex accepts up to 3 arguments ({} given)".
+ format(1 + num_length_args))
+ elif num_length_args == 2:
+ good_len = (length_args[0] <= len(s) <= length_args[1])
+ elif num_length_args == 1:
+ good_len = (len(s) == length_args[0])
+ else:
+ good_len = True
+ return bool(good_len and HEX_RE.match(s))
commit 961b595f9050c37f95a2064a78cfb9469594cf4c
Author: Brett Smith <brett at curoverse.com>
Date: Mon Jun 9 11:13:39 2014 -0400
2879: Split arv-put's ArgumentParser into module segments.
This will let us use different pieces as a parent parser for other
tools, which I plan to do for the Docker image uploader.
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index b9135b9..e4e1b6d 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -19,118 +19,122 @@ import arvados.commands._util as arv_cmd
CAUGHT_SIGNALS = [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]
-def parse_arguments(arguments):
- parser = argparse.ArgumentParser(
- description='Copy data from the local filesystem to Keep.')
-
- parser.add_argument('paths', metavar='path', type=str, nargs='*',
- help="""
- Local file or directory. Default: read from standard input.
- """)
-
- parser.add_argument('--max-manifest-depth', type=int, metavar='N',
- default=-1, help="""
- Maximum depth of directory tree to represent in the manifest
- structure. A directory structure deeper than this will be represented
- as a single stream in the manifest. If N=0, the manifest will contain
- a single stream. Default: -1 (unlimited), i.e., exactly one manifest
- stream per filesystem directory that contains files.
- """)
-
- group = parser.add_mutually_exclusive_group()
-
- group.add_argument('--as-stream', action='store_true', dest='stream',
- help="""
- Synonym for --stream.
- """)
-
- group.add_argument('--stream', action='store_true',
- help="""
- Store the file content and display the resulting manifest on
- stdout. Do not write the manifest to Keep or save a Collection object
- in Arvados.
- """)
-
- group.add_argument('--as-manifest', action='store_true', dest='manifest',
- help="""
- Synonym for --manifest.
- """)
-
- group.add_argument('--in-manifest', action='store_true', dest='manifest',
- help="""
- Synonym for --manifest.
- """)
-
- group.add_argument('--manifest', action='store_true',
- help="""
- Store the file data and resulting manifest in Keep, save a Collection
- object in Arvados, and display the manifest locator (Collection uuid)
- on stdout. This is the default behavior.
- """)
-
- group.add_argument('--as-raw', action='store_true', dest='raw',
- help="""
- Synonym for --raw.
- """)
-
- group.add_argument('--raw', action='store_true',
- help="""
- Store the file content and display the data block locators on stdout,
- separated by commas, with a trailing newline. Do not store a
- manifest.
- """)
-
- parser.add_argument('--use-filename', type=str, default=None,
- dest='filename', help="""
- Synonym for --filename.
- """)
-
- parser.add_argument('--filename', type=str, default=None,
- help="""
- Use the given filename in the manifest, instead of the name of the
- local file. This is useful when "-" or "/dev/stdin" is given as an
- input file. It can be used only if there is exactly one path given and
- it is not a directory. Implies --manifest.
- """)
-
- group = parser.add_mutually_exclusive_group()
- group.add_argument('--progress', action='store_true',
- help="""
- Display human-readable progress on stderr (bytes and, if possible,
- percentage of total data size). This is the default behavior when
- stderr is a tty.
- """)
+upload_opts = argparse.ArgumentParser(add_help=False)
+
+upload_opts.add_argument('paths', metavar='path', type=str, nargs='*',
+ help="""
+Local file or directory. Default: read from standard input.
+""")
+
+upload_opts.add_argument('--max-manifest-depth', type=int, metavar='N',
+ default=-1, help="""
+Maximum depth of directory tree to represent in the manifest
+structure. A directory structure deeper than this will be represented
+as a single stream in the manifest. If N=0, the manifest will contain
+a single stream. Default: -1 (unlimited), i.e., exactly one manifest
+stream per filesystem directory that contains files.
+""")
+
+_group = upload_opts.add_mutually_exclusive_group()
+
+_group.add_argument('--as-stream', action='store_true', dest='stream',
+ help="""
+Synonym for --stream.
+""")
+
+_group.add_argument('--stream', action='store_true',
+ help="""
+Store the file content and display the resulting manifest on
+stdout. Do not write the manifest to Keep or save a Collection object
+in Arvados.
+""")
+
+_group.add_argument('--as-manifest', action='store_true', dest='manifest',
+ help="""
+Synonym for --manifest.
+""")
+
+_group.add_argument('--in-manifest', action='store_true', dest='manifest',
+ help="""
+Synonym for --manifest.
+""")
+
+_group.add_argument('--manifest', action='store_true',
+ help="""
+Store the file data and resulting manifest in Keep, save a Collection
+object in Arvados, and display the manifest locator (Collection uuid)
+on stdout. This is the default behavior.
+""")
+
+_group.add_argument('--as-raw', action='store_true', dest='raw',
+ help="""
+Synonym for --raw.
+""")
+
+_group.add_argument('--raw', action='store_true',
+ help="""
+Store the file content and display the data block locators on stdout,
+separated by commas, with a trailing newline. Do not store a
+manifest.
+""")
+
+upload_opts.add_argument('--use-filename', type=str, default=None,
+ dest='filename', help="""
+Synonym for --filename.
+""")
+
+upload_opts.add_argument('--filename', type=str, default=None,
+ help="""
+Use the given filename in the manifest, instead of the name of the
+local file. This is useful when "-" or "/dev/stdin" is given as an
+input file. It can be used only if there is exactly one path given and
+it is not a directory. Implies --manifest.
+""")
+
+run_opts = argparse.ArgumentParser(add_help=False)
+_group = run_opts.add_mutually_exclusive_group()
+_group.add_argument('--progress', action='store_true',
+ help="""
+Display human-readable progress on stderr (bytes and, if possible,
+percentage of total data size). This is the default behavior when
+stderr is a tty.
+""")
+
+_group.add_argument('--no-progress', action='store_true',
+ help="""
+Do not display human-readable progress on stderr, even if stderr is a
+tty.
+""")
+
+_group.add_argument('--batch-progress', action='store_true',
+ help="""
+Display machine-readable progress on stderr (bytes and, if known,
+total data size).
+""")
+
+_group = run_opts.add_mutually_exclusive_group()
+_group.add_argument('--resume', action='store_true', default=True,
+ help="""
+Continue interrupted uploads from cached state (default).
+""")
+_group.add_argument('--no-resume', action='store_false', dest='resume',
+ help="""
+Do not continue interrupted uploads from cached state.
+""")
+
+arg_parser = argparse.ArgumentParser(
+ description='Copy data from the local filesystem to Keep.',
+ parents=[upload_opts, run_opts])
- group.add_argument('--no-progress', action='store_true',
- help="""
- Do not display human-readable progress on stderr, even if stderr is a
- tty.
- """)
-
- group.add_argument('--batch-progress', action='store_true',
- help="""
- Display machine-readable progress on stderr (bytes and, if known,
- total data size).
- """)
-
- group = parser.add_mutually_exclusive_group()
- group.add_argument('--resume', action='store_true', default=True,
- help="""
- Continue interrupted uploads from cached state (default).
- """)
- group.add_argument('--no-resume', action='store_false', dest='resume',
- help="""
- Do not continue interrupted uploads from cached state.
- """)
-
- args = parser.parse_args(arguments)
+def parse_arguments(arguments):
+ args = arg_parser.parse_args(arguments)
if len(args.paths) == 0:
args.paths += ['/dev/stdin']
if len(args.paths) != 1 or os.path.isdir(args.paths[0]):
if args.filename:
- parser.error("""
+ arg_parser.error("""
--filename argument cannot be used when storing a directory or
multiple files.
""")
commit 53119b27478c5cdad09e93ab9c8163bf4e23e602
Author: Brett Smith <brett at curoverse.com>
Date: Mon Jun 9 11:06:20 2014 -0400
2879: Factor out arv-put's cache creation to a utility library.
This function will be useful for the Docker image uploader, and
probably other tools as well.
diff --git a/sdk/python/arvados/commands/_util.py b/sdk/python/arvados/commands/_util.py
new file mode 100644
index 0000000..f7cb80d
--- /dev/null
+++ b/sdk/python/arvados/commands/_util.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+import errno
+import os
+
+def _ignore_error(error):
+ return None
+
+def _raise_error(error):
+ raise error
+
+def make_home_conf_dir(path, mode=None, errors='ignore'):
+ # Make the directory path under the user's home directory, making parent
+ # directories as needed.
+ # If the directory is newly created, and a mode is specified, chmod it
+ # with those permissions.
+ # If there's an error, return None if errors is 'ignore', else raise an
+ # exception.
+ error_handler = _ignore_error if (errors == 'ignore') else _raise_error
+ tilde_path = os.path.join('~', path)
+ abs_path = os.path.expanduser(tilde_path)
+ if abs_path == tilde_path:
+ return error_handler(ValueError("no home directory available"))
+ try:
+ os.makedirs(abs_path)
+ except OSError as error:
+ if error.errno != errno.EEXIST:
+ return error_handler(error)
+ else:
+ if mode is not None:
+ os.chmod(abs_path, mode)
+ return abs_path
diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py
index 01bae2f..b9135b9 100644
--- a/sdk/python/arvados/commands/put.py
+++ b/sdk/python/arvados/commands/put.py
@@ -15,6 +15,8 @@ import signal
import sys
import tempfile
+import arvados.commands._util as arv_cmd
+
CAUGHT_SIGNALS = [signal.SIGINT, signal.SIGQUIT, signal.SIGTERM]
def parse_arguments(arguments):
@@ -150,17 +152,11 @@ class ResumeCacheConflict(Exception):
class ResumeCache(object):
- CACHE_DIR = os.path.expanduser('~/.cache/arvados/arv-put')
+ CACHE_DIR = '.cache/arvados/arv-put'
@classmethod
def setup_user_cache(cls):
- try:
- os.makedirs(cls.CACHE_DIR)
- except OSError as error:
- if error.errno != errno.EEXIST:
- raise
- else:
- os.chmod(cls.CACHE_DIR, 0o700)
+ return arv_cmd.make_home_conf_dir(cls.CACHE_DIR, 0o700)
def __init__(self, file_spec):
self.cache_file = open(file_spec, 'a+')
@@ -339,16 +335,18 @@ def main(arguments=None):
reporter = None
bytes_expected = expected_bytes_for(args.paths)
+ resume_cache = None
try:
- ResumeCache.setup_user_cache()
- resume_cache = ResumeCache(ResumeCache.make_path(args))
+ if ResumeCache.setup_user_cache() is not None:
+ resume_cache = ResumeCache(ResumeCache.make_path(args))
except (IOError, OSError):
- # Couldn't open cache directory/file. Continue without it.
- resume_cache = None
- writer = ArvPutCollectionWriter(resume_cache, reporter, bytes_expected)
+ pass # Couldn't open cache directory/file. Continue without it.
except ResumeCacheConflict:
print "arv-put: Another process is already uploading this data."
sys.exit(1)
+
+ if resume_cache is None:
+ writer = ArvPutCollectionWriter(resume_cache, reporter, bytes_expected)
else:
if not args.resume:
resume_cache.restart()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list