[ARVADOS] created: 64046fa6553461bed50d36f585ea1762e4ff6fdd
git at public.curoverse.com
git at public.curoverse.com
Sun Mar 8 17:12:52 EDT 2015
at 64046fa6553461bed50d36f585ea1762e4ff6fdd (commit)
commit 64046fa6553461bed50d36f585ea1762e4ff6fdd
Author: Brett Smith <brett at curoverse.com>
Date: Sun Mar 8 17:12:40 2015 -0400
5104: Add Collection class to the Ruby SDK.
This provides an interface to do filesystem-like operations on a tree
expressed in a manifest. It currently supports copy, rename, and
remove operations, and will give us common ground for further work in
Ruby components.
diff --git a/sdk/ruby/lib/arvados/collection.rb b/sdk/ruby/lib/arvados/collection.rb
new file mode 100644
index 0000000..94f5138
--- /dev/null
+++ b/sdk/ruby/lib/arvados/collection.rb
@@ -0,0 +1,403 @@
+require "arvados/keep"
+
+module Arv
+ class Collection
+ def initialize(manifest_text="")
+ @tree = CollectionStream.new(".")
+ @manifest_text = ""
+ import_manifest!(manifest_text)
+ end
+
+ def manifest_text
+ @manifest_text ||= @tree.manifest_text
+ end
+
+ def import_manifest!(manifest_text)
+ manifest = Keep::Manifest.new(manifest_text)
+ manifest.each_line do |stream_root, locators, file_specs|
+ if stream_root.empty? or locators.empty? or file_specs.empty?
+ raise ArgumentError.new("manifest text includes malformed line")
+ end
+ file_specs.map { |s| manifest.split_file_token(s) }.
+ each do |file_start, file_len, file_path|
+ @tree.file_at(normalize_path(stream_root, file_path)).
+ add_range(locators, file_start, file_len)
+ end
+ end
+ if @manifest_text == ""
+ @manifest_text = manifest_text
+ self
+ else
+ modified!
+ end
+ end
+
+ def normalize!
+ # We generate normalized manifests, so all we have to do is force
+ # regeneration.
+ modified!
+ end
+
+ def copy!(source, target, source_collection=nil)
+ copy(:merge, source, target, source_collection)
+ end
+
+ def rename!(source, target)
+ copy(:add_copy, source, target) { remove!(source, recursive: true) }
+ end
+
+ def remove!(path, opts={})
+ stream, name = find(path)
+ if name.nil?
+ return self if @tree.leaf?
+ @tree = CollectionStream.new(".")
+ else
+ stream.delete(name, opts)
+ end
+ modified!
+ end
+
+ protected
+
+ def find(*parts)
+ normpath = normalize_path(*parts)
+ if normpath.empty?
+ [@tree, nil]
+ else
+ @tree.find(normpath)
+ end
+ end
+
+ private
+
+ def copy(copy_method, source, target, source_collection=nil)
+ # Find the item at path `source` in `source_collection`, find the
+ # destination stream at path `target`, and use `copy_method` to copy
+ # the found object there. If a block is passed in, it will be called
+ # right before we do the actual copy, after we confirm that everything
+ # is found and can be copied.
+ source_collection = self if source_collection.nil?
+ src_stream, src_tail = source_collection.find(source)
+ dst_stream, dst_tail = find(target)
+ if (source_collection.equal?(self) and
+ (src_stream.path == dst_stream.path) and (src_tail == dst_tail))
+ return self
+ elsif src_tail.nil?
+ src_item = src_stream
+ src_tail = src_stream.name
+ else
+ src_item = src_stream[src_tail]
+ end
+ dst_tail ||= src_tail
+ check_method = "check_can_#{copy_method}".to_sym
+ begin
+ # Find out if `target` refers to a stream we should copy into.
+ tail_stream = dst_stream[dst_tail]
+ tail_stream.send(check_method, src_item, src_tail)
+ rescue Errno::ENOENT, Errno::ENOTDIR
+ # It does not. Check that we can copy `source` to the full
+ # path specified by `target`.
+ dst_stream.send(check_method, src_item, dst_tail)
+ target_name = dst_tail
+ else
+ # Yes, `target` is a stream. Copy the item at `source` into it with
+ # the same name.
+ dst_stream = tail_stream
+ target_name = src_tail
+ end
+ # At this point, we know the operation will work. Call any block as
+ # a pre-copy hook.
+ if block_given?
+ yield
+ # Re-find the destination stream, in case the block removed
+ # the original (that's how rename is implemented).
+ dst_path = normalize_path(dst_stream.path)
+ if dst_path.empty?
+ dst_stream = @tree
+ else
+ dst_stream = @tree.stream_at(dst_path)
+ end
+ end
+ dst_stream.send(copy_method, src_item, target_name)
+ modified!
+ end
+
+ def modified!
+ @manifest_text = nil
+ self
+ end
+
+ def normalize_path(*parts)
+ path = File.join(*parts)
+ raise ArgumentError.new("empty path") if path.empty?
+ path.sub(/^\.(\/|$)/, "")
+ end
+
+ class CollectionItem
+ attr_reader :path, :name
+
+ def initialize(path)
+ @path = path
+ @name = File.basename(path)
+ end
+ end
+
+ LocatorRange = Struct.new(:locators, :start_pos, :length)
+
+ class CollectionFile < CollectionItem
+ def initialize(path)
+ super
+ @ranges = []
+ end
+
+ def self.human_name
+ "file"
+ end
+
+ def leaf?
+ true
+ end
+
+ def add_range(locators, start_pos, length)
+ # Given an array of locators, and this file's start position and
+ # length within them, store a LocatorRange with information about
+ # the locators actually used.
+ loc_sizes = locators.map { |s| Keep::Locator.parse(s).size.to_i }
+ start_index, start_pos = loc_size_index(loc_sizes, start_pos, 0, :>=)
+ end_index, _ = loc_size_index(loc_sizes, length, start_index, :>)
+ @ranges << LocatorRange.
+ new(locators[start_index..end_index], start_pos, length)
+ end
+
+ def each_range(&block)
+ @ranges.each(&block)
+ end
+
+ def check_can_add_copy(src_item, name)
+ raise Errno::ENOTDIR.new(path)
+ end
+
+ alias_method :check_can_merge, :check_can_add_copy
+
+ def copy_named(copy_path)
+ copy = self.class.new(copy_path)
+ each_range { |range| copy.add_range(*range) }
+ copy
+ end
+
+ private
+
+ def loc_size_index(loc_sizes, length, index, comp_op)
+ # Pass in an array of locator size hints (integers). Starting from
+ # `index`, step through the size array until they provide a number
+ # of bytes that is `comp_op` (:>= or :>) to `length`. Return the
+ # index of the end locator and the amount of data to read from it.
+ while length.send(comp_op, loc_sizes[index])
+ index += 1
+ length -= loc_sizes[index]
+ end
+ [index, length]
+ end
+ end
+
+ class CollectionStream < CollectionItem
+ def initialize(path)
+ super
+ @items = {}
+ end
+
+ def self.human_name
+ "stream"
+ end
+
+ def leaf?
+ items.empty?
+ end
+
+ def [](key)
+ items[key] or
+ raise Errno::ENOENT.new("%p not found in %p" % [key, path])
+ end
+
+ def delete(name, opts={})
+ item = self[name]
+ if item.leaf? or opts[:recursive]
+ items.delete(name)
+ else
+ raise Errno::ENOTEMPTY.new(path)
+ end
+ end
+
+ def find(find_path)
+ # Given a POSIX-style path, return the CollectionStream that
+ # contains the object at that path, and the name of the object
+ # inside it.
+ components = find_path.split("/")
+ tail = components.pop
+ [components.reduce(self, :[]), tail]
+ end
+
+ def stream_at(find_path)
+ key, rest = find_path.split("/", 2)
+ next_stream = get_or_new(key, CollectionStream)
+ if rest.nil?
+ next_stream
+ else
+ next_stream.stream_at(rest)
+ end
+ end
+
+ def file_at(find_path)
+ stream_path, _, file_name = find_path.rpartition("/")
+ if stream_path.empty?
+ get_or_new(file_name, CollectionFile)
+ else
+ stream_at(stream_path).file_at(file_name)
+ end
+ end
+
+ def manifest_text
+ # Return a string with the normalized manifest text for this stream,
+ # including all substreams.
+ file_keys, stream_keys = items.keys.sort.partition do |key|
+ items[key].is_a?(CollectionFile)
+ end
+ my_line = StreamManifest.new(path)
+ file_keys.each do |file_name|
+ my_line.add_file(items[file_name])
+ end
+ sub_lines = stream_keys.map do |sub_name|
+ items[sub_name].manifest_text
+ end
+ my_line.to_s + sub_lines.join("")
+ end
+
+ def check_can_add_copy(src_item, key)
+ if existing = check_can_merge(src_item, key) and not existing.leaf?
+ raise Errno::ENOTEMPTY.new(existing.path)
+ end
+ end
+
+ def check_can_merge(src_item, key)
+ if existing = items[key] and (existing.class != src_item.class)
+ raise Errno::ENOTDIR.new(existing.path)
+ end
+ existing
+ end
+
+ def add_copy(src_item, key)
+ items[key] = src_item.copy_named("#{path}/#{key}")
+ end
+
+ def merge(src_item, key)
+ # Do a recursive copy of the collection item `src_item` to destination
+ # `key`. If a simple copy is safe, do that; otherwise, recursively
+ # merge the contents of the stream `src_item` into the stream at
+ # `key`.
+ begin
+ check_can_add_copy(src_item, key)
+ add_copy(src_item, key)
+ rescue Errno::ENOTEMPTY
+ dest = self[key]
+ error = nil
+ # Copy as much as possible, then raise any error encountered.
+ src_item.items.each_pair do |sub_key, sub_item|
+ begin
+ dest.merge(sub_item, sub_key)
+ rescue Errno::ENOTDIR => error
+ end
+ end
+ raise error unless error.nil?
+ end
+ end
+
+ def copy_named(copy_path)
+ copy = self.class.new(copy_path)
+ items.each_pair do |key, item|
+ copy.add_copy(item, key)
+ end
+ copy
+ end
+
+ protected
+
+ attr_reader :items
+
+ private
+
+ def get_or_new(key, klass)
+ # Return the collection item at `key` and ensure that it's a `klass`.
+ # If `key` does not exist, create a new `klass` there.
+ # If the value for `key` is not a `klass`, raise an ArgumentError.
+ item = items[key]
+ if item.nil?
+ items[key] = klass.new("#{path}/#{key}")
+ elsif not item.is_a?(klass)
+ raise ArgumentError.
+ new("in stream %p, %p is a %s, not a %s" %
+ [path, key, items[key].class.human_name, klass.human_name])
+ else
+ item
+ end
+ end
+ end
+
+ class StreamManifest
+ # Build a manifest text for a single stream, without substreams.
+
+ def initialize(name)
+ @name = name
+ @locators = []
+ @loc_sizes = []
+ @file_specs = []
+ end
+
+ def add_file(coll_file)
+ coll_file.each_range do |range|
+ add(coll_file.name, *range)
+ end
+ end
+
+ def to_s
+ if @file_specs.empty?
+ ""
+ else
+ "%s %s %s\n" % [escape_name(@name), @locators.join(" "),
+ @file_specs.join(" ")]
+ end
+ end
+
+ private
+
+ def add(file_name, loc_a, file_start, file_len)
+ # Ensure that the locators in loc_a appear in this locator in sequence,
+ # adding as few as possible. Save a new file spec based on those
+ # locators' position.
+ loc_size = @locators.size
+ add_size = loc_a.size
+ loc_ii = 0
+ add_ii = 0
+ while (loc_ii < loc_size) and (add_ii < add_size)
+ if @locators[loc_ii] == loc_a[add_ii]
+ add_ii += 1
+ else
+ add_ii = 0
+ end
+ loc_ii += 1
+ end
+ loc_ii -= add_ii
+ to_add = loc_a[add_ii, add_size] || []
+ @locators += to_add
+ @loc_sizes += to_add.map { |s| Keep::Locator.parse(s).size.to_i }
+ start = @loc_sizes[0, loc_ii].reduce(0, &:+) + file_start
+ @file_specs << "#{start}:#{file_len}:#{escape_name(file_name)}"
+ end
+
+ def escape_name(name)
+ name.gsub(/\\/, "\\\\\\\\").gsub(/\s/) do |s|
+ s.each_byte.map { |c| "\\%03o" % c }.join("")
+ end
+ end
+ end
+ end
+end
diff --git a/sdk/ruby/test/sdk_fixtures.rb b/sdk/ruby/test/sdk_fixtures.rb
index 8af81cd..52d7377 100644
--- a/sdk/ruby/test/sdk_fixtures.rb
+++ b/sdk/ruby/test/sdk_fixtures.rb
@@ -7,6 +7,10 @@ module SDKFixtures
def random_block(size=nil)
sprintf("%032x+%d", rand(16 ** 32), size || rand(64 * 1024 * 1024))
end
+
+ def random_blocks(count, size=nil)
+ (0...count).map { |_| random_block(size) }
+ end
end
extend StaticMethods
@@ -50,4 +54,12 @@ module SDKFixtures
NONNORMALIZED_MANIFEST =
["./dir2 #{random_block} 0:0:z 0:0:y 0:0:x",
"./dir1 #{random_block} 0:0:p 0:0:o 0:0:n\n"].join("\n")
+
+ ### Non-tree manifests
+ # These manifests follow the spec, but they express a structure that can't
+ # can't be represented by a POSIX filesystem tree. For example, there's a
+ # name conflict between a stream and a filename.
+ NAME_CONFLICT_MANIFEST =
+ [". #{random_block(9)} 0:9:conflict",
+ "./conflict #{random_block} 0:0:name\n"].join("\n")
end
diff --git a/sdk/ruby/test/test_collection.rb b/sdk/ruby/test/test_collection.rb
new file mode 100644
index 0000000..5fba278
--- /dev/null
+++ b/sdk/ruby/test/test_collection.rb
@@ -0,0 +1,508 @@
+require "arvados/collection"
+require "minitest/autorun"
+require "sdk_fixtures"
+
+class CollectionTest < Minitest::Test
+ include SDKFixtures
+
+ TWO_BY_TWO_BLOCKS = SDKFixtures.random_blocks(2, 9)
+ TWO_BY_TWO_MANIFEST_A =
+ [". #{TWO_BY_TWO_BLOCKS.first} 0:5:f1 5:4:f2\n",
+ "./s1 #{TWO_BY_TWO_BLOCKS.last} 0:5:f1 5:4:f3\n"]
+ TWO_BY_TWO_MANIFEST_S = TWO_BY_TWO_MANIFEST_A.join("")
+
+ ### .new
+
+ def test_empty_construction
+ coll = Arv::Collection.new
+ assert_equal("", coll.manifest_text)
+ end
+
+ def test_successful_construction
+ [:SIMPLEST_MANIFEST, :MULTIBLOCK_FILE_MANIFEST, :MULTILEVEL_MANIFEST].
+ each do |manifest_name|
+ manifest_text = SDKFixtures.const_get(manifest_name)
+ coll = Arv::Collection.new(manifest_text)
+ assert_equal(manifest_text, coll.manifest_text,
+ "did not get same manifest back out from #{manifest_name}")
+ end
+ end
+
+ def test_non_manifest_construction_error
+ ["word", ". abc def", ". #{random_block} 0:", ". / !"].each do |m_text|
+ assert_raises(ArgumentError,
+ "built collection from manifest #{m_text.inspect}") do
+ Arv::Collection.new(m_text)
+ end
+ end
+ end
+
+ def test_file_directory_conflict_construction_error
+ assert_raises(ArgumentError) do
+ Arv::Collection.new(NAME_CONFLICT_MANIFEST)
+ end
+ end
+
+ def test_no_implicit_normalization
+ coll = Arv::Collection.new(NONNORMALIZED_MANIFEST)
+ assert_equal(NONNORMALIZED_MANIFEST, coll.manifest_text)
+ end
+
+ def test_no_implicit_normalization_from_first_import
+ coll = Arv::Collection.new
+ coll.import_manifest!(NONNORMALIZED_MANIFEST)
+ assert_equal(NONNORMALIZED_MANIFEST, coll.manifest_text)
+ end
+
+ ### .import_manifest!
+
+ def test_non_posix_path_handling
+ block = random_block(9)
+ coll = Arv::Collection.new("./.. #{block} 0:5:.\n")
+ coll.import_manifest!("./.. #{block} 5:4:..\n")
+ assert_equal("./.. #{block} 0:5:. 5:4:..\n", coll.manifest_text)
+ end
+
+ def test_escaping_through_normalization
+ coll = Arv::Collection.new(MANY_ESCAPES_MANIFEST)
+ coll.import_manifest!(MANY_ESCAPES_MANIFEST)
+ # The result should simply duplicate the file spec.
+ # The source file spec has an unescaped backslash in it.
+ # It's OK for the Collection class to properly escape that.
+ expect_text = MANY_ESCAPES_MANIFEST.sub(/ \d+:\d+:\S+/) do |file_spec|
+ file_spec.gsub(/([^\\])(\\[^\\\d])/, '\1\\\\\2') * 2
+ end
+ assert_equal(expect_text, coll.manifest_text)
+ end
+
+ def test_concatenation_from_multiple_imports(file_name="file.txt",
+ out_name=nil)
+ out_name ||= file_name
+ blocks = random_blocks(2, 9)
+ coll = Arv::Collection.new
+ blocks.each do |block|
+ coll.import_manifest!(". #{block} 1:8:#{file_name}\n")
+ end
+ assert_equal(". #{blocks.join(' ')} 1:8:#{out_name} 10:8:#{out_name}\n",
+ coll.manifest_text)
+ end
+
+ def test_concatenation_from_multiple_escaped_imports
+ test_concatenation_from_multiple_imports('a\040\141.txt', 'a\040a.txt')
+ end
+
+ def test_concatenation_with_locator_overlap(over_index=0)
+ blocks = random_blocks(4, 2)
+ coll = Arv::Collection.new(". #{blocks.join(' ')} 0:8:file\n")
+ coll.import_manifest!(". #{blocks[over_index, 2].join(' ')} 0:4:file\n")
+ assert_equal(". #{blocks.join(' ')} 0:8:file #{over_index * 2}:4:file\n",
+ coll.manifest_text)
+ end
+
+ def test_concatenation_with_middle_locator_overlap
+ test_concatenation_with_locator_overlap(1)
+ end
+
+ def test_concatenation_with_end_locator_overlap
+ test_concatenation_with_locator_overlap(2)
+ end
+
+ def test_concatenation_with_partial_locator_overlap
+ blocks = random_blocks(3, 3)
+ coll = Arv::Collection.new(". #{blocks[0, 2].join(' ')} 0:6:overlap\n")
+ coll.import_manifest!(". #{blocks[1, 2].join(' ')} 0:6:overlap\n")
+ assert_equal(". #{blocks.join(' ')} 0:6:overlap 3:6:overlap\n",
+ coll.manifest_text)
+ end
+
+ ### .normalize!
+
+ def test_normalize
+ block = random_block
+ coll = Arv::Collection.new(". #{block} 0:0:f2 0:0:f1\n")
+ coll.normalize!
+ assert_equal(". #{block} 0:0:f1 0:0:f2\n", coll.manifest_text)
+ end
+
+ ### .copy!
+
+ def test_simple_file_copy
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ coll.copy!("./simple.txt", "./new")
+ assert_equal(SIMPLEST_MANIFEST.sub(" 0:9:", " 0:9:new 0:9:"),
+ coll.manifest_text)
+ end
+
+ def test_copy_file_into_other_stream(target="./s1/f2", basename="f2")
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ coll.copy!("./f2", target)
+ expected = "%s./s1 %s 0:5:f1 14:4:%s 5:4:f3\n" %
+ [TWO_BY_TWO_MANIFEST_A.first,
+ TWO_BY_TWO_BLOCKS.reverse.join(" "), basename]
+ assert_equal(expected, coll.manifest_text)
+ end
+
+ def test_implicit_copy_file_into_other_stream
+ test_copy_file_into_other_stream("./s1")
+ end
+
+ def test_copy_file_into_other_stream_with_new_name
+ test_copy_file_into_other_stream("./s1/f2a", "f2a")
+ end
+
+ def test_copy_file_over_in_other_stream(target="./s1/f1")
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ coll.copy!("./f1", target)
+ expected = "%s./s1 %s 0:5:f1 14:4:f3\n" %
+ [TWO_BY_TWO_MANIFEST_A.first, TWO_BY_TWO_BLOCKS.join(" ")]
+ assert_equal(expected, coll.manifest_text)
+ end
+
+ def test_implicit_copy_file_over_in_other_stream
+ test_copy_file_over_in_other_stream("./s1")
+ end
+
+ def test_simple_stream_copy
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ coll.copy!("./s1", "./sNew")
+ new_line = TWO_BY_TWO_MANIFEST_A.last.sub("./s1 ", "./sNew ")
+ assert_equal(TWO_BY_TWO_MANIFEST_S + new_line, coll.manifest_text)
+ end
+
+ def test_copy_stream_into_other_stream(target="./dir2/subdir",
+ basename="subdir")
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ coll.copy!("./dir1/subdir", target)
+ new_line = MULTILEVEL_MANIFEST.lines[4].sub("./dir1/subdir ",
+ "./dir2/#{basename} ")
+ assert_equal(MULTILEVEL_MANIFEST + new_line, coll.manifest_text)
+ end
+
+ def test_implicit_copy_stream_into_other_stream
+ test_copy_stream_into_other_stream("./dir2")
+ end
+
+ def test_copy_stream_into_other_stream_with_new_name
+ test_copy_stream_into_other_stream("./dir2/newsub", "newsub")
+ end
+
+ def test_copy_stream_over_empty_stream
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ (1..3).each do |file_num|
+ coll.remove!("./dir0/subdir/file#{file_num}")
+ end
+ coll.copy!("./dir1/subdir", "./dir0")
+ expected = MULTILEVEL_MANIFEST.lines
+ expected[2] = expected[4].sub("./dir1/", "./dir0/")
+ assert_equal(expected.join(""), coll.manifest_text)
+ end
+
+ def test_copy_stream_over_file_raises_ENOTDIR
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ assert_raises(Errno::ENOTDIR) do
+ coll.copy!("./s1", "./f2")
+ end
+ end
+
+ def test_copy_stream_over_nonempty_stream_merges_and_overwrites
+ blocks = random_blocks(3, 9)
+ manifest_a =
+ ["./subdir #{blocks[0]} 0:1:s1 1:2:zero\n",
+ "./zdir #{blocks[1]} 0:9:zfile\n",
+ "./zdir/subdir #{blocks[2]} 0:1:s2 1:2:zero\n"]
+ coll = Arv::Collection.new(manifest_a.join(""))
+ coll.copy!("./subdir", "./zdir")
+ manifest_a[2] = "./zdir/subdir %s %s 0:1:s1 9:1:s2 1:2:zero\n" %
+ [blocks[0], blocks[2]]
+ assert_equal(manifest_a.join(""), coll.manifest_text)
+ end
+
+ def test_copy_stream_into_substream(source="./dir1",
+ target="./dir1/subdir/dir1")
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ coll.copy!(source, target)
+ expected = MULTILEVEL_MANIFEST.lines.flat_map do |line|
+ [line, line.gsub(/^#{Regexp.escape(source)}([\/ ])/, "#{target}\\1")].uniq
+ end
+ assert_equal(expected.sort.join(""), coll.manifest_text)
+ end
+
+ def test_copy_root
+ test_copy_stream_into_substream(".", "./root")
+ end
+
+ def test_adding_to_root_after_copy
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ coll.copy!(".", "./root")
+ coll.import_manifest!(COLON_FILENAME_MANIFEST)
+ got_lines = coll.manifest_text.lines
+ assert_equal(2, got_lines.size)
+ assert_match(/^\. \S{33,} \S{33,} 0:9:file:test\.txt 9:9:simple\.txt\n/,
+ got_lines.first)
+ assert_equal(SIMPLEST_MANIFEST.sub(". ", "./root "), got_lines.last)
+ end
+
+ def test_copy_chaining
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ coll.copy!("./simple.txt", "./a").copy!("./a", "./b")
+ assert_equal(SIMPLEST_MANIFEST.sub(" 0:9:", " 0:9:a 0:9:b 0:9:"),
+ coll.manifest_text)
+ end
+
+ def prep_two_collections_for_copy(src_stream, dst_stream)
+ blocks = random_blocks(2, 8)
+ src_text = "#{src_stream} #{blocks.first} 0:8:f1\n"
+ dst_text = "#{dst_stream} #{blocks.last} 0:8:f2\n"
+ return [blocks, src_text, dst_text,
+ Arv::Collection.new(src_text.dup),
+ Arv::Collection.new(dst_text.dup)]
+ end
+
+ def test_copy_file_from_other_collection(src_stream=".", dst_stream="./s1")
+ blocks, src_text, dst_text, src_coll, dst_coll =
+ prep_two_collections_for_copy(src_stream, dst_stream)
+ dst_coll.copy!("#{src_stream}/f1", dst_stream, src_coll)
+ assert_equal("#{dst_stream} #{blocks.join(' ')} 0:8:f1 8:8:f2\n",
+ dst_coll.manifest_text)
+ assert_equal(src_text, src_coll.manifest_text)
+ end
+
+ def test_copy_file_from_other_collection_to_root
+ test_copy_file_from_other_collection("./s1", ".")
+ end
+
+ def test_copy_stream_from_other_collection
+ blocks, src_text, dst_text, src_coll, dst_coll =
+ prep_two_collections_for_copy("./s2", "./s1")
+ dst_coll.copy!("./s2", "./s1", src_coll)
+ assert_equal(dst_text + src_text.sub("./s2 ", "./s1/s2 "),
+ dst_coll.manifest_text)
+ assert_equal(src_text, src_coll.manifest_text)
+ end
+
+ def test_copy_stream_from_other_collection_to_root
+ blocks, src_text, dst_text, src_coll, dst_coll =
+ prep_two_collections_for_copy("./s1", ".")
+ dst_coll.copy!("./s1", ".", src_coll)
+ assert_equal(dst_text + src_text, dst_coll.manifest_text)
+ assert_equal(src_text, src_coll.manifest_text)
+ end
+
+ def test_copy_empty_source_path_raises_ArgumentError(src="", dst="./s1")
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ assert_raises(ArgumentError) do
+ coll.copy!(src, dst)
+ end
+ end
+
+ def test_copy_empty_destination_path_raises_ArgumentError
+ test_copy_empty_source_path_raises_ArgumentError(".", "")
+ end
+
+ ### .rename!
+
+ def test_simple_file_rename
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ coll.rename!("./simple.txt", "./new")
+ assert_equal(SIMPLEST_MANIFEST.sub(":simple.txt", ":new"),
+ coll.manifest_text)
+ end
+
+ def test_rename_file_into_other_stream(target="./s1/f2", basename="f2")
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ coll.rename!("./f2", target)
+ expected = ". %s 0:5:f1\n./s1 %s 0:5:f1 14:4:%s 5:4:f3\n" %
+ [TWO_BY_TWO_BLOCKS.first,
+ TWO_BY_TWO_BLOCKS.reverse.join(" "), basename]
+ assert_equal(expected, coll.manifest_text)
+ end
+
+ def test_implicit_rename_file_into_other_stream
+ test_rename_file_into_other_stream("./s1")
+ end
+
+ def test_rename_file_into_other_stream_with_new_name
+ test_rename_file_into_other_stream("./s1/f2a", "f2a")
+ end
+
+ def test_rename_file_over_in_other_stream(target="./s1/f1")
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ coll.rename!("./f1", target)
+ expected = ". %s 5:4:f2\n./s1 %s 0:5:f1 14:4:f3\n" %
+ [TWO_BY_TWO_BLOCKS.first, TWO_BY_TWO_BLOCKS.join(" ")]
+ assert_equal(expected, coll.manifest_text)
+ end
+
+ def test_implicit_rename_file_over_in_other_stream
+ test_rename_file_over_in_other_stream("./s1")
+ end
+
+ def test_simple_stream_rename
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ coll.rename!("./s1", "./newS")
+ assert_equal(TWO_BY_TWO_MANIFEST_S.sub("\n./s1 ", "\n./newS "),
+ coll.manifest_text)
+ end
+
+ def test_rename_stream_into_other_stream(target="./dir2/subdir",
+ basename="subdir")
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ coll.rename!("./dir1/subdir", target)
+ expected = MULTILEVEL_MANIFEST.lines
+ replaced_line = expected.delete_at(4)
+ expected << replaced_line.sub("./dir1/subdir ", "./dir2/#{basename} ")
+ assert_equal(expected.join(""), coll.manifest_text)
+ end
+
+ def test_implicit_rename_stream_into_other_stream
+ test_rename_stream_into_other_stream("./dir2")
+ end
+
+ def test_rename_stream_into_other_stream_with_new_name
+ test_rename_stream_into_other_stream("./dir2/newsub", "newsub")
+ end
+
+ def test_rename_stream_over_empty_stream
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ (1..3).each do |file_num|
+ coll.remove!("./dir0/subdir/file#{file_num}")
+ end
+ coll.rename!("./dir1/subdir", "./dir0")
+ expected = MULTILEVEL_MANIFEST.lines
+ expected[2] = expected.delete_at(4).sub("./dir1/", "./dir0/")
+ assert_equal(expected.sort.join(""), coll.manifest_text)
+ end
+
+ def test_rename_stream_over_file_raises_ENOTDIR
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ assert_raises(Errno::ENOTDIR) do
+ coll.rename!("./s1", "./f2")
+ end
+ end
+
+ def test_rename_stream_over_nonempty_stream_raises_ENOTEMPTY
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ assert_raises(Errno::ENOTEMPTY) do
+ coll.rename!("./dir1/subdir", "./dir0")
+ end
+ end
+
+ def test_rename_stream_into_substream(source="./dir1",
+ target="./dir1/subdir/dir1")
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ coll.rename!(source, target)
+ assert_equal(MULTILEVEL_MANIFEST.gsub(/^#{Regexp.escape(source)}([\/ ])/m,
+ "#{target}\\1"),
+ coll.manifest_text)
+ end
+
+ def test_rename_root
+ test_rename_stream_into_substream(".", "./root")
+ end
+
+ def test_adding_to_root_after_rename
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ coll.rename!(".", "./root")
+ coll.import_manifest!(SIMPLEST_MANIFEST)
+ assert_equal(SIMPLEST_MANIFEST + SIMPLEST_MANIFEST.sub(". ", "./root "),
+ coll.manifest_text)
+ end
+
+ def test_rename_chaining
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ coll.rename!("./simple.txt", "./x").rename!("./x", "./simple.txt")
+ assert_equal(SIMPLEST_MANIFEST, coll.manifest_text)
+ end
+
+ ### .remove!
+
+ def test_simple_remove
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S.dup)
+ coll.remove!("./f2")
+ assert_equal(TWO_BY_TWO_MANIFEST_S.sub(" 5:4:f2", ""), coll.manifest_text)
+ end
+
+ def empty_stream_and_assert(expect_index=0)
+ coll = Arv::Collection.new(TWO_BY_TWO_MANIFEST_S)
+ yield coll
+ assert_equal(TWO_BY_TWO_MANIFEST_A[expect_index], coll.manifest_text)
+ end
+
+ def test_remove_all_files_in_substream
+ empty_stream_and_assert do |coll|
+ coll.remove!("./s1/f1")
+ coll.remove!("./s1/f3")
+ end
+ end
+
+ def test_remove_all_files_in_root_stream
+ empty_stream_and_assert(1) do |coll|
+ coll.remove!("./f1")
+ coll.remove!("./f2")
+ end
+ end
+
+ def test_remove_empty_stream
+ empty_stream_and_assert do |coll|
+ coll.remove!("./s1/f1")
+ coll.remove!("./s1/f3")
+ coll.remove!("./s1")
+ end
+ end
+
+ def test_recursive_remove
+ empty_stream_and_assert do |coll|
+ coll.remove!("./s1", recursive: true)
+ end
+ end
+
+ def test_recursive_remove_on_files
+ empty_stream_and_assert do |coll|
+ coll.remove!("./s1/f1", recursive: true)
+ coll.remove!("./s1/f3", recursive: true)
+ end
+ end
+
+ def test_chaining_removes
+ empty_stream_and_assert do |coll|
+ coll.remove!("./s1/f1").remove!("./s1/f3")
+ end
+ end
+
+ def test_remove_last_file
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ coll.remove!("./simple.txt")
+ assert_equal("", coll.manifest_text)
+ end
+
+ def test_remove_root_stream
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ coll.remove!(".", recursive: true)
+ assert_equal("", coll.manifest_text)
+ end
+
+ def test_remove_nonexistent_file_raises_ENOENT(path="./NoSuchFile")
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ assert_raises(Errno::ENOENT) do
+ coll.remove!(path)
+ end
+ end
+
+ def test_remove_from_nonexistent_stream_raises_ENOENT
+ test_remove_nonexistent_file_raises_ENOENT("./NoSuchStream/simple.txt")
+ end
+
+ def test_remove_nonempty_stream_raises_ENOTEMPTY
+ coll = Arv::Collection.new(MULTILEVEL_MANIFEST)
+ assert_raises(Errno::ENOTEMPTY) do
+ coll.remove!("./dir1/subdir")
+ end
+ end
+
+ def test_remove_empty_string_raises_ArgumentError
+ coll = Arv::Collection.new(SIMPLEST_MANIFEST)
+ assert_raises(ArgumentError) do
+ coll.remove!("")
+ end
+ end
+end
commit a9fdae7d5d78f2d7b89d338496f931288b3632ca
Author: Brett Smith <brett at curoverse.com>
Date: Thu Mar 5 12:44:02 2015 -0500
5104: Refactor Ruby SDK test infrastructure into sharable module.
diff --git a/sdk/ruby/test/sdk_fixtures.rb b/sdk/ruby/test/sdk_fixtures.rb
new file mode 100644
index 0000000..8af81cd
--- /dev/null
+++ b/sdk/ruby/test/sdk_fixtures.rb
@@ -0,0 +1,53 @@
+require "yaml"
+
+module SDKFixtures
+ module StaticMethods
+ # SDKFixtures will use these as class methods, and install them as
+ # instance methods on the test classes.
+ def random_block(size=nil)
+ sprintf("%032x+%d", rand(16 ** 32), size || rand(64 * 1024 * 1024))
+ end
+ end
+
+ extend StaticMethods
+
+ def self.included(base)
+ base.include(StaticMethods)
+ end
+
+ @@fixtures = {}
+ def fixtures name
+ @@fixtures[name] ||=
+ begin
+ path = File.
+ expand_path("../../../../services/api/test/fixtures/#{name}.yml",
+ __FILE__)
+ file = IO.read(path)
+ trim_index = file.index('# Test Helper trims the rest of the file')
+ file = file[0, trim_index] if trim_index
+ YAML.load(file)
+ end
+ end
+
+ ### Valid manifests
+ SIMPLEST_MANIFEST = ". #{random_block(9)} 0:9:simple.txt\n"
+ MULTIBLOCK_FILE_MANIFEST =
+ [". #{random_block(8)} 0:4:repfile 4:4:uniqfile",
+ "./s1 #{random_block(6)} 0:3:repfile 3:3:uniqfile",
+ ". #{random_block(8)} 0:7:uniqfile2 7:1:repfile\n"].join("\n")
+ MULTILEVEL_MANIFEST =
+ [". #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
+ "./dir0 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
+ "./dir0/subdir #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
+ "./dir1 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
+ "./dir1/subdir #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
+ "./dir2 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n"].join("")
+ COLON_FILENAME_MANIFEST = ". #{random_block(9)} 0:9:file:test.txt\n"
+ # Filename is `a a.txt`.
+ ESCAPED_FILENAME_MANIFEST = ". #{random_block(9)} 0:9:a\\040\\141.txt\n"
+ MANY_ESCAPES_MANIFEST =
+ "./dir\\040name #{random_block(9)} 0:9:file\\\\name\\011\\here.txt\n"
+ NONNORMALIZED_MANIFEST =
+ ["./dir2 #{random_block} 0:0:z 0:0:y 0:0:x",
+ "./dir1 #{random_block} 0:0:p 0:0:o 0:0:n\n"].join("\n")
+end
diff --git a/sdk/ruby/test/test_keep_manifest.rb b/sdk/ruby/test/test_keep_manifest.rb
index 8ad8134..2ad0941 100644
--- a/sdk/ruby/test/test_keep_manifest.rb
+++ b/sdk/ruby/test/test_keep_manifest.rb
@@ -1,22 +1,9 @@
-require "minitest/autorun"
require "arvados/keep"
-require "yaml"
-
-def random_block(size=nil)
- sprintf("%032x+%d", rand(16 ** 32), size || rand(64 * 1024 * 1024))
-end
+require "minitest/autorun"
+require "sdk_fixtures"
class ManifestTest < Minitest::Test
- SIMPLEST_MANIFEST = ". #{random_block(9)} 0:9:simple.txt\n"
- MULTIBLOCK_FILE_MANIFEST =
- [". #{random_block(8)} 0:4:repfile 4:4:uniqfile",
- "./s1 #{random_block(6)} 0:3:repfile 3:3:uniqfile",
- ". #{random_block(8)} 0:7:uniqfile2 7:1:repfile\n"].join("\n")
- MULTILEVEL_MANIFEST =
- [". #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
- "./dir1 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
- "./dir1/subdir #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n",
- "./dir2 #{random_block(9)} 0:3:file1 3:3:file2 6:3:file3\n"].join("")
+ include SDKFixtures
def check_stream(stream, exp_name, exp_blocks, exp_files)
assert_equal(exp_name, stream.first)
@@ -51,7 +38,8 @@ class ManifestTest < Minitest::Test
seen << stream
assert_equal(3, files.size, "wrong file count for stream #{stream}")
end
- assert_equal(4, seen.size, "wrong number of streams")
+ assert_equal(MULTILEVEL_MANIFEST.count("\n"), seen.size,
+ "wrong number of streams")
end
def test_empty_each_line
@@ -87,13 +75,12 @@ class ManifestTest < Minitest::Test
end
def test_backslash_escape_parsing
- m_text = "./dir\\040name #{random_block} 0:0:file\\\\name\\011\\here.txt\n"
- manifest = Keep::Manifest.new(m_text)
+ manifest = Keep::Manifest.new(MANY_ESCAPES_MANIFEST)
streams = manifest.each_line.to_a
assert_equal(1, streams.size, "wrong number of streams with whitespace")
assert_equal("./dir name", streams.first.first,
"wrong stream name with whitespace")
- assert_equal(["0:0:file\\name\t\\here.txt"], streams.first.last,
+ assert_equal(["0:9:file\\name\t\\here.txt"], streams.first.last,
"wrong filename(s) with whitespace")
end
@@ -118,12 +105,12 @@ class ManifestTest < Minitest::Test
end
def test_files_with_colons_in_names
- manifest = Keep::Manifest.new(". #{random_block(9)} 0:9:file:test.txt\n")
+ manifest = Keep::Manifest.new(COLON_FILENAME_MANIFEST)
assert_equal([[".", "file:test.txt", 9]], manifest.files)
end
def test_files_with_escape_sequence_in_filename
- manifest = Keep::Manifest.new(". #{random_block(9)} 0:9:a\\040\\141.txt\n")
+ manifest = Keep::Manifest.new(ESCAPED_FILENAME_MANIFEST)
assert_equal([[".", "a a.txt", 9]], manifest.files)
end
@@ -177,11 +164,11 @@ class ManifestTest < Minitest::Test
end
def test_has_file_with_spaces
- manifest = Keep::Manifest.new(". #{random_block(3)} 0:3:a\\040b\\040c\n")
- assert(manifest.has_file?("./a b c"), "one-arg 'a b c' not found")
- assert(manifest.has_file?(".", "a b c"), "two-arg 'a b c' not found")
- refute(manifest.has_file?("a\\040b\\040c"), "one-arg unescaped found")
- refute(manifest.has_file?(".", "a\\040b\\040c"), "two-arg unescaped found")
+ manifest = Keep::Manifest.new(ESCAPED_FILENAME_MANIFEST)
+ assert(manifest.has_file?("./a a.txt"), "one-arg path not found")
+ assert(manifest.has_file?(".", "a a.txt"), "two-arg path not found")
+ refute(manifest.has_file?("a\\040\\141"), "one-arg unescaped found")
+ refute(manifest.has_file?(".", "a\\040\\141"), "two-arg unescaped found")
end
def test_parse_all_fixtures
@@ -208,15 +195,4 @@ class ManifestTest < Minitest::Test
assert !file_name.empty?, "empty file_name in #{name} fixture"
end
end
-
- @@fixtures = nil
- def fixtures name
- return @@fixtures if @@fixtures
- path = File.expand_path("../../../../services/api/test/fixtures/#{name}.yml",
- __FILE__)
- file = IO.read(path)
- trim_index = file.index('# Test Helper trims the rest of the file')
- file = file[0, trim_index] if trim_index
- @@fixtures = YAML.load(file)
- end
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list