[ARVADOS] created: 1.2.0-5-gb758b7e5c
Git user
git at public.curoverse.com
Mon Nov 19 15:28:17 EST 2018
at b758b7e5c284f59f675e9fc6938d917bb4c1981e (commit)
commit b758b7e5c284f59f675e9fc6938d917bb4c1981e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Nov 6 13:57:14 2018 -0500
14345: Handle "MOVE foo/ bar/" requests.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index bb2df6b30..5ffb1ad6e 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -636,6 +636,25 @@ func (s *CollectionFSSuite) TestRenameError(c *check.C) {
c.Check(data, check.DeepEquals, []byte{1, 2, 3, 4, 5})
}
+func (s *CollectionFSSuite) TestRenameDirectory(c *check.C) {
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("foo", 0755)
+ c.Assert(err, check.IsNil)
+ err = fs.Mkdir("bar", 0755)
+ c.Assert(err, check.IsNil)
+ err = fs.Rename("bar", "baz")
+ c.Check(err, check.IsNil)
+ err = fs.Rename("foo", "baz")
+ c.Check(err, check.NotNil)
+ err = fs.Rename("foo", "baz/")
+ c.Check(err, check.IsNil)
+ err = fs.Rename("baz/foo", ".")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+ err = fs.Rename("baz/foo/", ".")
+ c.Check(err, check.Equals, ErrInvalidArgument)
+}
+
func (s *CollectionFSSuite) TestRename(c *check.C) {
fs, err := (&Collection{}).FileSystem(s.client, s.kc)
c.Assert(err, check.IsNil)
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index b89890956..44d0b0ffe 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -172,6 +172,16 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
},
{
path: writePath,
+ cmd: "move newdir1/ newdir1x/\n",
+ match: `(?ms).*Moving .* succeeded.*`,
+ },
+ {
+ path: writePath,
+ cmd: "move newdir1x newdir1\n",
+ match: `(?ms).*Moving .* succeeded.*`,
+ },
+ {
+ path: writePath,
cmd: "move newdir0/testfile newdir1/\n",
match: `(?ms).*Moving .* succeeded.*`,
},
diff --git a/services/keep-web/webdav.go b/services/keep-web/webdav.go
index 5b23c9c5f..f9b753869 100644
--- a/services/keep-web/webdav.go
+++ b/services/keep-web/webdav.go
@@ -100,6 +100,11 @@ func (fs *webdavFS) Rename(ctx context.Context, oldName, newName string) error {
if !fs.writing {
return errReadOnly
}
+ if strings.HasSuffix(oldName, "/") {
+ // WebDAV "MOVE foo/ bar/" means rename foo to bar.
+ oldName = oldName[:len(oldName)-1]
+ newName = strings.TrimSuffix(newName, "/")
+ }
fs.makeparents(newName)
return fs.collfs.Rename(oldName, newName)
}
commit 9777970b61c21a55565b09e0a918ed64f59b0b22
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Nov 6 09:54:10 2018 -0500
14345: Accept "." placeholder: create parent dir, but no fake file.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index 6396cbcf5..fccf3ca59 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1677,12 +1677,17 @@ class Collection(RichCollectionBase):
pos = int(file_segment.group(1))
size = int(file_segment.group(2))
name = self._unescape_manifest_path(file_segment.group(3))
- filepath = os.path.join(stream_name, name)
- afile = self.find_or_create(filepath, FILE)
- if isinstance(afile, ArvadosFile):
- afile.add_segment(blocks, pos, size)
+ if name.split('/')[-1] == '.':
+ # placeholder for persisting an empty directory, not a real file
+ if len(name) > 2:
+ self.find_or_create(os.path.join(stream_name, name[:-2]), COLLECTION)
else:
- raise errors.SyntaxError("File %s conflicts with stream of the same name.", filepath)
+ filepath = os.path.join(stream_name, name)
+ afile = self.find_or_create(filepath, FILE)
+ if isinstance(afile, ArvadosFile):
+ afile.add_segment(blocks, pos, size)
+ else:
+ raise errors.SyntaxError("File %s conflicts with stream of the same name.", filepath)
else:
# error!
raise errors.SyntaxError("Invalid manifest format, expected file segment but did not match format: '%s'" % tok)
commit 6c1092a7060389191b3340fd0e988d244c3a9119
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Oct 18 09:41:37 2018 -0400
14345: Accept lock/unlock requests as no-ops.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-web/cadaver_test.go b/services/keep-web/cadaver_test.go
index 0e2f17c35..b89890956 100644
--- a/services/keep-web/cadaver_test.go
+++ b/services/keep-web/cadaver_test.go
@@ -147,6 +147,16 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
},
{
path: writePath,
+ cmd: "lock newdir0/testfile\n",
+ match: `(?ms).*Locking .* succeeded.*`,
+ },
+ {
+ path: writePath,
+ cmd: "unlock newdir0/testfile\nasdf\n",
+ match: `(?ms).*Unlocking .* succeeded.*`,
+ },
+ {
+ path: writePath,
cmd: "ls\n",
match: `(?ms).*newdir0.* 0 +` + matchToday + ` \d+:\d+\n.*`,
},
@@ -243,6 +253,11 @@ func (s *IntegrationSuite) testCadaver(c *check.C, password string, pathFunc fun
cmd: "delete foo\n",
match: `(?ms).*Deleting .* failed:.*405 Method Not Allowed.*`,
},
+ {
+ path: pdhPath,
+ cmd: "lock foo\n",
+ match: `(?ms).*Locking .* failed:.*405 Method Not Allowed.*`,
+ },
} {
c.Logf("%s %+v", "http://"+s.testServer.Addr, trial)
if skip != nil && skip(trial.path) {
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 912398fa6..237943fb3 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -143,20 +143,24 @@ var (
writeMethod = map[string]bool{
"COPY": true,
"DELETE": true,
+ "LOCK": true,
"MKCOL": true,
"MOVE": true,
"PUT": true,
"RMCOL": true,
+ "UNLOCK": true,
}
webdavMethod = map[string]bool{
"COPY": true,
"DELETE": true,
+ "LOCK": true,
"MKCOL": true,
"MOVE": true,
"OPTIONS": true,
"PROPFIND": true,
"PUT": true,
"RMCOL": true,
+ "UNLOCK": true,
}
browserMethod = map[string]bool{
"GET": true,
commit caf33146ca9897ba93e0bc8e1a32247d0a94e3e7
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Nov 5 16:46:50 2018 -0500
14345: Unescape all \ooo in names in manifests, not just \040.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index e38a6bd47..6396cbcf5 100644
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -1627,6 +1627,9 @@ class Collection(RichCollectionBase):
_block_re = re.compile(r'[0-9a-f]{32}\+(\d+)(\+\S+)*')
_segment_re = re.compile(r'(\d+):(\d+):(\S+)')
+ def _unescape_manifest_path(self, path):
+ return re.sub('\\\\([0-3][0-7][0-7])', lambda m: chr(int(m.group(1), 8)), path)
+
@synchronized
def _import_manifest(self, manifest_text):
"""Import a manifest into a `Collection`.
@@ -1651,7 +1654,7 @@ class Collection(RichCollectionBase):
if state == STREAM_NAME:
# starting a new stream
- stream_name = tok.replace('\\040', ' ')
+ stream_name = self._unescape_manifest_path(tok)
blocks = []
segments = []
streamoffset = 0
@@ -1673,7 +1676,7 @@ class Collection(RichCollectionBase):
if file_segment:
pos = int(file_segment.group(1))
size = int(file_segment.group(2))
- name = file_segment.group(3).replace('\\040', ' ')
+ name = self._unescape_manifest_path(file_segment.group(3))
filepath = os.path.join(stream_name, name)
afile = self.find_or_create(filepath, FILE)
if isinstance(afile, ArvadosFile):
commit 69d591465ba136e6af9ba49b52e6fec9e6e94911
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Nov 5 16:15:15 2018 -0500
14345: Use "." placeholder to persist empty directories.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 7ce37aa24..4cac22729 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -633,6 +633,16 @@ func (dn *dirnode) marshalManifest(prefix string) (string, error) {
if err := dn.sync(); err != nil {
return "", err
}
+ if len(dn.inodes) == 0 {
+ if prefix == "." {
+ return "", nil
+ }
+ // Express the existence of an empty directory by
+ // adding an empty file named `\056`, which (unlike
+ // the more obvious spelling `.`) is accepted by the
+ // API's manifest validator.
+ return manifestEscape(prefix) + " d41d8cd98f00b204e9800998ecf8427e+0 0:0:\\056\n", nil
+ }
names := make([]string, 0, len(dn.inodes))
for name, node := range dn.inodes {
@@ -757,8 +767,14 @@ func (dn *dirnode) loadManifest(txt string) error {
}
name := dirname + "/" + manifestUnescape(toks[2])
fnode, err := dn.createFileAndParents(name)
- if err != nil {
- return fmt.Errorf("line %d: cannot use path %q: %s", lineno, name, err)
+ if fnode == nil && err == nil && length == 0 {
+ // Special case: an empty file used as
+ // a marker to preserve an otherwise
+ // empty directory in a manifest.
+ continue
+ }
+ if err != nil || (fnode == nil && length != 0) {
+ return fmt.Errorf("line %d: cannot use path %q with length %d: %s", lineno, name, length, err)
}
// Map the stream offset/range coordinates to
// block/offset/range coordinates and add
@@ -816,15 +832,14 @@ func (dn *dirnode) loadManifest(txt string) error {
return nil
}
-// only safe to call from loadManifest -- no locking
+// only safe to call from loadManifest -- no locking.
+//
+// If path is a "parent directory exists" marker (the last path
+// component is "."), the returned values are both nil.
func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
var node inode = dn
names := strings.Split(path, "/")
basename := names[len(names)-1]
- if !permittedName(basename) {
- err = fmt.Errorf("invalid file part %q in path %q", basename, path)
- return
- }
for _, name := range names[:len(names)-1] {
switch name {
case "", ".":
@@ -855,6 +870,12 @@ func (dn *dirnode) createFileAndParents(path string) (fn *filenode, err error) {
return
}
}
+ if basename == "." {
+ return
+ } else if !permittedName(basename) {
+ err = fmt.Errorf("invalid file part %q in path %q", basename, path)
+ return
+ }
_, err = node.Child(basename, func(child inode) (inode, error) {
switch child := child.(type) {
case nil:
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index d2f55d0e3..bb2df6b30 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -786,11 +786,11 @@ func (s *CollectionFSSuite) TestPersist(c *check.C) {
}
}
-func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
+func (s *CollectionFSSuite) TestPersistEmptyFilesAndDirs(c *check.C) {
var err error
s.fs, err = (&Collection{}).FileSystem(s.client, s.kc)
c.Assert(err, check.IsNil)
- for _, name := range []string{"dir", "dir/zerodir", "zero", "zero/zero"} {
+ for _, name := range []string{"dir", "dir/zerodir", "empty", "not empty", "not empty/empty", "zero", "zero/zero"} {
err = s.fs.Mkdir(name, 0755)
c.Assert(err, check.IsNil)
}
@@ -836,6 +836,23 @@ func (s *CollectionFSSuite) TestPersistEmptyFiles(c *check.C) {
c.Check(err, check.IsNil)
c.Check(buf, check.DeepEquals, data)
}
+
+ expectDir := map[string]int{
+ "empty": 0,
+ "not empty": 1,
+ "not empty/empty": 0,
+ }
+ for name, expectLen := range expectDir {
+ _, err := persisted.Open(name + "/bogus")
+ c.Check(err, check.NotNil)
+
+ d, err := persisted.Open(name)
+ defer d.Close()
+ c.Check(err, check.IsNil)
+ fi, err := d.Readdir(-1)
+ c.Check(err, check.IsNil)
+ c.Check(fi, check.HasLen, expectLen)
+ }
}
func (s *CollectionFSSuite) TestOpenFileFlags(c *check.C) {
@@ -987,6 +1004,12 @@ func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
". d41d8cd98f00b204e9800998ecf8427e+0 foo:0:foo\n",
". d41d8cd98f00b204e9800998ecf8427e+0 0:foo:foo\n",
". d41d8cd98f00b204e9800998ecf8427e+1 0:1:foo 1:1:bar\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:\\056\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:\\056\\057\\056\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:.\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+1 0:1:..\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:..\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/..\n",
". d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n",
"./foo d41d8cd98f00b204e9800998ecf8427e+1 0:0:bar\n. d41d8cd98f00b204e9800998ecf8427e+1 0:0:foo\n",
} {
@@ -1002,7 +1025,9 @@ func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
for _, txt := range []string{
"",
". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo\n",
- ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:...\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:. 0:0:. 0:0:\\056 0:0:\\056\n",
+ ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/. 0:0:. 0:0:foo\\057bar\\057\\056\n",
". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo 0:0:foo 0:0:bar\n",
". d41d8cd98f00b204e9800998ecf8427e+0 0:0:foo/bar\n./foo d41d8cd98f00b204e9800998ecf8427e+0 0:0:bar\n",
} {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list