[ARVADOS] updated: 1.1.1-251-g2204a8d
Git user
git at public.curoverse.com
Tue Dec 12 11:53:34 EST 2017
Summary of changes:
apps/workbench/app/views/application/404.html.erb | 25 ++++---
apps/workbench/test/integration/errors_test.rb | 13 ++++
sdk/ArvadosSDK_0.1.0_R_x86_64-pc-linux-gnu.tar.gz | Bin 0 -> 40806 bytes
sdk/R/.RData | Bin 301821 -> 345526 bytes
sdk/R/R/ArvadosFile.R | 6 +-
sdk/R/R/Collection.R | 72 +++++++++------------
sdk/R/R/HttpParser.R | 17 +++--
sdk/R/R/Subcollection.R | 9 +++
sdk/R/README | 17 +++--
sdk/cli/bin/crunch-job | 29 +++++++--
sdk/go/arvados/collection_fs.go | 2 +-
sdk/go/arvados/collection_fs_test.go | 17 ++++-
sdk/python/arvados/api.py | 30 ++++++---
sdk/python/arvados/commands/get.py | 28 +++++---
sdk/python/arvados/commands/put.py | 26 ++++++--
sdk/python/arvados/keep.py | 44 ++++++++-----
sdk/python/arvados/util.py | 17 +++++
sdk/python/tests/test_api.py | 27 ++++++++
sdk/python/tests/test_arv_get.py | 26 +++++++-
sdk/python/tests/test_arv_put.py | 22 ++++++-
sdk/python/tests/test_events.py | 2 +-
sdk/python/tests/test_keep_client.py | 72 +++++++++++++++++++++
.../arvados/v1/collections_controller.rb | 2 +-
.../controllers/arvados/v1/groups_controller.rb | 7 +-
services/api/app/models/arvados_model.rb | 48 ++++++++------
services/api/app/models/collection.rb | 3 -
services/api/app/models/container.rb | 6 +-
services/api/app/models/container_request.rb | 4 +-
services/api/lib/sweep_trashed_collections.rb | 4 +-
services/api/test/fixtures/container_requests.yml | 22 +++++++
.../arvados/v1/collections_controller_test.rb | 6 +-
services/api/test/unit/collection_test.rb | 21 +++---
services/api/test/unit/container_test.rb | 4 +-
33 files changed, 461 insertions(+), 167 deletions(-)
create mode 100644 sdk/ArvadosSDK_0.1.0_R_x86_64-pc-linux-gnu.tar.gz
via 2204a8d9305c85d2f7d65621a66443e7104c5f6b (commit)
via 296aa66f8770b18dcc1ea6b82cacd7c0c047922b (commit)
via 9b7a5de1e7ef2656fe58e7fcad180f4de2fcc619 (commit)
via 68054313a874a5b4a7e381623ac557ebdb7ebc35 (commit)
via a084df558c2bb054140a2bea08c973fa76209b97 (commit)
via 42377686908783fb9d043e616f099e153f7834bc (commit)
via b112b002ee3239803d1948e99463144812a2c213 (commit)
via dd9651140d782919e475a6252e4c770c46d96252 (commit)
via 34105f94a3b9a93d3efd1452c1e5e9643a4ed1dc (commit)
via 5123e035f3a9263afb76d2b1269b081a0b003544 (commit)
via 363fc6395f866a9631f41630cbff49ac53e9a211 (commit)
via a70b961cc07a399c123774f2acdc5c118efa2791 (commit)
via 4453022a516e2b1deb30a71d8ee811d6593c44c3 (commit)
via 9674a836236a91e820fd4faff32442a087501383 (commit)
via e13f1473fb0ea51414bc54136723ec3acd7045f2 (commit)
via 2321fa1ced5faa1ace9d5b72e3b2eb4afd0e721f (commit)
via 15f50edccc93c0c97eef30ff2662ab797a670e92 (commit)
via fbe39c0c33a9c9c3451ad6bf8ec4b336e7e3b24f (commit)
via fed9f70cdc99a3bc94a54bf1d182b13204a597be (commit)
from 6c2355d82da1326a125ce7adcb8945a1d8efd671 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 2204a8d9305c85d2f7d65621a66443e7104c5f6b
Merge: a084df5 296aa66
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Tue Dec 12 17:52:57 2017 +0100
Merge branch 'master' of git.curoverse.com:arvados into 11876-r-sdk
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
commit a084df558c2bb054140a2bea08c973fa76209b97
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Tue Dec 12 16:01:08 2017 +0100
Implemented feature to get size of ArvadosFile or Subcollection
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/ArvadosSDK_0.1.0_R_x86_64-pc-linux-gnu.tar.gz b/sdk/ArvadosSDK_0.1.0_R_x86_64-pc-linux-gnu.tar.gz
new file mode 100644
index 0000000..aa802dd
Binary files /dev/null and b/sdk/ArvadosSDK_0.1.0_R_x86_64-pc-linux-gnu.tar.gz differ
diff --git a/sdk/R/.RData b/sdk/R/.RData
index b08e775..88c2ce7 100644
Binary files a/sdk/R/.RData and b/sdk/R/.RData differ
diff --git a/sdk/R/R/ArvadosFile.R b/sdk/R/R/ArvadosFile.R
index f7c45dc..ce3a6ff 100644
--- a/sdk/R/R/ArvadosFile.R
+++ b/sdk/R/R/ArvadosFile.R
@@ -9,9 +9,10 @@ ArvadosFile <- R6::R6Class(
public = list(
- initialize = function(name, relativePath, api, collection)
+ initialize = function(name, relativePath, size, api, collection)
{
private$name <- name
+ private$size <- size
private$relativePath <- relativePath
private$api <- api
private$collection <- collection
@@ -23,6 +24,8 @@ ArvadosFile <- R6::R6Class(
getRelativePath = function() private$relativePath,
+ getSizeInBytes = function() private$size,
+
read = function(offset = 0, length = 0)
{
if(offset < 0 || length < 0)
@@ -50,6 +53,7 @@ ArvadosFile <- R6::R6Class(
name = NULL,
relativePath = NULL,
+ size = NULL,
parent = NULL,
api = NULL,
collection = NULL,
diff --git a/sdk/R/R/Collection.R b/sdk/R/R/Collection.R
index fed222f..14371a6 100644
--- a/sdk/R/R/Collection.R
+++ b/sdk/R/R/Collection.R
@@ -67,13 +67,17 @@ Collection <- R6::R6Class(
private$fileTree <- private$generateTree(private$fileItems)
},
- printFileContent = function(pretty = TRUE)
+ printFileContent = function()
{
- if(pretty)
- private$fileTree$printContent(0)
- else
- print(private$fileItems)
+ private$fileTree$printContent(0)
+ },
+ getFileContent = function()
+ {
+ sapply(private$fileItems, function(file)
+ {
+ file$name
+ })
},
get = function(relativePath)
@@ -111,8 +115,8 @@ Collection <- R6::R6Class(
private = list(
- api = NULL,
fileItems = NULL,
+ api = NULL,
fileTree = NULL,
createSubcollectionTree = function(treeNode)
@@ -132,23 +136,12 @@ Collection <- R6::R6Class(
else
{
if(treeNode$type == "file")
- return(ArvadosFile$new(treeNode$name, treeNode$relativePath, private$api, self))
- else if(treeNode$type == "folder" || treeNode$type == "root")
+ return(ArvadosFile$new(treeNode$name, treeNode$relativePath, treeNode$size, private$api, self))
+ else
return(Subcollection$new(treeNode$name, treeNode$relativePath, NULL))
}
},
- createSubcollectionFromNode = function(treeNode, children)
- {
- subcollection = NULL
- if(treeNode$type == "file")
- subcollection = ArvadosFile$new(treeNode$name, treeNode$relativePath)
- else if(treeNode$type == "folder" || treeNode$type == "root")
- subcollection = Subcollection$new(treeNode$name, treeNode$relativePath, children)
-
- subcollection
- },
-
getCollectionContent = function()
{
#TODO(Fudo): Use proper URL here.
@@ -170,14 +163,12 @@ Collection <- R6::R6Class(
{
treeBranches <- sapply(collectionContent, function(filePath)
{
- splitPath <- unlist(strsplit(filePath, "/", fixed = TRUE))
+ splitPath <- unlist(strsplit(filePath$name, "/", fixed = TRUE))
- pathEndsWithSlash <- substr(filePath, nchar(filePath), nchar(filePath)) == "/"
-
- branch = private$createBranch(splitPath, pathEndsWithSlash)
+ branch = private$createBranch(splitPath, filePath$fileSize)
})
- root <- TreeNode$new("./", "root")
+ root <- TreeNode$new("./", "root", NULL)
root$relativePath = ""
sapply(treeBranches, function(branch)
@@ -188,31 +179,27 @@ Collection <- R6::R6Class(
root
},
- createBranch = function(splitPath, pathEndsWithSlash)
+ createBranch = function(splitPath, fileSize)
{
branch <- NULL
lastElementIndex <- length(splitPath)
-
- lastElementInPathType = "file"
- if(pathEndsWithSlash)
- lastElementInPathType = "folder"
for(elementIndex in lastElementIndex:1)
{
if(elementIndex == lastElementIndex)
{
- branch = TreeNode$new(splitPath[[elementIndex]], lastElementInPathType)
+ branch = TreeNode$new(splitPath[[elementIndex]], "file", fileSize)
}
else
{
- newFolder = TreeNode$new(splitPath[[elementIndex]], "folder")
+ newFolder = TreeNode$new(splitPath[[elementIndex]], "folder", NULL)
newFolder$addChild(branch)
branch = newFolder
}
branch$relativePath <- paste(unlist(splitPath[1:elementIndex]), collapse = "/")
}
-
+
branch
},
@@ -226,6 +213,7 @@ Collection <- R6::R6Class(
}
else
{
+ child$type = "folder"
private$addNode(child, node$getFirstChild())
}
},
@@ -266,19 +254,18 @@ TreeNode <- R6::R6Class(
public = list(
- name = NULL,
+ name = NULL,
relativePath = NULL,
- children = NULL,
- parent = NULL,
- type = NULL,
+ size = NULL,
+ children = NULL,
+ parent = NULL,
+ type = NULL,
- initialize = function(name, type)
+ initialize = function(name, type, size)
{
- if(type == "folder")
- name <- paste0(name, "/")
-
self$name <- name
self$type <- type
+ self$size <- size
self$children <- list()
},
@@ -324,7 +311,10 @@ TreeNode <- R6::R6Class(
printContent = function(depth)
{
indentation <- paste(rep("....", depth), collapse = "")
- print(paste0(indentation, self$name))
+ if(self$type == "folder")
+ print(paste0(indentation, self$name, "/"))
+ else
+ print(paste0(indentation, self$size))
for(child in self$children)
child$printContent(depth + 1)
diff --git a/sdk/R/R/HttpParser.R b/sdk/R/R/HttpParser.R
index d54207e..9129bbc 100644
--- a/sdk/R/R/HttpParser.R
+++ b/sdk/R/R/HttpParser.R
@@ -18,17 +18,20 @@ HttpParser <- R6::R6Class(
parseWebDAVResponse = function(response, uri)
{
text <- rawToChar(response$content)
- print(text)
doc <- XML::xmlParse(text, asText=TRUE)
# calculate relative paths
base <- paste(paste("/", strsplit(uri, "/")[[1]][-1:-3], sep="", collapse=""), "/", sep="")
- result <- unlist(
- XML::xpathApply(doc, "//D:response/D:href", function(node) {
- sub(base, "", URLdecode(XML::xmlValue(node)), fixed=TRUE)
- })
- )
- result <- result[result != ""]
+ result <- XML::xpathApply(doc, "//D:response", function(node) {
+ result = list()
+ children = xmlChildren(node)
+
+ result$name = sub(base, "", URLdecode(XML::xmlValue(children$href)), fixed=TRUE)
+ sizeXMLNode = xmlChildren(xmlChildren(children$propstat)$prop)$getcontentlength
+ result$fileSize = as.numeric(xmlValue(sizeXMLNode))
+
+ result
+ })
result[-1]
}
diff --git a/sdk/R/R/Subcollection.R b/sdk/R/R/Subcollection.R
index 4053546..1d8c8a7 100644
--- a/sdk/R/R/Subcollection.R
+++ b/sdk/R/R/Subcollection.R
@@ -20,6 +20,15 @@ Subcollection <- R6::R6Class(
getRelativePath = function() private$relativePath,
+ getSizeInBytes = function()
+ {
+ overallSize = 0
+ for(child in private$children)
+ overallSize = overallSize + child$getSizeInBytes()
+
+ overallSize
+ },
+
setParent = function(parent) private$parent <- parent
),
diff --git a/sdk/R/README b/sdk/R/README
index 6f4e31e..344deab 100644
--- a/sdk/R/README
+++ b/sdk/R/README
@@ -55,13 +55,15 @@ collection <- Collection$new(arv, "uuid")
--------------------------------------------------------------------------------------------------------------------------------
-#Print content of the collection
+#Print content of the collection (directory/folder tree structure)
collection$printFileContent()
-#of if you just want a list of relative paths:
+--------------------------------------------------------------------------------------------------------------------------------
+
+#Get file/folder content as character vector
-collection$printFileContent(pretty = FALSE)
+collection$getFileContent()
--------------------------------------------------------------------------------------------------------------------------------
@@ -77,6 +79,13 @@ arvadosSubcollection <- collection.get("location/to/my/directory/")
#Read whole file or just a portion of it.
-arvadosFile$read(offset = 1024, length = 512)
+fileContent <- arvadosFile$read(offset = 1024, length = 512)
+
+--------------------------------------------------------------------------------------------------------------------------------
+
+#Get ArvadosFile or Subcollection size
+
+size <- arvadosFile$getSizeInBytes()
+size <- arvadosSubcollection$getSizeInBytes()
--------------------------------------------------------------------------------------------------------------------------------
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list