[ARVADOS] updated: 1.1.1-267-gf67c59f
Git user
git at public.curoverse.com
Wed Dec 13 12:09:05 EST 2017
Summary of changes:
build/run-build-docker-jobs-image.sh | 19 +-
doc/api/methods.html.textile.liquid | 27 ++-
sdk/R/.RData | Bin 345526 -> 399721 bytes
sdk/R/DESCRIPTION | 2 +-
sdk/R/R/ArvadosFile.R | 38 +++-
sdk/R/R/Collection.R | 190 +++----------------
sdk/R/R/FileTree.R | 192 ++++++++++++++++++++
sdk/R/R/HttpParser.R | 8 +-
sdk/R/R/HttpRequest.R | 20 +-
...53352_add_gin_index_to_collection_properties.rb | 8 +
services/api/db/structure.sql | 10 +
services/api/lib/record_filters.rb | 201 ++++++++++++++-------
services/api/test/fixtures/collections.yml | 98 ++++++++++
.../api/test/functional/arvados/v1/filters_test.rb | 128 +++++++++++++
.../arvnodeman/computenode/dispatch/transitions.py | 15 +-
services/nodemanager/arvnodeman/daemon.py | 5 +-
services/nodemanager/arvnodeman/nodelist.py | 4 +-
.../nodemanager/tests/test_computenode_dispatch.py | 7 +
18 files changed, 705 insertions(+), 267 deletions(-)
create mode 100644 sdk/R/R/FileTree.R
create mode 100644 services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb
via f67c59f3fea793ed3ccea7f5a8106014ec2dc108 (commit)
via 6e2965abc46832a4970b80bce84a64d932426285 (commit)
via e768a05df9fd75cee3724e6b68cb65beeebaaa38 (commit)
via a879823f631381cefc4458c28f06c36803e30530 (commit)
via a8c3d59c520942c14434cea310c2c8a2fe623a9c (commit)
via 91dc5aafab57c131973244b771392cab083ab469 (commit)
via efa2c6e65983a0917119d7eaf1daa42bc440f52a (commit)
via 5db84e82f71711b6897f06c7da05dc28bba205fa (commit)
via 48ef04b9f09e2f739d3f8c5aba1c7dede305d416 (commit)
via 9379719d46bb351f87aa9c32a0325a65900b9faa (commit)
via ba6e96dfc00cc6ce4f6ab299bdbff616a523ce90 (commit)
via 79a2d819d596e610f26c08beee53f5432bfbb360 (commit)
via 9ec15072e6631578584bd9c08d26025e807a8d48 (commit)
via f9a3dea1aed09b7b1d885c3116c63310e0440007 (commit)
via 40d5d40955a88dbd5cd7c25292268f1ab4536bda (commit)
via 667a7121e08d4fffc24cafdc3ed474374782b959 (commit)
from 2204a8d9305c85d2f7d65621a66443e7104c5f6b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f67c59f3fea793ed3ccea7f5a8106014ec2dc108
Merge: 6e2965a e768a05
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Wed Dec 13 18:08:49 2017 +0100
Merge branch 'master' of git.curoverse.com:arvados into 11876-r-sdk
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
commit 6e2965abc46832a4970b80bce84a64d932426285
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Wed Dec 13 18:07:54 2017 +0100
Refactored collection tree into separate file.
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/R/R/ArvadosFile.R b/sdk/R/R/ArvadosFile.R
index f13bf57..da8692d 100644
--- a/sdk/R/R/ArvadosFile.R
+++ b/sdk/R/R/ArvadosFile.R
@@ -37,7 +37,6 @@ ArvadosFile <- R6::R6Class(
range = paste0(range, offset + length - 1)
fileURL = paste0(private$api$getWebDavHostName(), "c=", private$collection$uuid, "/", private$relativePath);
- print(fileURL)
headers <- list(Authorization = paste("OAuth2", private$api$getToken()),
Range = range)
@@ -77,7 +76,8 @@ ArvadosFile <- R6::R6Class(
private$size <- fileInfo[[1]]$fileSize
private$collection$update(self, "File size changed")
- #parsed_response <- httr::content(serverResponse, "text")
+
+ parsed_response <- httr::content(serverResponse, "text")
}
),
diff --git a/sdk/R/R/Collection.R b/sdk/R/R/Collection.R
index cecf657..c29f8f0 100644
--- a/sdk/R/R/Collection.R
+++ b/sdk/R/R/Collection.R
@@ -1,5 +1,6 @@
source("./R/Subcollection.R")
source("./R/ArvadosFile.R")
+source("./R/FileTree.R")
#' Arvados Collection Object
#'
@@ -63,12 +64,12 @@ Collection <- R6::R6Class(
private$fileItems <- private$getCollectionContent()
- private$fileTree <- private$generateTree(private$fileItems)
+ private$fileTree <- FileTree$new(private$fileItems)
},
printFileContent = function()
{
- private$fileTree$printContent(0)
+ private$fileTree$printContent(private$fileTree$getRoot(), 0)
},
getFileContent = function()
@@ -81,7 +82,7 @@ Collection <- R6::R6Class(
get = function(relativePath)
{
- treeNode <- private$traverseInOrder(private$fileTree, function(node)
+ treeNode <- private$fileTree$traverseInOrder(private$fileTree$getRoot(), function(node)
{
if(node$relativePath == relativePath)
return(node)
@@ -130,9 +131,7 @@ Collection <- R6::R6Class(
handleFileSizeChange = function(filePath, newSize)
{
- print(paste(filePath, newSize))
-
- node <- private$getNode(filePath)
+ node <- private$fileTree$getNode(filePath)
node$size <- newSize
},
@@ -172,189 +171,9 @@ Collection <- R6::R6Class(
parsedResponse <- HttpParser$new()$parseWebDAVResponse(response, uri)
parsedResponse[-1]
- },
-
- #Todo(Fudo): Move tree creation to another file.
- generateTree = function(collectionContent)
- {
- treeBranches <- sapply(collectionContent, function(filePath)
- {
- splitPath <- unlist(strsplit(filePath$name, "/", fixed = TRUE))
-
- branch = private$createBranch(splitPath, filePath$fileSize)
- })
-
- root <- TreeNode$new("./", "root", NULL)
- root$relativePath = ""
-
- sapply(treeBranches, function(branch)
- {
- private$addNode(root, branch)
- })
-
- root
- },
-
- createBranch = function(splitPath, fileSize)
- {
- branch <- NULL
- lastElementIndex <- length(splitPath)
-
- for(elementIndex in lastElementIndex:1)
- {
- if(elementIndex == lastElementIndex)
- {
- branch = TreeNode$new(splitPath[[elementIndex]], "file", fileSize)
- }
- else
- {
- newFolder = TreeNode$new(splitPath[[elementIndex]], "folder", NULL)
- newFolder$addChild(branch)
- branch = newFolder
- }
-
- branch$relativePath <- paste(unlist(splitPath[1:elementIndex]), collapse = "/")
- }
-
- branch
- },
-
- addNode = function(container, node)
- {
- child = container$getChild(node$name)
-
- if(is.null(child))
- {
- container$addChild(node)
- }
- else
- {
- child$type = "folder"
- private$addNode(child, node$getFirstChild())
- }
- },
-
- traverseInOrder = function(node, predicate)
- {
- if(node$hasChildren())
- {
- result <- predicate(node)
-
- if(!is.null(result))
- return(result)
-
- for(child in node$children)
- {
- result <- private$traverseInOrder(child, predicate)
-
- if(!is.null(result))
- return(result)
- }
-
- return(NULL)
- }
- else
- {
- return(predicate(node))
- }
- },
-
- getNode = function(relativePathToNode)
- {
- treeBranches <- sapply(relativePathToNode, function(filePath)
- {
- splitPath <- unlist(strsplit(filePath, "/", fixed = TRUE))
-
- node = private$fileTree
- for(pathFragment in splitPath)
- {
- child = node$getChild(pathFragment)
- if(is.null(child))
- stop("Subcollection/ArvadosFile you are looking for doesn't exist.")
- node = child
- }
-
- node
- })
}
),
cloneable = FALSE
)
-
-TreeNode <- R6::R6Class(
-
- "TreeNode",
-
- public = list(
-
- name = NULL,
- relativePath = NULL,
- size = NULL,
- children = NULL,
- parent = NULL,
- type = NULL,
-
- initialize = function(name, type, size)
- {
- self$name <- name
- self$type <- type
- self$size <- size
- self$children <- list()
- },
-
- addChild = function(node)
- {
- self$children <- c(self$children, node)
- node$setParent(self)
- self
- },
-
- setParent = function(parent)
- {
- self$parent = parent
- },
-
- getChild = function(childName)
- {
- for(child in self$children)
- {
- if(childName == child$name)
- return(child)
- }
-
- return(NULL)
- },
-
- hasChildren = function()
- {
- if(length(self$children) != 0)
- return(TRUE)
- else
- return(FALSE)
- },
-
- getFirstChild = function()
- {
- if(!self$hasChildren())
- return(NULL)
- else
- return(self$children[[1]])
- },
-
- printContent = function(depth)
- {
- indentation <- paste(rep("....", depth), collapse = "")
- if(self$type == "folder")
- print(paste0(indentation, self$name, "/"))
- else
- print(paste0(indentation, self$name))
-
- for(child in self$children)
- child$printContent(depth + 1)
- }
- ),
-
- cloneable = FALSE
-)
diff --git a/sdk/R/R/FileTree.R b/sdk/R/R/FileTree.R
new file mode 100644
index 0000000..ee54bd9
--- /dev/null
+++ b/sdk/R/R/FileTree.R
@@ -0,0 +1,192 @@
+FileTree <- R6::R6Class(
+ "FileTree",
+ public = list(
+ initialize = function(collectionContent)
+ {
+ treeBranches <- sapply(collectionContent, function(filePath)
+ {
+ splitPath <- unlist(strsplit(filePath$name, "/", fixed = TRUE))
+
+ branch = private$createBranch(splitPath, filePath$fileSize)
+ })
+
+ root <- TreeNode$new("./", "root", NULL)
+ root$relativePath = ""
+
+ sapply(treeBranches, function(branch)
+ {
+ private$addBranch(root, branch)
+ })
+
+ private$tree <- root
+ },
+
+ getRoot = function() private$tree,
+
+ printContent = function(node, depth)
+ {
+ indentation <- paste(rep("....", depth), collapse = "")
+ if(node$type == "folder")
+ print(paste0(indentation, node$name, "/"))
+ else
+ print(paste0(indentation, node$name))
+
+ for(child in node$children)
+ self$printContent(child, depth + 1)
+ },
+
+ traverseInOrder = function(node, predicate)
+ {
+ if(node$hasChildren())
+ {
+ result <- predicate(node)
+
+ if(!is.null(result))
+ return(result)
+
+ for(child in node$children)
+ {
+ result <- self$traverseInOrder(child, predicate)
+
+ if(!is.null(result))
+ return(result)
+ }
+
+ return(NULL)
+ }
+ else
+ {
+ return(predicate(node))
+ }
+ },
+
+ getNode = function(relativePathToNode)
+ {
+ treeBranches <- sapply(relativePathToNode, function(filePath)
+ {
+ splitPath <- unlist(strsplit(filePath, "/", fixed = TRUE))
+
+ node <- private$tree
+ for(pathFragment in splitPath)
+ {
+ child = node$getChild(pathFragment)
+ if(is.null(child))
+ stop("Subcollection/ArvadosFile you are looking for doesn't exist.")
+ node = child
+ }
+
+ node
+ })
+ }
+ ),
+
+ private = list(
+ tree = NULL,
+
+ createBranch = function(splitPath, fileSize)
+ {
+ branch <- NULL
+ lastElementIndex <- length(splitPath)
+
+ for(elementIndex in lastElementIndex:1)
+ {
+ if(elementIndex == lastElementIndex)
+ {
+ branch = TreeNode$new(splitPath[[elementIndex]], "file", fileSize)
+ }
+ else
+ {
+ newFolder = TreeNode$new(splitPath[[elementIndex]], "folder", NULL)
+ newFolder$addChild(branch)
+ branch = newFolder
+ }
+
+ branch$relativePath <- paste(unlist(splitPath[1:elementIndex]), collapse = "/")
+ }
+
+ branch
+ },
+
+ addBranch = function(container, node)
+ {
+ child = container$getChild(node$name)
+
+ if(is.null(child))
+ {
+ container$addChild(node)
+ }
+ else
+ {
+ child$type = "folder"
+ private$addBranch(child, node$getFirstChild())
+ }
+ }
+ ),
+
+ cloneable = FALSE
+)
+
+TreeNode <- R6::R6Class(
+
+ "TreeNode",
+
+ public = list(
+
+ name = NULL,
+ relativePath = NULL,
+ size = NULL,
+ children = NULL,
+ parent = NULL,
+ type = NULL,
+
+ initialize = function(name, type, size)
+ {
+ self$name <- name
+ self$type <- type
+ self$size <- size
+ self$children <- list()
+ },
+
+ addChild = function(node)
+ {
+ self$children <- c(self$children, node)
+ node$setParent(self)
+ self
+ },
+
+ setParent = function(parent)
+ {
+ self$parent = parent
+ },
+
+ getChild = function(childName)
+ {
+ for(child in self$children)
+ {
+ if(childName == child$name)
+ return(child)
+ }
+
+ return(NULL)
+ },
+
+ hasChildren = function()
+ {
+ if(length(self$children) != 0)
+ return(TRUE)
+ else
+ return(FALSE)
+ },
+
+ getFirstChild = function()
+ {
+ if(!self$hasChildren())
+ return(NULL)
+ else
+ return(self$children[[1]])
+ }
+
+ ),
+
+ cloneable = FALSE
+)
commit a8c3d59c520942c14434cea310c2c8a2fe623a9c
Author: Fuad Muhic <fmuhic at capeannenterprises.com>
Date: Wed Dec 13 15:59:36 2017 +0100
Added ability to write to a collection file.
Arvados-DCO-1.1-Signed-off-by: Fuad Muhic <fmuhic at capeannenterprises.com>
diff --git a/sdk/R/.RData b/sdk/R/.RData
index 88c2ce7..4f88320 100644
Binary files a/sdk/R/.RData and b/sdk/R/.RData differ
diff --git a/sdk/R/DESCRIPTION b/sdk/R/DESCRIPTION
index b0343c7..1b07dff 100644
--- a/sdk/R/DESCRIPTION
+++ b/sdk/R/DESCRIPTION
@@ -2,7 +2,7 @@ Package: ArvadosSDK
Type: Package
Title: What the Package Does (Title Case)
Version: 0.1.0
-Author: Who wrote it
+Author: Fuad Muhic
Maintainer: The package maintainer <yourself at somewhere.net>
Description: More about what it does (maybe more than one line)
Use four spaces when indenting paragraphs within the Description.
diff --git a/sdk/R/R/ArvadosFile.R b/sdk/R/R/ArvadosFile.R
index ce3a6ff..f13bf57 100644
--- a/sdk/R/R/ArvadosFile.R
+++ b/sdk/R/R/ArvadosFile.R
@@ -37,15 +37,47 @@ ArvadosFile <- R6::R6Class(
range = paste0(range, offset + length - 1)
fileURL = paste0(private$api$getWebDavHostName(), "c=", private$collection$uuid, "/", private$relativePath);
+ print(fileURL)
headers <- list(Authorization = paste("OAuth2", private$api$getToken()),
Range = range)
- #TODO(Fudo): Move this to HttpRequest.R
- # serverResponse <- httr::GET(url = fileURL,
- # config = httr::add_headers(unlist(headers)))
serverResponse <- private$http$GET(fileURL, headers)
+
+ if(serverResponse$status_code != 206)
+ stop(paste("Server code:", serverResponse$status_code))
+
+ collection
parsed_response <- httr::content(serverResponse, "raw")
+ },
+
+ write = function(content, contentType)
+ {
+ fileURL = paste0(private$api$getWebDavHostName(), "c=", private$collection$uuid, "/", private$relativePath);
+ headers <- list(Authorization = paste("OAuth2", private$api$getToken()),
+ "Content-Type" = contentType)
+ body <- content
+
+ serverResponse <- private$http$PUT(fileURL, headers, body)
+
+ if(serverResponse$status_code != 201)
+ stop(paste("Server code:", serverResponse$status_code))
+
+ #Note(Fudo): Everything went well we need to update file size
+ # in collection tree.
+
+ #Todo(Fudo): Move this into HttpRequest
+ uri <- URLencode(paste0(private$api$getWebDavHostName(), "c=", private$collection$uuid))
+ h <- curl::new_handle()
+ curl::handle_setopt(h, customrequest = "PROPFIND")
+
+ curl::handle_setheaders(h, "Authorization" = paste("OAuth2", private$api$getToken()))
+ propfindResponse <- curl::curl_fetch_memory(fileURL, h)
+
+ fileInfo <- private$httpParser$parseWebDAVResponse(propfindResponse, uri)
+ private$size <- fileInfo[[1]]$fileSize
+ private$collection$update(self, "File size changed")
+ #parsed_response <- httr::content(serverResponse, "text")
}
),
diff --git a/sdk/R/R/Collection.R b/sdk/R/R/Collection.R
index 14371a6..cecf657 100644
--- a/sdk/R/R/Collection.R
+++ b/sdk/R/R/Collection.R
@@ -61,7 +61,6 @@ Collection <- R6::R6Class(
self$trash_at <- result$trash_at
self$is_trashed <- result$is_trashed
- #Todo(Fudo): Replace this when you get access to webDAV server.
private$fileItems <- private$getCollectionContent()
private$fileTree <- private$generateTree(private$fileItems)
@@ -98,6 +97,16 @@ Collection <- R6::R6Class(
{
return(NULL)
}
+ },
+
+ update = function(subcollection, event)
+ {
+ #Todo(Fudo): Add some king of check here later on.
+ if(event == "File size changed")
+ {
+ private$handleFileSizeChange(subcollection$getRelativePath(),
+ subcollection$getSizeInBytes())
+ }
}
),
@@ -119,6 +128,14 @@ Collection <- R6::R6Class(
api = NULL,
fileTree = NULL,
+ handleFileSizeChange = function(filePath, newSize)
+ {
+ print(paste(filePath, newSize))
+
+ node <- private$getNode(filePath)
+ node$size <- newSize
+ },
+
createSubcollectionTree = function(treeNode)
{
if(treeNode$hasChildren())
@@ -144,18 +161,17 @@ Collection <- R6::R6Class(
getCollectionContent = function()
{
- #TODO(Fudo): Use proper URL here.
uri <- URLencode(paste0(private$api$getWebDavHostName(), "c=", self$uuid))
# fetch directory listing via curl and parse XML response
h <- curl::new_handle()
curl::handle_setopt(h, customrequest = "PROPFIND")
- #TODO(Fudo): Use proper token here.
curl::handle_setheaders(h, "Authorization" = paste("OAuth2", private$api$getToken()))
response <- curl::curl_fetch_memory(uri, h)
- HttpParser$new()$parseWebDAVResponse(response, uri)
+ parsedResponse <- HttpParser$new()$parseWebDAVResponse(response, uri)
+ parsedResponse[-1]
},
#Todo(Fudo): Move tree creation to another file.
@@ -241,6 +257,25 @@ Collection <- R6::R6Class(
{
return(predicate(node))
}
+ },
+
+ getNode = function(relativePathToNode)
+ {
+ treeBranches <- sapply(relativePathToNode, function(filePath)
+ {
+ splitPath <- unlist(strsplit(filePath, "/", fixed = TRUE))
+
+ node = private$fileTree
+ for(pathFragment in splitPath)
+ {
+ child = node$getChild(pathFragment)
+ if(is.null(child))
+ stop("Subcollection/ArvadosFile you are looking for doesn't exist.")
+ node = child
+ }
+
+ node
+ })
}
),
@@ -314,7 +349,7 @@ TreeNode <- R6::R6Class(
if(self$type == "folder")
print(paste0(indentation, self$name, "/"))
else
- print(paste0(indentation, self$size))
+ print(paste0(indentation, self$name))
for(child in self$children)
child$printContent(depth + 1)
diff --git a/sdk/R/R/HttpParser.R b/sdk/R/R/HttpParser.R
index 9129bbc..faa22f0 100644
--- a/sdk/R/R/HttpParser.R
+++ b/sdk/R/R/HttpParser.R
@@ -24,16 +24,16 @@ HttpParser <- R6::R6Class(
base <- paste(paste("/", strsplit(uri, "/")[[1]][-1:-3], sep="", collapse=""), "/", sep="")
result <- XML::xpathApply(doc, "//D:response", function(node) {
result = list()
- children = xmlChildren(node)
+ children = XML::xmlChildren(node)
result$name = sub(base, "", URLdecode(XML::xmlValue(children$href)), fixed=TRUE)
- sizeXMLNode = xmlChildren(xmlChildren(children$propstat)$prop)$getcontentlength
- result$fileSize = as.numeric(xmlValue(sizeXMLNode))
+ sizeXMLNode = XML::xmlChildren(XML::xmlChildren(children$propstat)$prop)$getcontentlength
+ result$fileSize = as.numeric(XML::xmlValue(sizeXMLNode))
result
})
- result[-1]
+ result
}
)
)
diff --git a/sdk/R/R/HttpRequest.R b/sdk/R/R/HttpRequest.R
index ea46bea..6dd8e4c 100644
--- a/sdk/R/R/HttpRequest.R
+++ b/sdk/R/R/HttpRequest.R
@@ -55,7 +55,7 @@ HttpRequest <- R6::R6Class(
# Python array from R list (recursion?)
createQuery = function(filters, limit, offset)
{
- finalQuery <- "?alt=json"
+ finalQuery <- NULL
if(!is.null(filters))
{
@@ -95,10 +95,12 @@ HttpRequest <- R6::R6Class(
encodedQuery <- URLencode(filters, reserved = T, repeated = T)
- finalQuery <- paste0(finalQuery, "&filters=", encodedQuery)
-
#Todo(Fudo): This is a hack for now. Find a proper solution.
- finalQuery <- stringr::str_replace_all(finalQuery, "%2B", "+")
+ encodedQuery <- stringr::str_replace_all(encodedQuery, "%2B", "+")
+
+ finalQuery <- c(finalQuery, paste0("filters=", encodedQuery))
+
+ finalQuery
}
if(!is.null(limit))
@@ -106,7 +108,7 @@ HttpRequest <- R6::R6Class(
if(!is.numeric(limit))
stop("Limit must be a numeric type.")
- finalQuery <- paste0(finalQuery, "&limit=", limit)
+ finalQuery <- c(finalQuery, paste0("limit=", limit))
}
if(!is.null(offset))
@@ -114,7 +116,13 @@ HttpRequest <- R6::R6Class(
if(!is.numeric(offset))
stop("Offset must be a numeric type.")
- finalQuery <- paste0(finalQuery, "&offset=", offset)
+ finalQuery <- c(finalQuery, paste0("offset=", offset))
+ }
+
+ if(length(finalQuery) > 1)
+ {
+ finalQuery <- paste0(finalQuery, collapse = "&")
+ finalQuery <- paste0("?", finalQuery)
}
finalQuery
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list