[ARVADOS] updated: eb772d62ecc9d1ad61df7bbf08a572fda689c5be
Git user
git at public.curoverse.com
Mon Oct 30 18:45:36 EDT 2017
Summary of changes:
sdk/go/arvados/link.go | 25 ++++++++
tools/arv-sync-groups/arv-sync-groups.go | 30 ++++------
tools/arv-sync-groups/arv-sync-groups_test.go | 86 +++++++++++++++++++++++++++
3 files changed, 121 insertions(+), 20 deletions(-)
create mode 100644 sdk/go/arvados/link.go
via eb772d62ecc9d1ad61df7bbf08a572fda689c5be (commit)
via 43d9a42f838e134b5f7f639375f1f0752292f694 (commit)
from 7327a28eb1d90b161708e9e4e855cf80f41f20ae (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit eb772d62ecc9d1ad61df7bbf08a572fda689c5be
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Oct 30 19:42:06 2017 -0300
12018: Enhanced error message when having a parse error.
Added test cleanup.
Added test to confirm records with empty fields are ignored.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index a1a73ca..40c6934 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -336,14 +336,16 @@ func doMain(cfg *ConfigParams) error {
// ProcessFile reads the CSV file and process every record
func ProcessFile(cfg *ConfigParams, f *os.File, userIDToUUID map[string]string, groupNameToUUID map[string]string, remoteGroups map[string]*GroupInfo, allUsers map[string]arvados.User) (groupsCreated, membersAdded, membersSkipped int, err error) {
+ lineNo := 0
csvReader := csv.NewReader(f)
for {
record, e := csvReader.Read()
if e == io.EOF {
break
}
+ lineNo++
if e != nil {
- err = fmt.Errorf("error reading %q: %s", cfg.Path, err)
+ err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo)
return
}
groupName := strings.TrimSpace(record[0])
diff --git a/tools/arv-sync-groups/arv-sync-groups_test.go b/tools/arv-sync-groups/arv-sync-groups_test.go
index d4dd854..e41ad88 100644
--- a/tools/arv-sync-groups/arv-sync-groups_test.go
+++ b/tools/arv-sync-groups/arv-sync-groups_test.go
@@ -75,6 +75,65 @@ func (s *TestSuite) SetUpSuite(c *C) {
c.Assert(len(s.users), Not(Equals), 0)
}
+// Clean any membership link and remote group created by the test
+func (s *TestSuite) TearDownTest(c *C) {
+ gl := arvados.GroupList{}
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "group_class",
+ Operator: "=",
+ Operand: "role",
+ }, {
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: s.cfg.ParentGroupUUID,
+ }},
+ }
+ err := s.cfg.Client.RequestAndDecode(&gl, "GET", "/arvados/v1/groups", nil, params)
+ c.Assert(err, IsNil)
+ for _, group := range gl.Items {
+ ll := arvados.LinkList{}
+ // Delete user->group links
+ params = arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }},
+ }
+ err = s.cfg.Client.RequestAndDecode(&ll, "GET", "/arvados/v1/links", nil, params)
+ c.Assert(err, IsNil)
+ for _, link := range ll.Items {
+ err = DeleteLink(s.cfg, link.UUID)
+ c.Assert(err, IsNil)
+ }
+ // Delete group->user links
+ params = arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }},
+ }
+ s.cfg.Client.RequestAndDecode(&ll, "GET", "/arvados/v1/links", nil, params)
+ for _, link := range ll.Items {
+ err = DeleteLink(s.cfg, link.UUID)
+ c.Assert(err, IsNil)
+ }
+ // Delete group
+ err = s.cfg.Client.RequestAndDecode(&arvados.Group{}, "DELETE", "/arvados/v1/groups/"+group.UUID, nil, nil)
+ c.Assert(err, IsNil)
+ }
+}
+
var _ = Suite(&TestSuite{})
// MakeTempCVSFile creates a temp file with data as comma separated values
@@ -341,3 +400,30 @@ func (s *TestSuite) TestIgnoreNonexistantUsers(c *C) {
c.Assert(groupUUID, Not(Equals), "")
c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
}
+
+// Users listed on the file that don't exist on the system are ignored
+func (s *TestSuite) TestIgnoreEmptyFields(c *C) {
+ activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
+ activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ // Confirm that group doesn't exist
+ groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup4")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ // Create file & run command
+ data := [][]string{
+ {"", activeUserEmail}, // Empty field
+ {"TestGroup5", ""}, // Empty field
+ {"TestGroup4", activeUserEmail},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Confirm that memberships exist
+ groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+}
commit 43d9a42f838e134b5f7f639375f1f0752292f694
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Oct 30 18:17:52 2017 -0300
12018: Moved Link & LinkList types to GoSDK arvados package
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/go/arvados/link.go b/sdk/go/arvados/link.go
new file mode 100644
index 0000000..dee1355
--- /dev/null
+++ b/sdk/go/arvados/link.go
@@ -0,0 +1,25 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+// Link is an arvados#link record
+type Link struct {
+ UUID string `json:"uuid,omiempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
+ Name string `json:"name,omitempty"`
+ LinkClass string `json:"link_class,omitempty"`
+ HeadUUID string `json:"head_uuid,omitempty"`
+ HeadKind string `json:"head_kind,omitempty"`
+ TailUUID string `json:"tail_uuid,omitempty"`
+ TailKind string `json:"tail_kind,omitempty"`
+}
+
+// UserList is an arvados#userList resource.
+type LinkList struct {
+ Items []Link `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
+}
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index 1e1306d..a1a73ca 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -79,21 +79,9 @@ func (l GroupList) GetItems() (out []interface{}) {
return
}
-// Link is an arvados#link record
-type Link struct {
- UUID string `json:"uuid,omiempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
- Name string `json:"name,omitempty"`
- LinkClass string `json:"link_class,omitempty"`
- HeadUUID string `json:"head_uuid,omitempty"`
- HeadKind string `json:"head_kind,omitempty"`
- TailUUID string `json:"tail_uuid,omitempty"`
- TailKind string `json:"tail_kind,omitempty"`
-}
-
// LinkList implements resourceList interface
type LinkList struct {
- Items []Link `json:"items"`
+ arvados.LinkList
}
// Len returns the amount of items this list holds
@@ -535,11 +523,11 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot
membersSet := make(map[string]bool)
u2gLinkSet := make(map[string]bool)
for _, l := range u2gLinks {
- linkedMemberUUID := l.(Link).TailUUID
+ linkedMemberUUID := l.(arvados.Link).TailUUID
u2gLinkSet[linkedMemberUUID] = true
}
for _, item := range g2uLinks {
- link := item.(Link)
+ link := item.(arvados.Link)
// We may have received an old link pointing to a removed account.
if _, found := allUsers[link.HeadUUID]; !found {
continue
@@ -613,7 +601,7 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G
}
}
for _, item := range links {
- link := item.(Link)
+ link := item.(arvados.Link)
userID, _ := GetUserID(user, cfg.UserID)
if cfg.Verbose {
log.Printf("Removing %q permission link for %q on group %q", link.Name, userID, group.Name)
@@ -627,7 +615,7 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G
// AddMemberToGroup create membership links
func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
- var newLink Link
+ var newLink arvados.Link
linkData := map[string]string{
"owner_uuid": cfg.SysUserUUID,
"link_class": "permission",
@@ -664,7 +652,7 @@ func GetGroup(cfg *ConfigParams, dst *arvados.Group, groupUUID string) error {
}
// CreateLink creates a link with linkData parameters, assigns it to dst
-func CreateLink(cfg *ConfigParams, dst *Link, linkData map[string]string) error {
+func CreateLink(cfg *ConfigParams, dst *arvados.Link, linkData map[string]string) error {
return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil)
}
@@ -673,7 +661,7 @@ func DeleteLink(cfg *ConfigParams, linkUUID string) error {
if linkUUID == "" {
return fmt.Errorf("cannot delete link with invalid UUID: %q", linkUUID)
}
- return cfg.Client.RequestAndDecode(&Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil)
+ return cfg.Client.RequestAndDecode(&arvados.Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil)
}
// GetResourceList fetches res list using params
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list