[ARVADOS] updated: f38cea6fa1ac48dd50789d9e2ec653b6d961c461
Git user
git at public.curoverse.com
Wed Oct 25 11:22:36 EDT 2017
Summary of changes:
tools/arv-sync-groups/arv-sync-groups.go | 164 ++++++++++++++++++++-----------
1 file changed, 105 insertions(+), 59 deletions(-)
via f38cea6fa1ac48dd50789d9e2ec653b6d961c461 (commit)
via 26b510d785df7f548f41a11445c07df34b60328c (commit)
from 3de17b282e8fc732862bd028c6d63ecfe70075af (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f38cea6fa1ac48dd50789d9e2ec653b6d961c461
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Oct 25 12:21:15 2017 -0300
12018: CSV file processing code separated to its own function.
Fixed members permission link, replaced "manage" with "can_write".
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index 87f89b6..56541d3 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -304,32 +304,57 @@ func doMain() error {
}
log.Printf("Found %d remote groups", len(remoteGroups))
- groupsCreated := 0
- membershipsAdded := 0
membershipsRemoved := 0
- membershipsSkipped := 0
// Read the CSV file
+ groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(&cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers)
+ if err != nil {
+ return err
+ }
+
+ // Remove previous members not listed on this run
+ for groupUUID := range remoteGroups {
+ gi := remoteGroups[groupUUID]
+ evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers)
+ groupName := gi.Group.Name
+ if len(evictedMembers) > 0 {
+ log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
+ }
+ for evictedUser := range evictedMembers {
+ if err := RemoveMemberFromGroup(&cfg, allUsers[evictedUser], gi.Group); err != nil {
+ return err
+ }
+ membershipsRemoved++
+ }
+ }
+ log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped)
+
+ return nil
+}
+
+// ProcessFile reads the CSV file and process every record
+func ProcessFile(cfg *ConfigParams, f *os.File, userIDToUUID map[string]string, groupNameToUUID map[string]string, remoteGroups map[string]*GroupInfo, allUsers map[string]arvados.User) (groupsCreated, membersAdded, membersSkipped int, err error) {
csvReader := csv.NewReader(f)
for {
- record, err := csvReader.Read()
- if err == io.EOF {
+ record, e := csvReader.Read()
+ if e == io.EOF {
break
}
- if err != nil {
- return fmt.Errorf("error reading %q: %s", cfg.Path, err)
+ if e != nil {
+ err = fmt.Errorf("error reading %q: %s", cfg.Path, err)
+ return
}
groupName := strings.TrimSpace(record[0])
groupMember := strings.TrimSpace(record[1]) // User ID (username or email)
if groupName == "" || groupMember == "" {
log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember)
- membershipsSkipped++
+ membersSkipped++
continue
}
if _, found := userIDToUUID[groupMember]; !found {
// User not present on the system, skip.
log.Printf("Warning: there's no user with %s %q on the system, skipping.", cfg.UserID, groupMember)
- membershipsSkipped++
+ membersSkipped++
continue
}
if _, found := groupNameToUUID[groupName]; !found {
@@ -342,8 +367,9 @@ func doMain() error {
"name": groupName,
"owner_uuid": cfg.ParentGroupUUID,
}
- if err := CreateGroup(&cfg, &newGroup, groupData); err != nil {
- return fmt.Errorf("error creating group named %q: %s", groupName, err)
+ if e := CreateGroup(cfg, &newGroup, groupData); e != nil {
+ err = fmt.Errorf("error creating group named %q: %s", groupName, err)
+ return
}
// Update cached group data
groupNameToUUID[groupName] = newGroup.UUID
@@ -362,32 +388,15 @@ func doMain() error {
log.Printf("Adding %q to group %q", groupMember, groupName)
}
// User wasn't a member, but should be.
- if err := AddMemberToGroup(&cfg, allUsers[userIDToUUID[groupMember]], gi.Group); err != nil {
- return nil
+ if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group); e != nil {
+ err = e
+ return
}
- membershipsAdded++
+ membersAdded++
}
gi.CurrentMembers[groupMember] = true
}
-
- // Remove previous members not listed on this run
- for groupUUID := range remoteGroups {
- gi := remoteGroups[groupUUID]
- evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers)
- groupName := gi.Group.Name
- if len(evictedMembers) > 0 {
- log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
- }
- for evictedUser := range evictedMembers {
- if err := RemoveMemberFromGroup(&cfg, allUsers[evictedUser], gi.Group); err != nil {
- return err
- }
- membershipsRemoved++
- }
- }
- log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped)
-
- return nil
+ return
}
// GetAll : Adds all objects of type 'resource' to the 'allItems' list
@@ -488,7 +497,7 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot
}, {
Attr: "name",
Operator: "=",
- Operand: "manage",
+ Operand: "can_write",
}, {
Attr: "head_uuid",
Operator: "=",
@@ -505,7 +514,7 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot
}
u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{})
if err != nil {
- return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (manage) links for group %q: %s", group.Name, err)
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_write) links for group %q: %s", group.Name, err)
}
// Build a list of user ids (email or username) belonging to this group
membersSet := make(map[string]bool)
@@ -618,13 +627,13 @@ func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group)
linkData = map[string]string{
"owner_uuid": cfg.SysUserUUID,
"link_class": "permission",
- "name": "manage",
+ "name": "can_write",
"tail_uuid": user.UUID,
"head_uuid": group.UUID,
}
if err := CreateLink(cfg, &newLink, linkData); err != nil {
userID, _ := GetUserID(user, cfg.UserID)
- return fmt.Errorf("error adding user %q -> group %q manage permission: %s", userID, group.Name, err)
+ return fmt.Errorf("error adding user %q -> group %q write permission: %s", userID, group.Name, err)
}
return nil
}
commit 26b510d785df7f548f41a11445c07df34b60328c
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Wed Oct 25 11:22:58 2017 -0300
12018: Splitted api client calls to their own funcs.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index 9a7daad..87f89b6 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -205,7 +205,7 @@ func SetParentGroup(cfg *ConfigParams) error {
"name": cfg.ParentGroupName,
"owner_uuid": cfg.SysUserUUID,
}
- if err := cfg.Client.RequestAndDecode(&parentGroup, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil); err != nil {
+ if err := CreateGroup(cfg, &parentGroup, groupData); err != nil {
return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err)
}
} else if len(gl.Items) == 1 {
@@ -219,7 +219,7 @@ func SetParentGroup(cfg *ConfigParams) error {
cfg.ParentGroupUUID = parentGroup.UUID
} else {
// UUID provided. Check if exists and if it's owned by system user
- if err := cfg.Client.RequestAndDecode(&parentGroup, "GET", "/arvados/v1/groups/"+cfg.ParentGroupUUID, nil, nil); err != nil {
+ if err := GetGroup(cfg, &parentGroup, cfg.ParentGroupUUID); err != nil {
return fmt.Errorf("error searching for parent group with UUID %q: %s", cfg.ParentGroupUUID, err)
}
if parentGroup.OwnerUUID != cfg.SysUserUUID {
@@ -342,7 +342,7 @@ func doMain() error {
"name": groupName,
"owner_uuid": cfg.ParentGroupUUID,
}
- if err := cfg.Client.RequestAndDecode(&newGroup, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil); err != nil {
+ if err := CreateGroup(&cfg, &newGroup, groupData); err != nil {
return fmt.Errorf("error creating group named %q: %s", groupName, err)
}
// Update cached group data
@@ -362,26 +362,8 @@ func doMain() error {
log.Printf("Adding %q to group %q", groupMember, groupName)
}
// User wasn't a member, but should be.
- var newLink Link
- linkData := map[string]string{
- "owner_uuid": cfg.SysUserUUID,
- "link_class": "permission",
- "name": "can_read",
- "tail_uuid": groupUUID,
- "head_uuid": userIDToUUID[groupMember],
- }
- if err := cfg.Client.RequestAndDecode(&newLink, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil); err != nil {
- return fmt.Errorf("error adding group %q -> user %q read permission: %s", groupName, groupMember, err)
- }
- linkData = map[string]string{
- "owner_uuid": cfg.SysUserUUID,
- "link_class": "permission",
- "name": "manage",
- "tail_uuid": userIDToUUID[groupMember],
- "head_uuid": groupUUID,
- }
- if err = cfg.Client.RequestAndDecode(&newLink, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil); err != nil {
- return fmt.Errorf("error adding user %q -> group %q manage permission: %s", groupMember, groupName, err)
+ if err := AddMemberToGroup(&cfg, allUsers[userIDToUUID[groupMember]], gi.Group); err != nil {
+ return nil
}
membershipsAdded++
}
@@ -416,7 +398,7 @@ func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, pa
params.Offset = 0
params.Order = "uuid"
for {
- if err = c.RequestAndDecode(&page, "GET", "/arvados/v1/"+res, nil, params); err != nil {
+ if err = GetResourceList(c, &page, res, params); err != nil {
return allItems, err
}
// Have we finished paging?
@@ -599,7 +581,8 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G
} {
l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{})
if err != nil {
- return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", user.Email, group.Name, err)
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", userID, group.Name, err)
}
for _, link := range l {
links = append(links, link)
@@ -610,9 +593,63 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G
if cfg.Verbose {
log.Printf("Removing permission link for %q on group %q", user.Email, group.Name)
}
- if err := cfg.Client.RequestAndDecode(&link, "DELETE", "/arvados/v1/links/"+link.UUID, nil, nil); err != nil {
- return fmt.Errorf("error removing user %q from group %q: %s", user.Email, group.Name, err)
+ if err := DeleteLink(cfg, link.UUID); err != nil {
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error removing user %q from group %q: %s", userID, group.Name, err)
}
}
return nil
}
+
+// AddMemberToGroup create membership links
+func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+ var newLink Link
+ linkData := map[string]string{
+ "owner_uuid": cfg.SysUserUUID,
+ "link_class": "permission",
+ "name": "can_read",
+ "tail_uuid": group.UUID,
+ "head_uuid": user.UUID,
+ }
+ if err := CreateLink(cfg, &newLink, linkData); err != nil {
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error adding group %q -> user %q read permission: %s", userID, user.Email, err)
+ }
+ linkData = map[string]string{
+ "owner_uuid": cfg.SysUserUUID,
+ "link_class": "permission",
+ "name": "manage",
+ "tail_uuid": user.UUID,
+ "head_uuid": group.UUID,
+ }
+ if err := CreateLink(cfg, &newLink, linkData); err != nil {
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error adding user %q -> group %q manage permission: %s", userID, group.Name, err)
+ }
+ return nil
+}
+
+// CreateGroup creates a group with groupData parameters, assigns it to dst
+func CreateGroup(cfg *ConfigParams, dst *arvados.Group, groupData map[string]string) error {
+ return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil)
+}
+
+// GetGroup fetches a group by its UUID
+func GetGroup(cfg *ConfigParams, dst *arvados.Group, groupUUID string) error {
+ return cfg.Client.RequestAndDecode(&dst, "GET", "/arvados/v1/groups/"+groupUUID, nil, nil)
+}
+
+// CreateLink creates a link with linkData parameters, assigns it to dst
+func CreateLink(cfg *ConfigParams, dst *Link, linkData map[string]string) error {
+ return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil)
+}
+
+// DeleteLink deletes a link by its UUID
+func DeleteLink(cfg *ConfigParams, linkUUID string) error {
+ return cfg.Client.RequestAndDecode(&Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil)
+}
+
+// GetResourceList fetches res list using params
+func GetResourceList(c *arvados.Client, dst *resourceList, res string, params interface{}) error {
+ return c.RequestAndDecode(dst, "GET", "/arvados/v1/"+res, nil, params)
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list