[ARVADOS] updated: a0dcf5e4b837826ff75b9785bd3f3e695ac7bdca
Git user
git at public.curoverse.com
Tue Oct 24 08:09:22 EDT 2017
Summary of changes:
sdk/go/arvados/group.go | 20 ++
sdk/go/arvados/user.go | 9 +
tools/arv-sync-groups/arv-sync-groups.go | 527 ++++++++++++++++---------------
3 files changed, 309 insertions(+), 247 deletions(-)
create mode 100644 sdk/go/arvados/group.go
via a0dcf5e4b837826ff75b9785bd3f3e695ac7bdca (commit)
via 96b24c1aabe0b9b475e8c743e548258775507481 (commit)
via 880351aef15100c0bae893174f47628b0100ba06 (commit)
via 904fb42f9e39872526ad14c89d3298afa7bd08d6 (commit)
via 92df2dab0bbb70c0b5ef99bac78d1a322b20648e (commit)
via e4e97733a237b45ea7202946c7fd20935bc47a2a (commit)
via c8decf3f2f88611166603ca48677470b478c06a6 (commit)
via b3667766f108542aa3d0e479e4a1179a47f2a653 (commit)
from ea10340803abade2d35212866fcbc1beb1acd533 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a0dcf5e4b837826ff75b9785bd3f3e695ac7bdca
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Tue Oct 24 09:07:04 2017 -0300
12018: User, UserList, Group & GroupList types go into Go SDK.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/sdk/go/arvados/group.go b/sdk/go/arvados/group.go
new file mode 100644
index 0000000..b00809f
--- /dev/null
+++ b/sdk/go/arvados/group.go
@@ -0,0 +1,20 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+// Group is an arvados#group record
+type Group struct {
+ UUID string `json:"uuid,omitempty"`
+ Name string `json:"name,omitempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
+}
+
+// GroupList is an arvados#groupList resource.
+type GroupList struct {
+ Items []Group `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
+}
diff --git a/sdk/go/arvados/user.go b/sdk/go/arvados/user.go
index 38d2edd..3a36e5e 100644
--- a/sdk/go/arvados/user.go
+++ b/sdk/go/arvados/user.go
@@ -10,6 +10,15 @@ type User struct {
IsActive bool `json:"is_active"`
IsAdmin bool `json:"is_admin"`
Username string `json:"username,omitempty"`
+ Email string `json:"email,omitempty"`
+}
+
+// UserList is an arvados#userList resource.
+type UserList struct {
+ Items []User `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
}
// CurrentUser calls arvados.v1.users.current, and returns the User
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index ea863a3..87cee51 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -25,18 +25,13 @@ type resourceList interface {
}
type groupInfo struct {
- Group Group
+ Group arvados.Group
PreviousMembers map[string]bool
CurrentMembers map[string]bool
}
-type user struct {
- UUID string `json:"uuid,omitempty"`
- Email string `json:"email,omitempty"`
- Username string `json:"username,omitempty"`
-}
-
-func (u user) GetID(idSelector string) (string, error) {
+// GetUserID returns the correct user id value depending on the selector
+func GetUserID(u arvados.User, idSelector string) (string, error) {
switch idSelector {
case "email":
return u.Email, nil
@@ -47,32 +42,27 @@ func (u user) GetID(idSelector string) (string, error) {
}
}
-// userList implements resourceList interface
-type userList struct {
- Items []user `json:"items"`
+// UserList implements resourceList interface
+type UserList struct {
+ arvados.UserList
}
-func (l userList) Len() int {
+// Len returns the amount of items this list holds
+func (l UserList) Len() int {
return len(l.Items)
}
-func (l userList) GetItems() (out []interface{}) {
+// GetItems returns the list of items
+func (l UserList) GetItems() (out []interface{}) {
for _, item := range l.Items {
out = append(out, item)
}
return
}
-// Group is an arvados#group record
-type Group struct {
- UUID string `json:"uuid,omitempty"`
- Name string `json:"name,omitempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
-}
-
// GroupList implements resourceList interface
type GroupList struct {
- Items []Group `json:"items"`
+ arvados.GroupList
}
// Len returns the amount of items this list holds
@@ -141,7 +131,7 @@ func doMain() error {
userID := flags.String(
"user-id",
"email",
- "Attribute by which every user is identified. Valid values are: email (the default) and username.")
+ "Attribute by which every user is identified. Valid values are: email and username.")
verbose := flags.Bool(
"verbose",
false,
@@ -187,7 +177,7 @@ func doMain() error {
sysUserUUID := u.UUID[:12] + "000000000000000"
// Find/create parent group
- var parentGroup Group
+ var parentGroup arvados.Group
if *parentGroupUUID == "" {
// UUID not provided, search for preexisting parent group
var gl GroupList
@@ -238,17 +228,17 @@ func doMain() error {
log.Printf("Group sync starting. Using %q as users id and parent group UUID %q", *userID, parentGroup.UUID)
// Get the complete user list to minimize API Server requests
- allUsers := make(map[string]user)
+ allUsers := make(map[string]arvados.User)
userIDToUUID := make(map[string]string) // Index by email or username
- results, err := ListAll(ac, "users", arvados.ResourceListParams{}, &userList{})
+ results, err := ListAll(ac, "users", arvados.ResourceListParams{}, &UserList{})
if err != nil {
return fmt.Errorf("error getting user list: %s", err)
}
log.Printf("Found %d users", len(results))
for _, item := range results {
- u := item.(user)
+ u := item.(arvados.User)
allUsers[u.UUID] = u
- uID, err := u.GetID(*userID)
+ uID, err := GetUserID(u, *userID)
if err != nil {
return err
}
@@ -273,7 +263,7 @@ func doMain() error {
return fmt.Errorf("error getting remote groups: %s", err)
}
for _, item := range results {
- group := item.(Group)
+ group := item.(arvados.Group)
// Group -> User filter
g2uFilter := arvados.ResourceListParams{
Filters: []arvados.Filter{{
@@ -349,7 +339,7 @@ func doMain() error {
if _, found := u2gLinkSet[link.HeadUUID]; !found {
continue
}
- memberID, err := allUsers[link.HeadUUID].GetID(*userID)
+ memberID, err := GetUserID(allUsers[link.HeadUUID], *userID)
if err != nil {
return err
}
@@ -396,7 +386,7 @@ func doMain() error {
if *verbose {
log.Printf("Remote group %q not found, creating...", groupName)
}
- var newGroup Group
+ var newGroup arvados.Group
groupData := map[string]string{
"name": groupName,
"owner_uuid": parentGroup.UUID,
commit 96b24c1aabe0b9b475e8c743e548258775507481
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Oct 23 20:56:13 2017 -0300
12018: Group membership need two-way links between the user and
the group. On group members loading, check that both links exist.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index 9420a94..ea863a3 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -274,7 +274,8 @@ func doMain() error {
}
for _, item := range results {
group := item.(Group)
- params := arvados.ResourceListParams{
+ // Group -> User filter
+ g2uFilter := arvados.ResourceListParams{
Filters: []arvados.Filter{{
Attr: "owner_uuid",
Operator: "=",
@@ -297,18 +298,57 @@ func doMain() error {
Operand: "arvados#user",
}},
}
- results, err := ListAll(ac, "links", params, &linkList{})
+ // User -> Group filter
+ u2gFilter := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: sysUserUUID,
+ }, {
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "name",
+ Operator: "=",
+ Operand: "manage",
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "tail_kind",
+ Operator: "=",
+ Operand: "arvados#user",
+ }},
+ }
+ g2uLinks, err := ListAll(ac, "links", g2uFilter, &linkList{})
+ if err != nil {
+ return fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err)
+ }
+ u2gLinks, err := ListAll(ac, "links", u2gFilter, &linkList{})
if err != nil {
- return fmt.Errorf("error getting member links for group %q: %s", group.Name, err)
+ return fmt.Errorf("error getting member (manage) links for group %q: %s", group.Name, err)
}
// Build a list of user ids (email or username) belonging to this group
membersSet := make(map[string]bool)
- for _, item := range results {
+ u2gLinkSet := make(map[string]bool)
+ for _, l := range u2gLinks {
+ linkedMemberUUID := l.(Link).TailUUID
+ u2gLinkSet[linkedMemberUUID] = true
+ }
+ for _, item := range g2uLinks {
link := item.(Link)
- // We may receive an old link pointing to a removed user
+ // We may have received an old link pointing to a removed account.
if _, found := allUsers[link.HeadUUID]; !found {
continue
}
+ // The matching User -> Group link may not exist if the link
+ // creation failed on a previous run. If that's the case, don't
+ // include this account on the previous members list.
+ if _, found := u2gLinkSet[link.HeadUUID]; !found {
+ continue
+ }
memberID, err := allUsers[link.HeadUUID].GetID(*userID)
if err != nil {
return err
@@ -341,7 +381,7 @@ func doMain() error {
groupName := strings.TrimSpace(record[0])
groupMember := strings.TrimSpace(record[1]) // User ID (username or email)
if groupName == "" || groupMember == "" {
- log.Printf("Warning: CSV record has at least one field empty (%s, %s). Skipping", groupName, groupMember)
+ log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember)
membershipsSkipped++
continue
}
@@ -380,7 +420,7 @@ func doMain() error {
if *verbose {
log.Printf("Adding %q to group %q", groupMember, groupName)
}
- // User wasn't a member, but should.
+ // User wasn't a member, but should be.
var newLink Link
linkData := map[string]string{
"owner_uuid": sysUserUUID,
commit 880351aef15100c0bae893174f47628b0100ba06
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Oct 23 19:30:54 2017 -0300
12018: Do not tag remote groups with tag links. Use the parent
group as an indication that the group is synced from a remote
source.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index 8027479..9420a94 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -25,7 +25,7 @@ type resourceList interface {
}
type groupInfo struct {
- Group group
+ Group Group
PreviousMembers map[string]bool
CurrentMembers map[string]bool
}
@@ -63,29 +63,33 @@ func (l userList) GetItems() (out []interface{}) {
return
}
-type group struct {
+// Group is an arvados#group record
+type Group struct {
UUID string `json:"uuid,omitempty"`
Name string `json:"name,omitempty"`
OwnerUUID string `json:"owner_uuid,omitempty"`
}
-// groupList implements resourceList interface
-type groupList struct {
- Items []group `json:"items"`
+// GroupList implements resourceList interface
+type GroupList struct {
+ Items []Group `json:"items"`
}
-func (l groupList) Len() int {
+// Len returns the amount of items this list holds
+func (l GroupList) Len() int {
return len(l.Items)
}
-func (l groupList) GetItems() (out []interface{}) {
+// GetItems returns the list of items
+func (l GroupList) GetItems() (out []interface{}) {
for _, item := range l.Items {
out = append(out, item)
}
return
}
-type link struct {
+// Link is an arvados#link record
+type Link struct {
UUID string `json:"uuid,omiempty"`
OwnerUUID string `json:"owner_uuid,omitempty"`
Name string `json:"name,omitempty"`
@@ -96,15 +100,17 @@ type link struct {
TailKind string `json:"tail_kind,omitempty"`
}
-// linkList implements resourceList interface
+// LinkList implements resourceList interface
type linkList struct {
- Items []link `json:"items"`
+ Items []Link `json:"items"`
}
+// Len returns the amount of items this list holds
func (l linkList) Len() int {
return len(l.Items)
}
+// GetItems returns the list of items
func (l linkList) GetItems() (out []interface{}) {
for _, item := range l.Items {
out = append(out, item)
@@ -119,7 +125,6 @@ func main() {
}
func doMain() error {
- const groupTag string = "remote_group"
const remoteGroupParentName string = "Externally synchronized groups"
// Acceptable attributes to identify a user on the CSV file
userIDOpts := map[string]bool{
@@ -182,10 +187,10 @@ func doMain() error {
sysUserUUID := u.UUID[:12] + "000000000000000"
// Find/create parent group
- var parentGroup group
+ var parentGroup Group
if *parentGroupUUID == "" {
// UUID not provided, search for preexisting parent group
- var gl groupList
+ var gl GroupList
params := arvados.ResourceListParams{
Filters: []arvados.Filter{{
Attr: "name",
@@ -253,59 +258,22 @@ func doMain() error {
}
}
- // Request all UUIDs for groups tagged as remote
- remoteGroupUUIDs := make(map[string]bool)
- params := arvados.ResourceListParams{
- Filters: []arvados.Filter{{
- Attr: "owner_uuid",
- Operator: "=",
- Operand: sysUserUUID,
- }, {
- Attr: "link_class",
- Operator: "=",
- Operand: "tag",
- }, {
- Attr: "name",
- Operator: "=",
- Operand: groupTag,
- }, {
- Attr: "head_kind",
- Operator: "=",
- Operand: "arvados#group",
- }},
- }
- results, err = ListAll(ac, "links", params, &linkList{})
- if err != nil {
- return fmt.Errorf("error getting remote group UUIDs: %s", err)
- }
- for _, item := range results {
- link := item.(link)
- remoteGroupUUIDs[link.HeadUUID] = true
- }
// Get remote groups and their members
- var uuidList []string
- for uuid := range remoteGroupUUIDs {
- uuidList = append(uuidList, uuid)
- }
remoteGroups := make(map[string]*groupInfo)
groupNameToUUID := make(map[string]string) // Index by group name
- params = arvados.ResourceListParams{
+ params := arvados.ResourceListParams{
Filters: []arvados.Filter{{
- Attr: "uuid",
- Operator: "in",
- Operand: uuidList,
- }, {
Attr: "owner_uuid",
Operator: "=",
Operand: parentGroup.UUID,
}},
}
- results, err = ListAll(ac, "groups", params, &groupList{})
+ results, err = ListAll(ac, "groups", params, &GroupList{})
if err != nil {
- return fmt.Errorf("error getting remote groups by UUID: %s", err)
+ return fmt.Errorf("error getting remote groups: %s", err)
}
for _, item := range results {
- group := item.(group)
+ group := item.(Group)
params := arvados.ResourceListParams{
Filters: []arvados.Filter{{
Attr: "owner_uuid",
@@ -324,7 +292,7 @@ func doMain() error {
Operator: "=",
Operand: group.UUID,
}, {
- Attr: "head_uuid",
+ Attr: "head_kind",
Operator: "=",
Operand: "arvados#user",
}},
@@ -336,7 +304,7 @@ func doMain() error {
// Build a list of user ids (email or username) belonging to this group
membersSet := make(map[string]bool)
for _, item := range results {
- link := item.(link)
+ link := item.(Link)
// We may receive an old link pointing to a removed user
if _, found := allUsers[link.HeadUUID]; !found {
continue
@@ -384,11 +352,11 @@ func doMain() error {
continue
}
if _, found := groupNameToUUID[groupName]; !found {
- // Group doesn't exist, create and tag it before continuing
+ // Group doesn't exist, create it before continuing
if *verbose {
log.Printf("Remote group %q not found, creating...", groupName)
}
- var newGroup group
+ var newGroup Group
groupData := map[string]string{
"name": groupName,
"owner_uuid": parentGroup.UUID,
@@ -396,16 +364,6 @@ func doMain() error {
if err := ac.RequestAndDecode(&newGroup, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil); err != nil {
return fmt.Errorf("error creating group named %q: %s", groupName, err)
}
- var newLink link
- linkData := map[string]string{
- "owner_uuid": sysUserUUID,
- "link_class": "tag",
- "name": groupTag,
- "head_uuid": newGroup.UUID,
- }
- if err = ac.RequestAndDecode(&newLink, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil); err != nil {
- return fmt.Errorf("error creating tag for newly created group %q (%s): %s", newGroup.Name, newGroup.UUID, err)
- }
// Update cached group data
groupNameToUUID[groupName] = newGroup.UUID
remoteGroups[newGroup.UUID] = &groupInfo{
@@ -423,7 +381,7 @@ func doMain() error {
log.Printf("Adding %q to group %q", groupMember, groupName)
}
// User wasn't a member, but should.
- var newLink link
+ var newLink Link
linkData := map[string]string{
"owner_uuid": sysUserUUID,
"link_class": "permission",
@@ -502,11 +460,11 @@ func doMain() error {
}
}
for _, item := range links {
- link := item.(link)
+ link := item.(Link)
if *verbose {
- log.Printf("Removing %q from group %q", evictedUser, gi.Group.Name)
+ log.Printf("Removing permission link for %q on group %q", evictedUser, gi.Group.Name)
}
- if err := ac.RequestAndDecode(&link, "DELETE", "/arvados/v1/links"+link.UUID, nil, nil); err != nil {
+ if err := ac.RequestAndDecode(&link, "DELETE", "/arvados/v1/links/"+link.UUID, nil, nil); err != nil {
return fmt.Errorf("error removing user %q from group %q: %s", evictedUser, groupName, err)
}
}
commit 904fb42f9e39872526ad14c89d3298afa7bd08d6
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Mon Oct 23 16:36:54 2017 -0300
12018: Stop using arvadosclient in favor of arvados package for API
server access.
Drop "-retries" parameter as it's not supported on the newer SDK.
Use specific types for resources instead of [][]string.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index a40dcfc..8027479 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -5,16 +5,18 @@
package main
import (
+ "bytes"
"encoding/csv"
+ "encoding/json"
"flag"
"fmt"
"io"
"log"
+ "net/url"
"os"
"strings"
"git.curoverse.com/arvados.git/sdk/go/arvados"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
)
type resourceList interface {
@@ -84,7 +86,8 @@ func (l groupList) GetItems() (out []interface{}) {
}
type link struct {
- UUID string `json:"uuid, omiempty"`
+ UUID string `json:"uuid,omiempty"`
+ OwnerUUID string `json:"owner_uuid,omitempty"`
Name string `json:"name,omitempty"`
LinkClass string `json:"link_class,omitempty"`
HeadUUID string `json:"head_uuid,omitempty"`
@@ -138,10 +141,6 @@ func doMain() error {
"verbose",
false,
"Log informational messages. Off by default.")
- retries := flags.Int(
- "retries",
- 3,
- "Maximum number of times to retry server requests that encounter temporary failures (e.g., server down). Default 3.")
parentGroupUUID := flags.String(
"parent-group-uuid",
"",
@@ -151,9 +150,6 @@ func doMain() error {
flags.Parse(os.Args[1:])
// Validations
- if *retries < 0 {
- return fmt.Errorf("retry quantity must be >= 0")
- }
if *srcPath == "" {
return fmt.Errorf("please provide a path to an input file")
}
@@ -174,11 +170,6 @@ func doMain() error {
// Arvados Client setup
ac := arvados.NewClientFromEnv()
- arv, err := arvadosclient.New(ac)
- if err != nil {
- return fmt.Errorf("error setting up arvados client %s", err)
- }
- arv.Retries = *retries
// Check current user permissions & get System user's UUID
u, err := ac.CurrentUser()
@@ -195,11 +186,18 @@ func doMain() error {
if *parentGroupUUID == "" {
// UUID not provided, search for preexisting parent group
var gl groupList
- if err := arv.List("groups", arvadosclient.Dict{
- "filters": [][]string{
- {"name", "=", remoteGroupParentName},
- {"owner_uuid", "=", sysUserUUID}},
- }, &gl); err != nil {
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "name",
+ Operator: "=",
+ Operand: remoteGroupParentName,
+ }, {
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: sysUserUUID,
+ }},
+ }
+ if err := ac.RequestAndDecode(&gl, "GET", "/arvados/v1/groups", nil, params); err != nil {
return fmt.Errorf("error searching for parent group: %s", err)
}
if len(gl.Items) == 0 {
@@ -207,12 +205,12 @@ func doMain() error {
if *verbose {
log.Println("Default parent group not found, creating...")
}
- if err := arv.Create("groups", arvadosclient.Dict{
- "group": arvadosclient.Dict{
- "name": remoteGroupParentName,
- "owner_uuid": sysUserUUID},
- }, &parentGroup); err != nil {
- return fmt.Errorf("error creating system user owned group named %q: %s", remoteGroupParentName, err)
+ groupData := map[string]string{
+ "name": remoteGroupParentName,
+ "owner_uuid": sysUserUUID,
+ }
+ if err := ac.RequestAndDecode(&parentGroup, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil); err != nil {
+ return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err)
}
} else if len(gl.Items) == 1 {
// Default parent group found.
@@ -220,11 +218,11 @@ func doMain() error {
} else {
// This should never happen, as there's an unique index for
// (owner_uuid, name) on groups.
- return fmt.Errorf("found %d groups owned by system user and named %q", len(gl.Items), remoteGroupParentName)
+ return fmt.Errorf("bug: found %d groups owned by system user and named %q", len(gl.Items), remoteGroupParentName)
}
} else {
// UUID provided. Check if exists and if it's owned by system user
- if err := arv.Get("groups", *parentGroupUUID, arvadosclient.Dict{}, &parentGroup); err != nil {
+ if err := ac.RequestAndDecode(&parentGroup, "GET", "/arvados/v1/groups/"+*parentGroupUUID, nil, nil); err != nil {
return fmt.Errorf("error searching for parent group with UUID %q: %s", *parentGroupUUID, err)
}
if parentGroup.OwnerUUID != sysUserUUID {
@@ -237,7 +235,7 @@ func doMain() error {
// Get the complete user list to minimize API Server requests
allUsers := make(map[string]user)
userIDToUUID := make(map[string]string) // Index by email or username
- results, err := ListAll(arv, "users", arvadosclient.Dict{}, &userList{})
+ results, err := ListAll(ac, "users", arvados.ResourceListParams{}, &userList{})
if err != nil {
return fmt.Errorf("error getting user list: %s", err)
}
@@ -257,14 +255,26 @@ func doMain() error {
// Request all UUIDs for groups tagged as remote
remoteGroupUUIDs := make(map[string]bool)
- results, err = ListAll(arv, "links", arvadosclient.Dict{
- "filters": [][]string{
- {"owner_uuid", "=", sysUserUUID},
- {"link_class", "=", "tag"},
- {"name", "=", groupTag},
- {"head_kind", "=", "arvados#group"},
- },
- }, &linkList{})
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: sysUserUUID,
+ }, {
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "tag",
+ }, {
+ Attr: "name",
+ Operator: "=",
+ Operand: groupTag,
+ }, {
+ Attr: "head_kind",
+ Operator: "=",
+ Operand: "arvados#group",
+ }},
+ }
+ results, err = ListAll(ac, "links", params, &linkList{})
if err != nil {
return fmt.Errorf("error getting remote group UUIDs: %s", err)
}
@@ -279,26 +289,47 @@ func doMain() error {
}
remoteGroups := make(map[string]*groupInfo)
groupNameToUUID := make(map[string]string) // Index by group name
- results, err = ListAll(arv, "groups", arvadosclient.Dict{
- "filters": [][]interface{}{
- {"uuid", "in", uuidList},
- {"owner_uuid", "=", parentGroup.UUID},
- },
- }, &groupList{})
+ params = arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "uuid",
+ Operator: "in",
+ Operand: uuidList,
+ }, {
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: parentGroup.UUID,
+ }},
+ }
+ results, err = ListAll(ac, "groups", params, &groupList{})
if err != nil {
return fmt.Errorf("error getting remote groups by UUID: %s", err)
}
for _, item := range results {
group := item.(group)
- results, err := ListAll(arv, "links", arvadosclient.Dict{
- "filters": [][]string{
- {"owner_uuid", "=", sysUserUUID},
- {"link_class", "=", "permission"},
- {"name", "=", "can_read"},
- {"tail_uuid", "=", group.UUID},
- {"head_kind", "=", "arvados#user"},
- },
- }, &linkList{})
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: sysUserUUID,
+ }, {
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "name",
+ Operator: "=",
+ Operand: "can_read",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: "arvados#user",
+ }},
+ }
+ results, err := ListAll(ac, "links", params, &linkList{})
if err != nil {
return fmt.Errorf("error getting member links for group %q: %s", group.Name, err)
}
@@ -339,8 +370,8 @@ func doMain() error {
if err != nil {
return fmt.Errorf("error reading %q: %s", *srcPath, err)
}
- groupName := record[0]
- groupMember := record[1] // User ID (username or email)
+ groupName := strings.TrimSpace(record[0])
+ groupMember := strings.TrimSpace(record[1]) // User ID (username or email)
if groupName == "" || groupMember == "" {
log.Printf("Warning: CSV record has at least one field empty (%s, %s). Skipping", groupName, groupMember)
membershipsSkipped++
@@ -357,30 +388,28 @@ func doMain() error {
if *verbose {
log.Printf("Remote group %q not found, creating...", groupName)
}
- var group group
- if err := arv.Create("groups", arvadosclient.Dict{
- "group": arvadosclient.Dict{
- "name": groupName,
- "owner_uuid": parentGroup.UUID,
- },
- }, &group); err != nil {
+ var newGroup group
+ groupData := map[string]string{
+ "name": groupName,
+ "owner_uuid": parentGroup.UUID,
+ }
+ if err := ac.RequestAndDecode(&newGroup, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil); err != nil {
return fmt.Errorf("error creating group named %q: %s", groupName, err)
}
- link := make(map[string]interface{})
- if err = arv.Create("links", arvadosclient.Dict{
- "link": arvadosclient.Dict{
- "owner_uuid": sysUserUUID,
- "link_class": "tag",
- "name": groupTag,
- "head_uuid": group.UUID,
- },
- }, &link); err != nil {
- return fmt.Errorf("error creating tag for newly created group %q (%s): %s", groupName, group.UUID, err)
+ var newLink link
+ linkData := map[string]string{
+ "owner_uuid": sysUserUUID,
+ "link_class": "tag",
+ "name": groupTag,
+ "head_uuid": newGroup.UUID,
+ }
+ if err = ac.RequestAndDecode(&newLink, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil); err != nil {
+ return fmt.Errorf("error creating tag for newly created group %q (%s): %s", newGroup.Name, newGroup.UUID, err)
}
// Update cached group data
- groupNameToUUID[groupName] = group.UUID
- remoteGroups[group.UUID] = &groupInfo{
- Group: group,
+ groupNameToUUID[groupName] = newGroup.UUID
+ remoteGroups[newGroup.UUID] = &groupInfo{
+ Group: newGroup,
PreviousMembers: make(map[string]bool), // Empty set
CurrentMembers: make(map[string]bool), // Empty set
}
@@ -394,28 +423,26 @@ func doMain() error {
log.Printf("Adding %q to group %q", groupMember, groupName)
}
// User wasn't a member, but should.
- link := make(map[string]interface{})
- if err := arv.Create("links", arvadosclient.Dict{
- "link": arvadosclient.Dict{
- "owner_uuid": sysUserUUID,
- "link_class": "permission",
- "name": "can_read",
- "tail_uuid": groupUUID,
- "head_uuid": userIDToUUID[groupMember],
- },
- }, &link); err != nil {
- return fmt.Errorf("error adding read group %q -> user %q permission: %s", groupName, groupMember, err)
+ var newLink link
+ linkData := map[string]string{
+ "owner_uuid": sysUserUUID,
+ "link_class": "permission",
+ "name": "can_read",
+ "tail_uuid": groupUUID,
+ "head_uuid": userIDToUUID[groupMember],
+ }
+ if err := ac.RequestAndDecode(&newLink, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil); err != nil {
+ return fmt.Errorf("error adding group %q -> user %q read permission: %s", groupName, groupMember, err)
+ }
+ linkData = map[string]string{
+ "owner_uuid": sysUserUUID,
+ "link_class": "permission",
+ "name": "manage",
+ "tail_uuid": userIDToUUID[groupMember],
+ "head_uuid": groupUUID,
}
- if err = arv.Create("links", arvadosclient.Dict{
- "link": arvadosclient.Dict{
- "owner_uuid": sysUserUUID,
- "link_class": "permission",
- "name": "manage",
- "tail_uuid": userIDToUUID[groupMember],
- "head_uuid": groupUUID,
- },
- }, &link); err != nil {
- return fmt.Errorf("error adding manage user %q -> group %q permission: %s", groupMember, groupName, err)
+ if err = ac.RequestAndDecode(&newLink, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil); err != nil {
+ return fmt.Errorf("error adding user %q -> group %q manage permission: %s", groupMember, groupName, err)
}
membershipsAdded++
}
@@ -436,16 +463,37 @@ func doMain() error {
}
var links []interface{}
// Search for all group<->user links (both ways)
- for _, filter := range [][][]string{
+ for _, filterset := range [][]arvados.Filter{
// Group -> User
- {{"link_class", "=", "permission"},
- {"tail_uuid", "=", groupUUID},
- {"head_uuid", "=", userIDToUUID[evictedUser]}},
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: groupUUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: userIDToUUID[evictedUser],
+ }},
// Group <- User
- {{"link_class", "=", "permission"},
- {"tail_uuid", "=", userIDToUUID[evictedUser]},
- {"head_uuid", "=", groupUUID}}} {
- l, err := ListAll(arv, "links", arvadosclient.Dict{"filters": filter}, &linkList{})
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: userIDToUUID[evictedUser],
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: groupUUID,
+ }},
+ } {
+ l, err := ListAll(ac, "links", arvados.ResourceListParams{Filters: filterset}, &linkList{})
if err != nil {
return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", evictedUser, groupName, err)
}
@@ -455,11 +503,10 @@ func doMain() error {
}
for _, item := range links {
link := item.(link)
- var l map[string]interface{}
if *verbose {
log.Printf("Removing %q from group %q", evictedUser, gi.Group.Name)
}
- if err := arv.Delete("links", link.UUID, arvadosclient.Dict{}, &l); err != nil {
+ if err := ac.RequestAndDecode(&link, "DELETE", "/arvados/v1/links"+link.UUID, nil, nil); err != nil {
return fmt.Errorf("error removing user %q from group %q: %s", evictedUser, groupName, err)
}
}
@@ -472,23 +519,24 @@ func doMain() error {
}
// ListAll : Adds all objects of type 'resource' to the 'allItems' list
-func ListAll(arv *arvadosclient.ArvadosClient, res string, params arvadosclient.Dict, rl resourceList) (allItems []interface{}, err error) {
+func ListAll(c *arvados.Client, res string, params arvados.ResourceListParams, page resourceList) (allItems []interface{}, err error) {
// Use the maximum page size the server allows
limit := 1<<31 - 1
- params["limit"] = limit
- params["offset"] = 0
- params["order"] = "uuid"
+ params.Limit = &limit
+ params.Offset = 0
+ params.Order = "uuid"
for {
- if err = arv.List(res, params, &rl); err != nil {
+ if err = c.RequestAndDecode(&page, "GET", "/arvados/v1/"+res, nil, params); err != nil {
return allItems, err
}
- if rl.Len() == 0 {
+ // Have we finished paging?
+ if page.Len() == 0 {
break
}
- for _, i := range rl.GetItems() {
+ for _, i := range page.GetItems() {
allItems = append(allItems, i)
}
- params["offset"] = params["offset"].(int) + rl.Len()
+ params.Offset += page.Len()
}
return allItems, nil
}
@@ -502,3 +550,13 @@ func subtract(setA map[string]bool, setB map[string]bool) map[string]bool {
}
return result
}
+
+func jsonReader(rscName string, ob interface{}) io.Reader {
+ j, err := json.Marshal(ob)
+ if err != nil {
+ panic(err)
+ }
+ v := url.Values{}
+ v[rscName] = []string{string(j)}
+ return bytes.NewBufferString(v.Encode())
+}
commit 92df2dab0bbb70c0b5ef99bac78d1a322b20648e
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Fri Oct 20 11:33:04 2017 -0300
12018: resourceList interface simplification. Code re-stying.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index d7a2bda..a40dcfc 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -18,9 +18,8 @@ import (
)
type resourceList interface {
- items() []interface{}
- itemsAvailable() int
- offset() int
+ Len() int
+ GetItems() []interface{}
}
type groupInfo struct {
@@ -48,25 +47,18 @@ func (u user) GetID(idSelector string) (string, error) {
// userList implements resourceList interface
type userList struct {
- Items []user `json:"items"`
- ItemsAvailable int `json:"items_available"`
- Offset int `json:"offset"`
+ Items []user `json:"items"`
}
-func (l userList) items() []interface{} {
- var out []interface{}
+func (l userList) Len() int {
+ return len(l.Items)
+}
+
+func (l userList) GetItems() (out []interface{}) {
for _, item := range l.Items {
out = append(out, item)
}
- return out
-}
-
-func (l userList) itemsAvailable() int {
- return l.ItemsAvailable
-}
-
-func (l userList) offset() int {
- return l.Offset
+ return
}
type group struct {
@@ -77,25 +69,18 @@ type group struct {
// groupList implements resourceList interface
type groupList struct {
- Items []group `json:"items"`
- ItemsAvailable int `json:"items_available"`
- Offset int `json:"offset"`
+ Items []group `json:"items"`
+}
+
+func (l groupList) Len() int {
+ return len(l.Items)
}
-func (l groupList) items() []interface{} {
- var out []interface{}
+func (l groupList) GetItems() (out []interface{}) {
for _, item := range l.Items {
out = append(out, item)
}
- return out
-}
-
-func (l groupList) itemsAvailable() int {
- return l.ItemsAvailable
-}
-
-func (l groupList) offset() int {
- return l.Offset
+ return
}
type link struct {
@@ -110,30 +95,22 @@ type link struct {
// linkList implements resourceList interface
type linkList struct {
- Items []link `json:"items"`
- ItemsAvailable int `json:"items_available"`
- Offset int `json:"offset"`
+ Items []link `json:"items"`
+}
+
+func (l linkList) Len() int {
+ return len(l.Items)
}
-func (l linkList) items() []interface{} {
- var out []interface{}
+func (l linkList) GetItems() (out []interface{}) {
for _, item := range l.Items {
out = append(out, item)
}
- return out
-}
-
-func (l linkList) itemsAvailable() int {
- return l.ItemsAvailable
-}
-
-func (l linkList) offset() int {
- return l.Offset
+ return
}
func main() {
- err := doMain()
- if err != nil {
+ if err := doMain(); err != nil {
log.Fatalf("%v", err)
}
}
@@ -218,12 +195,11 @@ func doMain() error {
if *parentGroupUUID == "" {
// UUID not provided, search for preexisting parent group
var gl groupList
- err := arv.List("groups", arvadosclient.Dict{
+ if err := arv.List("groups", arvadosclient.Dict{
"filters": [][]string{
{"name", "=", remoteGroupParentName},
{"owner_uuid", "=", sysUserUUID}},
- }, &gl)
- if err != nil {
+ }, &gl); err != nil {
return fmt.Errorf("error searching for parent group: %s", err)
}
if len(gl.Items) == 0 {
@@ -231,12 +207,11 @@ func doMain() error {
if *verbose {
log.Println("Default parent group not found, creating...")
}
- err := arv.Create("groups", arvadosclient.Dict{
+ if err := arv.Create("groups", arvadosclient.Dict{
"group": arvadosclient.Dict{
"name": remoteGroupParentName,
"owner_uuid": sysUserUUID},
- }, &parentGroup)
- if err != nil {
+ }, &parentGroup); err != nil {
return fmt.Errorf("error creating system user owned group named %q: %s", remoteGroupParentName, err)
}
} else if len(gl.Items) == 1 {
@@ -249,8 +224,7 @@ func doMain() error {
}
} else {
// UUID provided. Check if exists and if it's owned by system user
- err := arv.Get("groups", *parentGroupUUID, arvadosclient.Dict{}, &parentGroup)
- if err != nil {
+ if err := arv.Get("groups", *parentGroupUUID, arvadosclient.Dict{}, &parentGroup); err != nil {
return fmt.Errorf("error searching for parent group with UUID %q: %s", *parentGroupUUID, err)
}
if parentGroup.OwnerUUID != sysUserUUID {
@@ -384,25 +358,23 @@ func doMain() error {
log.Printf("Remote group %q not found, creating...", groupName)
}
var group group
- err := arv.Create("groups", arvadosclient.Dict{
+ if err := arv.Create("groups", arvadosclient.Dict{
"group": arvadosclient.Dict{
"name": groupName,
"owner_uuid": parentGroup.UUID,
},
- }, &group)
- if err != nil {
+ }, &group); err != nil {
return fmt.Errorf("error creating group named %q: %s", groupName, err)
}
link := make(map[string]interface{})
- err = arv.Create("links", arvadosclient.Dict{
+ if err = arv.Create("links", arvadosclient.Dict{
"link": arvadosclient.Dict{
"owner_uuid": sysUserUUID,
"link_class": "tag",
"name": groupTag,
"head_uuid": group.UUID,
},
- }, &link)
- if err != nil {
+ }, &link); err != nil {
return fmt.Errorf("error creating tag for newly created group %q (%s): %s", groupName, group.UUID, err)
}
// Update cached group data
@@ -423,7 +395,7 @@ func doMain() error {
}
// User wasn't a member, but should.
link := make(map[string]interface{})
- err := arv.Create("links", arvadosclient.Dict{
+ if err := arv.Create("links", arvadosclient.Dict{
"link": arvadosclient.Dict{
"owner_uuid": sysUserUUID,
"link_class": "permission",
@@ -431,11 +403,10 @@ func doMain() error {
"tail_uuid": groupUUID,
"head_uuid": userIDToUUID[groupMember],
},
- }, &link)
- if err != nil {
+ }, &link); err != nil {
return fmt.Errorf("error adding read group %q -> user %q permission: %s", groupName, groupMember, err)
}
- err = arv.Create("links", arvadosclient.Dict{
+ if err = arv.Create("links", arvadosclient.Dict{
"link": arvadosclient.Dict{
"owner_uuid": sysUserUUID,
"link_class": "permission",
@@ -443,8 +414,7 @@ func doMain() error {
"tail_uuid": userIDToUUID[groupMember],
"head_uuid": groupUUID,
},
- }, &link)
- if err != nil {
+ }, &link); err != nil {
return fmt.Errorf("error adding manage user %q -> group %q permission: %s", groupMember, groupName, err)
}
membershipsAdded++
@@ -489,8 +459,7 @@ func doMain() error {
if *verbose {
log.Printf("Removing %q from group %q", evictedUser, gi.Group.Name)
}
- err := arv.Delete("links", link.UUID, arvadosclient.Dict{}, &l)
- if err != nil {
+ if err := arv.Delete("links", link.UUID, arvadosclient.Dict{}, &l); err != nil {
return fmt.Errorf("error removing user %q from group %q: %s", evictedUser, groupName, err)
}
}
@@ -503,24 +472,23 @@ func doMain() error {
}
// ListAll : Adds all objects of type 'resource' to the 'allItems' list
-func ListAll(arv *arvadosclient.ArvadosClient, resource string, parameters arvadosclient.Dict, rl resourceList) (allItems []interface{}, err error) {
+func ListAll(arv *arvadosclient.ArvadosClient, res string, params arvadosclient.Dict, rl resourceList) (allItems []interface{}, err error) {
// Use the maximum page size the server allows
limit := 1<<31 - 1
- parameters["limit"] = limit
- parameters["offset"] = 0
- parameters["order"] = "uuid"
+ params["limit"] = limit
+ params["offset"] = 0
+ params["order"] = "uuid"
for {
- err = arv.List(resource, parameters, &rl)
- if err != nil {
+ if err = arv.List(res, params, &rl); err != nil {
return allItems, err
}
- if len(rl.items()) == 0 {
+ if rl.Len() == 0 {
break
}
- for _, i := range rl.items() {
+ for _, i := range rl.GetItems() {
allItems = append(allItems, i)
}
- parameters["offset"] = rl.offset() + len(rl.items())
+ params["offset"] = params["offset"].(int) + rl.Len()
}
return allItems, nil
}
commit e4e97733a237b45ea7202946c7fd20935bc47a2a
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Thu Oct 19 23:32:59 2017 -0300
12018: Simplify ListAll()
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index 9cddae4..d7a2bda 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -502,25 +502,25 @@ func doMain() error {
return nil
}
-// ListAll : Adds all objects of type 'resource' to the 'output' list
+// ListAll : Adds all objects of type 'resource' to the 'allItems' list
func ListAll(arv *arvadosclient.ArvadosClient, resource string, parameters arvadosclient.Dict, rl resourceList) (allItems []interface{}, err error) {
- if _, ok := parameters["limit"]; !ok {
- // Default limit value: use the maximum page size the server allows
- parameters["limit"] = 1<<31 - 1
- }
- offset := 0
- itemsAvailable := parameters["limit"].(int)
- for len(allItems) < itemsAvailable {
- parameters["offset"] = offset
+ // Use the maximum page size the server allows
+ limit := 1<<31 - 1
+ parameters["limit"] = limit
+ parameters["offset"] = 0
+ parameters["order"] = "uuid"
+ for {
err = arv.List(resource, parameters, &rl)
if err != nil {
return allItems, err
}
+ if len(rl.items()) == 0 {
+ break
+ }
for _, i := range rl.items() {
allItems = append(allItems, i)
}
- offset = rl.offset() + len(rl.items())
- itemsAvailable = rl.itemsAvailable()
+ parameters["offset"] = rl.offset() + len(rl.items())
}
return allItems, nil
}
commit c8decf3f2f88611166603ca48677470b478c06a6
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Thu Oct 19 21:41:25 2017 -0300
12018: Manage two way links for group memberships.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index 1e3a57c..9cddae4 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -326,12 +326,16 @@ func doMain() error {
},
}, &linkList{})
if err != nil {
- return fmt.Errorf("error getting member links: %s", err)
+ return fmt.Errorf("error getting member links for group %q: %s", group.Name, err)
}
// Build a list of user ids (email or username) belonging to this group
membersSet := make(map[string]bool)
for _, item := range results {
link := item.(link)
+ // We may receive an old link pointing to a removed user
+ if _, found := allUsers[link.HeadUUID]; !found {
+ continue
+ }
memberID, err := allUsers[link.HeadUUID].GetID(*userID)
if err != nil {
return err
@@ -429,7 +433,19 @@ func doMain() error {
},
}, &link)
if err != nil {
- return fmt.Errorf("error adding user %q to group %q: %s", groupMember, groupName, err)
+ return fmt.Errorf("error adding read group %q -> user %q permission: %s", groupName, groupMember, err)
+ }
+ err = arv.Create("links", arvadosclient.Dict{
+ "link": arvadosclient.Dict{
+ "owner_uuid": sysUserUUID,
+ "link_class": "permission",
+ "name": "manage",
+ "tail_uuid": userIDToUUID[groupMember],
+ "head_uuid": groupUUID,
+ },
+ }, &link)
+ if err != nil {
+ return fmt.Errorf("error adding manage user %q -> group %q permission: %s", groupMember, groupName, err)
}
membershipsAdded++
}
@@ -442,23 +458,30 @@ func doMain() error {
evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers)
groupName := gi.Group.Name
if len(evictedMembers) > 0 {
- log.Printf("Removing %d users from group %q: %v", len(evictedMembers), groupName, evictedMembers)
+ log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
}
for evictedUser := range evictedMembers {
if *verbose {
- log.Printf("Getting group membership link for user %q (%s) on group %q (%s)", evictedUser, userIDToUUID[evictedUser], groupName, groupUUID)
+ log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", evictedUser, userIDToUUID[evictedUser], groupName, groupUUID)
}
- links, err := ListAll(arv, "links", arvadosclient.Dict{
- "filters": [][]string{
- {"owner_uuid", "=", sysUserUUID},
- {"link_class", "=", "permission"},
- {"name", "=", "can_read"},
+ var links []interface{}
+ // Search for all group<->user links (both ways)
+ for _, filter := range [][][]string{
+ // Group -> User
+ {{"link_class", "=", "permission"},
{"tail_uuid", "=", groupUUID},
- {"head_uuid", "=", userIDToUUID[evictedUser]},
- },
- }, &linkList{})
- if err != nil {
- return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", evictedUser, groupName, err)
+ {"head_uuid", "=", userIDToUUID[evictedUser]}},
+ // Group <- User
+ {{"link_class", "=", "permission"},
+ {"tail_uuid", "=", userIDToUUID[evictedUser]},
+ {"head_uuid", "=", groupUUID}}} {
+ l, err := ListAll(arv, "links", arvadosclient.Dict{"filters": filter}, &linkList{})
+ if err != nil {
+ return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", evictedUser, groupName, err)
+ }
+ for _, link := range l {
+ links = append(links, link)
+ }
}
for _, item := range links {
link := item.(link)
commit b3667766f108542aa3d0e479e4a1179a47f2a653
Author: Lucas Di Pentima <ldipentima at veritasgenetics.com>
Date: Thu Oct 19 19:06:47 2017 -0300
12018: Various updates, described below:
* Code styling.
* Added system_user as permission links owner.
* Improved userId option handling.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima at veritasgenetics.com>
diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go
index 58fa775..1e3a57c 100644
--- a/tools/arv-sync-groups/arv-sync-groups.go
+++ b/tools/arv-sync-groups/arv-sync-groups.go
@@ -141,38 +141,34 @@ func main() {
func doMain() error {
const groupTag string = "remote_group"
const remoteGroupParentName string = "Externally synchronized groups"
- userIDOpts := []string{"email", "username"}
+ // Acceptable attributes to identify a user on the CSV file
+ userIDOpts := map[string]bool{
+ "email": true, // default
+ "username": true,
+ }
+ // Command arguments
flags := flag.NewFlagSet("arv-sync-groups", flag.ExitOnError)
-
srcPath := flags.String(
"path",
"",
- "Local file path containing a CSV format.")
-
+ "Local file path containing a CSV format: GroupName,UserID")
userID := flags.String(
"user-id",
"email",
- "Attribute by which every user is identified. "+
- "Valid values are: email (the default) and username.")
-
+ "Attribute by which every user is identified. Valid values are: email (the default) and username.")
verbose := flags.Bool(
"verbose",
false,
- "Log informational messages. By default is deactivated.")
-
+ "Log informational messages. Off by default.")
retries := flags.Int(
"retries",
3,
- "Maximum number of times to retry server requests that encounter "+
- "temporary failures (e.g., server down). Default 3.")
-
+ "Maximum number of times to retry server requests that encounter temporary failures (e.g., server down). Default 3.")
parentGroupUUID := flags.String(
"parent-group-uuid",
"",
- "Use given group UUID as a parent for the remote groups. Should "+
- "be owned by the system user. If not specified, a group named '"+
- remoteGroupParentName+"' will be used (and created if nonexistant).")
+ "Use given group UUID as a parent for the remote groups. Should be owned by the system user. If not specified, a group named '"+remoteGroupParentName+"' will be used (and created if nonexistant).")
// Parse args; omit the first arg which is the command name
flags.Parse(os.Args[1:])
@@ -181,10 +177,16 @@ func doMain() error {
if *retries < 0 {
return fmt.Errorf("retry quantity must be >= 0")
}
-
if *srcPath == "" {
return fmt.Errorf("please provide a path to an input file")
}
+ if !userIDOpts[*userID] {
+ var options []string
+ for opt := range userIDOpts {
+ options = append(options, opt)
+ }
+ return fmt.Errorf("user ID must be one of: %s", strings.Join(options, ", "))
+ }
// Try opening the input file early, just in case there's problems.
f, err := os.Open(*srcPath)
@@ -193,17 +195,6 @@ func doMain() error {
}
defer f.Close()
- validUserID := false
- for _, opt := range userIDOpts {
- if *userID == opt {
- validUserID = true
- }
- }
- if !validUserID {
- return fmt.Errorf("user ID must be one of: %s",
- strings.Join(userIDOpts, ", "))
- }
-
// Arvados Client setup
ac := arvados.NewClientFromEnv()
arv, err := arvadosclient.New(ac)
@@ -294,6 +285,7 @@ func doMain() error {
remoteGroupUUIDs := make(map[string]bool)
results, err = ListAll(arv, "links", arvadosclient.Dict{
"filters": [][]string{
+ {"owner_uuid", "=", sysUserUUID},
{"link_class", "=", "tag"},
{"name", "=", groupTag},
{"head_kind", "=", "arvados#group"},
@@ -326,6 +318,7 @@ func doMain() error {
group := item.(group)
results, err := ListAll(arv, "links", arvadosclient.Dict{
"filters": [][]string{
+ {"owner_uuid", "=", sysUserUUID},
{"link_class", "=", "permission"},
{"name", "=", "can_read"},
{"tail_uuid", "=", group.UUID},
@@ -355,9 +348,9 @@ func doMain() error {
log.Printf("Found %d remote groups", len(remoteGroups))
groupsCreated := 0
- membersAdded := 0
- membersRemoved := 0
- membersSkipped := 0
+ membershipsAdded := 0
+ membershipsRemoved := 0
+ membershipsSkipped := 0
csvReader := csv.NewReader(f)
for {
@@ -372,13 +365,13 @@ func doMain() error {
groupMember := record[1] // User ID (username or email)
if groupName == "" || groupMember == "" {
log.Printf("Warning: CSV record has at least one field empty (%s, %s). Skipping", groupName, groupMember)
- membersSkipped++
+ membershipsSkipped++
continue
}
if _, found := userIDToUUID[groupMember]; !found {
// User not present on the system, skip.
log.Printf("Warning: there's no user with %s %q on the system, skipping.", *userID, groupMember)
- membersSkipped++
+ membershipsSkipped++
continue
}
if _, found := groupNameToUUID[groupName]; !found {
@@ -394,20 +387,19 @@ func doMain() error {
},
}, &group)
if err != nil {
- return fmt.Errorf("error creating group named %q: %s",
- groupName, err)
+ return fmt.Errorf("error creating group named %q: %s", groupName, err)
}
link := make(map[string]interface{})
err = arv.Create("links", arvadosclient.Dict{
"link": arvadosclient.Dict{
+ "owner_uuid": sysUserUUID,
"link_class": "tag",
"name": groupTag,
"head_uuid": group.UUID,
},
}, &link)
if err != nil {
- return fmt.Errorf("error creating tag for group %q: %s",
- groupName, err)
+ return fmt.Errorf("error creating tag for newly created group %q (%s): %s", groupName, group.UUID, err)
}
// Update cached group data
groupNameToUUID[groupName] = group.UUID
@@ -429,6 +421,7 @@ func doMain() error {
link := make(map[string]interface{})
err := arv.Create("links", arvadosclient.Dict{
"link": arvadosclient.Dict{
+ "owner_uuid": sysUserUUID,
"link_class": "permission",
"name": "can_read",
"tail_uuid": groupUUID,
@@ -436,10 +429,9 @@ func doMain() error {
},
}, &link)
if err != nil {
- return fmt.Errorf("error adding user %q to group %q: %s",
- groupMember, groupName, err)
+ return fmt.Errorf("error adding user %q to group %q: %s", groupMember, groupName, err)
}
- membersAdded++
+ membershipsAdded++
}
gi.CurrentMembers[groupMember] = true
}
@@ -450,11 +442,15 @@ func doMain() error {
evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers)
groupName := gi.Group.Name
if len(evictedMembers) > 0 {
- log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
+ log.Printf("Removing %d users from group %q: %v", len(evictedMembers), groupName, evictedMembers)
}
for evictedUser := range evictedMembers {
+ if *verbose {
+ log.Printf("Getting group membership link for user %q (%s) on group %q (%s)", evictedUser, userIDToUUID[evictedUser], groupName, groupUUID)
+ }
links, err := ListAll(arv, "links", arvadosclient.Dict{
"filters": [][]string{
+ {"owner_uuid", "=", sysUserUUID},
{"link_class", "=", "permission"},
{"name", "=", "can_read"},
{"tail_uuid", "=", groupUUID},
@@ -475,10 +471,10 @@ func doMain() error {
return fmt.Errorf("error removing user %q from group %q: %s", evictedUser, groupName, err)
}
}
- membersRemoved++
+ membershipsRemoved++
}
}
- log.Printf("Groups created: %d, members added: %d, members removed: %d, members skipped: %d", groupsCreated, membersAdded, membersRemoved, membersSkipped)
+ log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped)
return nil
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list