[ARVADOS] created: 1.3.0-2581-g0ad80a159
Git user
git at public.arvados.org
Thu May 21 19:48:35 UTC 2020
at 0ad80a1594e583be9821feb8369e8dc4f619ab65 (commit)
commit 0ad80a1594e583be9821feb8369e8dc4f619ab65
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu May 21 15:48:19 2020 -0400
16427: Add arvados-server undelete command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index fcea2223d..1b2de11ac 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -15,6 +15,7 @@ import (
"git.arvados.org/arvados.git/lib/crunchrun"
"git.arvados.org/arvados.git/lib/dispatchcloud"
"git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/lib/undelete"
"git.arvados.org/arvados.git/services/ws"
)
@@ -33,6 +34,7 @@ var (
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
"install": install.Command,
+ "undelete": undelete.Command,
"ws": ws.Command,
})
)
diff --git a/lib/undelete/cmd.go b/lib/undelete/cmd.go
new file mode 100644
index 000000000..4c7bc5a85
--- /dev/null
+++ b/lib/undelete/cmd.go
@@ -0,0 +1,211 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package undelete
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "strings"
+ "sync"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ logger := ctxlog.New(stderr, "text", "info")
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Error("fatal")
+ }
+ logger.Info("exiting")
+ }()
+
+ loader := config.NewLoader(stdin, logger)
+
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ loader.SetupFlags(flags)
+ loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ }
+
+ if len(flags.Args()) == 0 {
+ fmt.Fprintf(stderr, "Usage: %s [options] uuid_or_file ...\n", prog)
+ flags.PrintDefaults()
+ return 2
+ }
+
+ lvl, err := logrus.ParseLevel(*loglevel)
+ if err != nil {
+ return 2
+ }
+ logger.SetLevel(lvl)
+
+ cfg, err := loader.Load()
+ if err != nil {
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return 1
+ }
+ client, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return 1
+ }
+ client.AuthToken = cluster.SystemRootToken
+ und := undeleter{
+ client: client,
+ cluster: cluster,
+ logger: logger,
+ }
+
+ exitcode := 0
+ for _, src := range flags.Args() {
+ logger := logger.WithField("src", src)
+ if len(src) == 27 && src[5:12] == "-57u5n-" {
+ logger.Error("log entry lookup not implemented")
+ exitcode = 1
+ continue
+ } else {
+ mtxt, err := ioutil.ReadFile(src)
+ if err != nil {
+ logger.WithError(err).Error("error loading manifest data")
+ exitcode = 1
+ continue
+ }
+ err = und.RecoverManifest(string(mtxt))
+ if err != nil {
+ logger.WithError(err).Error("recovery failed")
+ exitcode = 1
+ continue
+ }
+ logger.WithError(err).Info("recovery succeeded")
+ }
+ }
+ return exitcode
+}
+
+type undeleter struct {
+ client *arvados.Client
+ cluster *arvados.Cluster
+ logger logrus.FieldLogger
+}
+
+func (und undeleter) RecoverManifest(mtxt string) error {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ coll := arvados.Collection{ManifestText: mtxt}
+ blks, err := coll.SizedDigests()
+ if err != nil {
+ return err
+ }
+ todo := make(chan int, len(blks))
+ for idx := range blks {
+ todo <- idx
+ }
+ go close(todo)
+
+ var services []arvados.KeepService
+ err = und.client.EachKeepService(func(svc arvados.KeepService) error {
+ if svc.ServiceType == "proxy" {
+ und.logger.WithField("service", svc).Debug("ignore proxy service")
+ } else {
+ services = append(services, svc)
+ }
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("error getting list of keep services: %s", err)
+ }
+ und.logger.WithField("services", services).Debug("got list of services")
+
+ blkFound := make([]bool, len(blks))
+ var wg sync.WaitGroup
+ for i := 0; i < 2*len(services); i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ nextblk:
+ for idx := range todo {
+ blk := strings.SplitN(string(blks[idx]), "+", 2)[0]
+ logger := und.logger.WithField("block", blk)
+ for _, svc := range services {
+ logger := logger.WithField("service", fmt.Sprintf("%s:%d", svc.ServiceHost, svc.ServicePort))
+ if found, err := svc.Index(und.client, blk); err != nil {
+ logger.WithError(err).Warn("error getting index")
+ } else if len(found) > 0 {
+ blkFound[idx] = true
+ logger.Debug("found")
+ continue nextblk
+ } else {
+ logger.Debug("not found")
+ }
+ }
+ for _, svc := range services {
+ logger := logger.WithField("service", fmt.Sprintf("%s:%d", svc.ServiceHost, svc.ServicePort))
+ if err := svc.Untrash(ctx, und.client, blk); err != nil {
+ logger.WithError(err).Debug("untrash failed")
+ } else {
+ blkFound[idx] = true
+ logger.Info("untrashed")
+ continue nextblk
+ }
+ }
+ logger.Debug("unrecoverable")
+ }
+ }()
+ }
+ wg.Wait()
+
+ var have, havenot int
+ for _, ok := range blkFound {
+ if ok {
+ have++
+ } else {
+ havenot++
+ }
+ }
+ if havenot > 0 {
+ if have > 0 {
+ und.logger.Warn("partial recovery is not implemented")
+ }
+ return fmt.Errorf("unable to recover %d of %d blocks", havenot, have+havenot)
+ }
+
+ if und.cluster.Collections.BlobSigning {
+ ttl := und.cluster.Collections.BlobSigningTTL.Duration()
+ key := []byte(und.cluster.Collections.BlobSigningKey)
+ coll.ManifestText = arvados.SignManifest(coll.ManifestText, und.client.AuthToken, ttl, key)
+ }
+ und.logger.Info(coll.ManifestText)
+ err = und.client.RequestAndDecodeContext(ctx, &coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "manifest_text": coll.ManifestText,
+ },
+ })
+ if err != nil {
+ return fmt.Errorf("error saving new collection: %s", err)
+ }
+ und.logger.WithField("UUID", coll.UUID).Info("created new collection")
+ return nil
+}
diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go
index 97a62fa7b..a2263d179 100644
--- a/sdk/go/arvados/keep_service.go
+++ b/sdk/go/arvados/keep_service.go
@@ -6,7 +6,9 @@ package arvados
import (
"bufio"
+ "context"
"fmt"
+ "io/ioutil"
"net/http"
"strconv"
"strings"
@@ -102,6 +104,24 @@ func (s *KeepService) Mounts(c *Client) ([]KeepMount, error) {
return mounts, nil
}
+// Untrash moves/copies the given block out of trash.
+func (s *KeepService) Untrash(ctx context.Context, c *Client, blk string) error {
+ req, err := http.NewRequest("PUT", s.url("untrash/"+blk), nil)
+ if err != nil {
+ return err
+ }
+ resp, err := c.Do(req.WithContext(ctx))
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, _ := ioutil.ReadAll(resp.Body)
+ return fmt.Errorf("%s %s: %s", resp.Proto, resp.Status, body)
+ }
+ return nil
+}
+
// Index returns an unsorted list of blocks at the given mount point.
func (s *KeepService) IndexMount(c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
return s.index(c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
commit 74afa26b1ab0349f5a07f0cd88a9ed0e7e5e9545
Author: Tom Clegg <tom at tomclegg.ca>
Date: Thu May 21 09:48:24 2020 -0400
16427: Move signing code to sdk/go/arvados, add SignManifest.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/sdk/go/keepclient/perms.go b/sdk/go/arvados/blob_signature.go
similarity index 86%
copy from sdk/go/keepclient/perms.go
copy to sdk/go/arvados/blob_signature.go
index a77983322..4a936026f 100644
--- a/sdk/go/keepclient/perms.go
+++ b/sdk/go/arvados/blob_signature.go
@@ -6,7 +6,7 @@
//
// See https://dev.arvados.org/projects/arvados/wiki/Keep_locator_format
-package keepclient
+package arvados
import (
"crypto/hmac"
@@ -46,6 +46,24 @@ func makePermSignature(blobHash, apiToken, expiry, blobSignatureTTL string, perm
return fmt.Sprintf("%x", digest)
}
+var (
+ mBlkRe = regexp.MustCompile(`^[0-9a-f]{32}.*`)
+ mPermHintRe = regexp.MustCompile(`\+A[^+]*`)
+)
+
+// SignManifest signs all locators in the given manifest, discarding
+// any existing signatures.
+func SignManifest(manifest string, apiToken string, ttl time.Duration, permissionSecret []byte) string {
+ expiry := time.Now().Add(ttl)
+ return regexp.MustCompile(`\S+`).ReplaceAllStringFunc(manifest, func(tok string) string {
+ if mBlkRe.MatchString(tok) {
+ return SignLocator(mPermHintRe.ReplaceAllString(tok, ""), apiToken, expiry, ttl, permissionSecret)
+ } else {
+ return tok
+ }
+ })
+}
+
// SignLocator returns blobLocator with a permission signature
// added. If either permissionSecret or apiToken is empty, blobLocator
// is returned untouched.
diff --git a/sdk/go/keepclient/perms.go b/sdk/go/keepclient/perms.go
index a77983322..23ca7d2f2 100644
--- a/sdk/go/keepclient/perms.go
+++ b/sdk/go/keepclient/perms.go
@@ -2,108 +2,15 @@
//
// SPDX-License-Identifier: Apache-2.0
-// Generate and verify permission signatures for Keep locators.
-//
-// See https://dev.arvados.org/projects/arvados/wiki/Keep_locator_format
-
package keepclient
-import (
- "crypto/hmac"
- "crypto/sha1"
- "errors"
- "fmt"
- "regexp"
- "strconv"
- "strings"
- "time"
-)
+import "git.arvados.org/arvados.git/sdk/go/arvados"
var (
- // ErrSignatureExpired - a signature was rejected because the
- // expiry time has passed.
- ErrSignatureExpired = errors.New("Signature expired")
- // ErrSignatureInvalid - a signature was rejected because it
- // was badly formatted or did not match the given secret key.
- ErrSignatureInvalid = errors.New("Invalid signature")
- // ErrSignatureMissing - the given locator does not have a
- // signature hint.
- ErrSignatureMissing = errors.New("Missing signature")
+ ErrSignatureExpired = arvados.ErrSignatureExpired
+ ErrSignatureInvalid = arvados.ErrSignatureInvalid
+ ErrSignatureMissing = arvados.ErrSignatureMissing
+ SignLocator = arvados.SignLocator
+ SignedLocatorRe = arvados.SignedLocatorRe
+ VerifySignature = arvados.VerifySignature
)
-
-// makePermSignature generates a SHA-1 HMAC digest for the given blob,
-// token, expiry, and site secret.
-func makePermSignature(blobHash, apiToken, expiry, blobSignatureTTL string, permissionSecret []byte) string {
- hmac := hmac.New(sha1.New, permissionSecret)
- hmac.Write([]byte(blobHash))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(apiToken))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(expiry))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(blobSignatureTTL))
- digest := hmac.Sum(nil)
- return fmt.Sprintf("%x", digest)
-}
-
-// SignLocator returns blobLocator with a permission signature
-// added. If either permissionSecret or apiToken is empty, blobLocator
-// is returned untouched.
-//
-// This function is intended to be used by system components and admin
-// utilities: userland programs do not know the permissionSecret.
-func SignLocator(blobLocator, apiToken string, expiry time.Time, blobSignatureTTL time.Duration, permissionSecret []byte) string {
- if len(permissionSecret) == 0 || apiToken == "" {
- return blobLocator
- }
- // Strip off all hints: only the hash is used to sign.
- blobHash := strings.Split(blobLocator, "+")[0]
- timestampHex := fmt.Sprintf("%08x", expiry.Unix())
- blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
- return blobLocator +
- "+A" + makePermSignature(blobHash, apiToken, timestampHex, blobSignatureTTLHex, permissionSecret) +
- "@" + timestampHex
-}
-
-var SignedLocatorRe = regexp.MustCompile(
- //1 2 34 5 6 7 89
- `^([[:xdigit:]]{32})(\+[0-9]+)?((\+[B-Z][A-Za-z0-9 at _-]*)*)(\+A([[:xdigit:]]{40})@([[:xdigit:]]{8}))((\+[B-Z][A-Za-z0-9 at _-]*)*)$`)
-
-// VerifySignature returns nil if the signature on the signedLocator
-// can be verified using the given apiToken. Otherwise it returns
-// ErrSignatureExpired (if the signature's expiry time has passed,
-// which is something the client could have figured out
-// independently), ErrSignatureMissing (if there is no signature hint
-// at all), or ErrSignatureInvalid (if the signature is present but
-// badly formatted or incorrect).
-//
-// This function is intended to be used by system components and admin
-// utilities: userland programs do not know the permissionSecret.
-func VerifySignature(signedLocator, apiToken string, blobSignatureTTL time.Duration, permissionSecret []byte) error {
- matches := SignedLocatorRe.FindStringSubmatch(signedLocator)
- if matches == nil {
- return ErrSignatureMissing
- }
- blobHash := matches[1]
- signatureHex := matches[6]
- expiryHex := matches[7]
- if expiryTime, err := parseHexTimestamp(expiryHex); err != nil {
- return ErrSignatureInvalid
- } else if expiryTime.Before(time.Now()) {
- return ErrSignatureExpired
- }
- blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
- if signatureHex != makePermSignature(blobHash, apiToken, expiryHex, blobSignatureTTLHex, permissionSecret) {
- return ErrSignatureInvalid
- }
- return nil
-}
-
-func parseHexTimestamp(timestampHex string) (ts time.Time, err error) {
- if tsInt, e := strconv.ParseInt(timestampHex, 16, 0); e == nil {
- ts = time.Unix(tsInt, 0)
- } else {
- err = e
- }
- return ts, err
-}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list